pax_global_header00006660000000000000000000000064147121157510014516gustar00rootroot0000000000000052 comment=6f4aaa79a3a9e13a20e8c4dd69555625388c73c1 dataiku-api-client-python-13.2.2/000077500000000000000000000000001471211575100165675ustar00rootroot00000000000000dataiku-api-client-python-13.2.2/.gitignore000066400000000000000000000000751471211575100205610ustar00rootroot00000000000000*.pyc .idea *.iml egg-info dist dataiku_api_client.egg-info/ dataiku-api-client-python-13.2.2/.travis.yml000066400000000000000000000001521471211575100206760ustar00rootroot00000000000000language: python python: - "2.7" - "3.4" install: "pip install -r requirements.txt" script: nosetests dataiku-api-client-python-13.2.2/HISTORY.txt000066400000000000000000000061121471211575100204710ustar00rootroot00000000000000Changelog ========== 13.2.2 (2024-11-04) --------------------- * Initial release for DSS 13.2.2 13.2.1 (2024-10-17) --------------------- * Initial release for DSS 13.2.1 13.2.0 (2024-10-10) --------------------- * Initial release for DSS 13.2.0 13.1.4 (2024-09-19) --------------------- * Initial release for DSS 13.1.4 13.1.3 (2024-09-16) --------------------- * Initial release for DSS 13.1.3 13.1.2 (2024-08-30) --------------------- * Initial release for DSS 13.1.2 13.1.1 (2024-08-27) --------------------- * Initial release for DSS 13.1.1 13.0.3 (2024-08-01) --------------------- * Initial release for DSS 13.0.3 13.0.2 (2024-07-25) --------------------- * Initial release for DSS 13.0.2 13.0.1 (2024-07-24) --------------------- * Initial release for DSS 13.0.1 13.0.0 (2024-06-26) --------------------- * Initial release for DSS 13.0.0 12.6.4 (2024-06-12) --------------------- * Initial release for DSS 12.6.4 12.6.3 (2024-05-31) --------------------- * Initial release for DSS 12.6.3 12.6.2 (2024-05-16) --------------------- * Initial release for DSS 12.6.2 12.6.0 (2024-04-09) --------------------- * Initial release for DSS 12.6.0 12.5.2 (2024-02-26) --------------------- * Initial release for DSS 12.5.2 12.5.0 (2024-01-23) --------------------- * Initial release for DSS 12.5.0 12.4.2 (2024-01-15) --------------------- * Initial release for DSS 12.4.2 12.4.0 (2023-12-06) --------------------- * Initial release for DSS 12.4.0 12.3.2 (2023-11-16) --------------------- * Initial release for DSS 12.3.2 12.2.3 (2023-10-12) --------------------- * Initial release for DSS 12.2.3 12.1.0 (2023-06-29) --------------------- * Initial release for DSS 12.1.0 12.0.1 (2023-06-29) --------------------- * Initial release for DSS 12.0.1 12.0.0 (2023-06-13) --------------------- * Initial release for DSS 12.0.0 11.4.1.1 (2023-04-06) --------------------- * Initial release for DSS 11.4.1.1 11.4.1 (2023-04-06) ------------------- * Initial release for DSS 11.4.1 11.4.0 (2023-03-16) ------------------- * Initial release for DSS 11.4.0 11.1.0 (2022-10-04) ------------------- * Initial release for DSS 11.1.0 11.0.0 (2022-07-12) ------------------- * Initial release for DSS 11.0.0 9.0.0 (2021-05-07) ------------------ * Initial release for DSS 9.0 8.0.0 (2020-08-14) ------------------- * Initial release for DSS 8.0 7.0.0 (2020-07-21) ------------------ * Initial release for DSS 7.0 5.1.0 (2019-03-01) ------------------ * Initial release for DSS 5.1 5.0.0 (2018-07-26) ------------------ * Initial release for DSS 5.0 4.3.0 (2018-06-01) ------------------ * Initial release for DSS 4.3 4.2.1 (2018-04-30) ------------------- * Initial release for DSS 4.2 4.1.0 (2017-11-10) ------------------- * Initial release for DSS 4.1 4.0.0 (2017-01-10) ------------------ * Initial release for DSS 4.0 3.1.4 (2017-01-03) ------------------- * Pypi packaging fixes 3.1.3 (2017-01-03) ------------------- * Fix user creation 3.1.2 (2016-08-23) ------------------- * Add proper Pypi tags 3.1.0 (2016-08-22) ------------------- * Initial release for DSS 3.1 dataiku-api-client-python-13.2.2/LICENSE.txt000066400000000000000000000010551471211575100204130ustar00rootroot00000000000000Copyright 2015-2021 Dataiku SAS Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. dataiku-api-client-python-13.2.2/MANIFEST.in000066400000000000000000000000721471211575100203240ustar00rootroot00000000000000include README include *.txt recursive-include dataikuapi dataiku-api-client-python-13.2.2/README000066400000000000000000000002171471211575100174470ustar00rootroot00000000000000API client for Dataiku Data Science Studio For more information, see: https://doc.dataiku.com/dss/latest/python-api/rest-api-client/index.htmldataiku-api-client-python-13.2.2/dataikuapi/000077500000000000000000000000001471211575100207035ustar00rootroot00000000000000dataiku-api-client-python-13.2.2/dataikuapi/__init__.py000066400000000000000000000012321471211575100230120ustar00rootroot00000000000000from .dssclient import DSSClient from .fmclient import FMClientAWS, FMClientAzure, FMClientGCP from .govern_client import GovernClient from .apinode_client import APINodeClient from .apinode_admin_client import APINodeAdminClient from .dss.recipe import GroupingRecipeCreator, JoinRecipeCreator, StackRecipeCreator, WindowRecipeCreator, SyncRecipeCreator, SamplingRecipeCreator, SQLQueryRecipeCreator, CodeRecipeCreator, SplitRecipeCreator, SortRecipeCreator, TopNRecipeCreator, DistinctRecipeCreator, DownloadRecipeCreator, PredictionScoringRecipeCreator, ClusteringScoringRecipeCreator from .dss.admin import DSSUserImpersonationRule, DSSGroupImpersonationRule dataiku-api-client-python-13.2.2/dataikuapi/apinode_admin/000077500000000000000000000000001471211575100234725ustar00rootroot00000000000000dataiku-api-client-python-13.2.2/dataikuapi/apinode_admin/__init__.py000066400000000000000000000000001471211575100255710ustar00rootroot00000000000000dataiku-api-client-python-13.2.2/dataikuapi/apinode_admin/auth.py000066400000000000000000000014611471211575100250070ustar00rootroot00000000000000 class APINodeAuth(object): """ A handle to interact with authentication settings on API node """ def __init__(self, client): self.client = client def list_keys(self): """Lists the Admin API keys""" return self.client._perform_json("GET", "keys") def add_key(self, label=None, description=None, created_by=None): """Add an Admin API key. Returns the key details""" key = { "label" : label, "description" : description, "createdBy" : created_by } return self.client._perform_json("POST", "keys", body=key) def delete_key(self, key): self.client._perform_empty("DELETE", "keys/%s" % key) def delete_key_by_id(self, id): self.client._perform_empty("DELETE", "keys/%s/key" % id)dataiku-api-client-python-13.2.2/dataikuapi/apinode_admin/service.py000066400000000000000000000041751471211575100255130ustar00rootroot00000000000000from os import path as osp class APINodeService(object): """ A handle to interact with the settings of an API node service """ def __init__(self, client, service_id): self.client = client self.service_id = service_id def delete(self): """Deletes the API node service""" self.client._perform_empty("DELETE", "services/%s" % self.service_id) ######################################################## # On-disk generations management ######################################################## def list_generations(self): return self.client._perform_json("GET", "services/%s/generations" % self.service_id)["generations"] def import_generation_from_archive(self, file_path): self.client._perform_empty("POST", "services/%s/generations/actions/importFromArchive" % (self.service_id), params = { "filePath" : osp.abspath(file_path) }) def preload_generation(self, generation): self.client._perform_empty("POST", "services/%s/generations/%s/preload" % (self.service_id, generation)) ######################################################## # Switch / mapping management ######################################################## def disable(self): """Disable the service.""" self.client._perform_empty("POST", "services/%s/actions/disable" % self.service_id) def enable(self): self.client._perform_empty("POST", "services/%s/actions/enable" % self.service_id) def set_generations_mapping(self, mapping): """Setting a generations mapping automatically enables the service""" self.client._perform_empty("POST", "services/%s/actions/setMapping" % self.service_id, body = mapping) def switch_to_newest(self): return self.client._perform_json("POST", "services/%s/actions/switchToNewest" % self.service_id) def switch_to_single_generation(self, generation): self.client._perform_empty("POST", "services/%s/actions/switchTo/%s" % (self.service_id, generation)) dataiku-api-client-python-13.2.2/dataikuapi/apinode_admin_client.py000066400000000000000000000076611471211575100254140ustar00rootroot00000000000000import json from .apinode_admin.service import APINodeService from .apinode_admin.auth import APINodeAuth from .utils import DataikuException from .base_client import DSSBaseClient class APINodeAdminClient(DSSBaseClient): """Entry point for the DSS APINode admin client""" def __init__(self, uri, api_key, insecure_tls=False): """ Instantiate a new DSS API client on the given base uri with the given API key. """ DSSBaseClient.__init__(self, "%s/%s" % (uri, "admin/api"), api_key, insecure_tls=insecure_tls) ######################################################## # Services generations ######################################################## def create_service(self, service_id): """ Creates a new API service :param service_id: id of the created API service """ self._perform_empty("POST", "services/", body = { "serviceId" : service_id }) def list_services(self): """ Lists the currently declared services and their enabled/disabled state :return: a dict of services containing their id and state, as a JSON object :rtype: dict """ return self._perform_json("GET", "services") def service(self, service_id): """ Gets a handle to interact with a service :param service_id: id of requested service :rtype: :class: `dataikuapi.apinode_admin.service.APINodeService` """ return APINodeService(self, service_id) def auth(self): """ Returns a handle to interact with authentication :rtype: :class: `dataikuapi.apinode_admin.auth.APINodeAuth` """ return APINodeAuth(self) def get_metrics(self): """ Get the metrics for this API Node :return: the metrics, as a JSON object :rtype: dict """ return self._perform_json("GET", "metrics") def import_code_env_in_cache(self, file_dir, language): """ Import a code env in global cache from an exported code env base folder :param file_dir: path of an exported code env base folder :param language: language of the code env (`python` or `R`) """ return self._perform_json("POST", "cached-code-envs", params={ "fileDir": file_dir, "language": language }) def register_code_env_in_cache(self, exported_env_dir, built_env_dir, language): """ Import a code env in global cache from an exported code env base folder :param exported_env_dir: path of an exported code env base folder :param built_env_dir: path where the code env was built and is available :param language: language of the code env (`python` or `R`) """ return self._perform_json("POST", "register-global-code-env", params={ "exportedEnvDir": exported_env_dir, "builtEnvDir": built_env_dir, "language": language }) def import_model_archive_in_cache(self, model_archive_path): """ Import a model in model cache from an exported model archive :param model_archive_path: path of an exported model archive """ return self._perform_json("POST", "model-cache", params={ "modelArchivePath": model_archive_path }) def clear_model_cache(self): """ Clear the model cache """ self._perform_empty("DELETE", "model-cache") def clean_unused_services_and_generations(self): """ Deletes disabled services, unused generations and unused code environments """ resp = self._perform_json("DELETE", "services-clean-unused") print(json.dumps(resp, indent=4)) def clean_code_env_cache(self): """ Deletes unused code envs from cache """ resp = self._perform_json("DELETE", "cached-code-envs") print(json.dumps(resp, indent=4)) dataiku-api-client-python-13.2.2/dataikuapi/apinode_client.py000066400000000000000000000306071471211575100242400ustar00rootroot00000000000000from .utils import DataikuException from .base_client import DSSBaseClient class APINodeClient(DSSBaseClient): """Entry point for the DSS API Node client This is an API client for the user-facing API of DSS API Node server (user facing API) """ def __init__(self, uri, service_id, api_key=None, bearer_token=None, insecure_tls=False): """ Instantiate a new DSS API client on the given base URI with the given API key. :param str uri: Base URI of the DSS API node server (http://host:port/ or https://host:port/) :param str service_id: Identifier of the service to query :param str api_key: Optional, API key for the service. Only required if the service has its authorization setup to API keys :param str bearer_token: Optional, The bearer token. Only required if the service has its authorization setup to OAuth2/JWT """ DSSBaseClient.__init__(self, "%s/%s" % (uri, "public/api/v1/%s" % service_id), api_key=api_key, bearer_token=bearer_token, insecure_tls=insecure_tls) @staticmethod def _set_dispatch(obj, forced_generation, dispatch_key): if forced_generation is not None: obj["dispatch"] = {"forcedGeneration": forced_generation} elif dispatch_key is not None: obj["dispatch"] = {"dispatchKey": dispatch_key} def predict_record(self, endpoint_id, features, forced_generation=None, dispatch_key=None, context=None, with_explanations=None, explanation_method=None, n_explanations=None, n_explanations_mc_steps=None): """ Predicts a single record on a DSS API node endpoint (standard or custom prediction) :param str endpoint_id: Identifier of the endpoint to query :param features: Python dictionary of features of the record :param forced_generation: See documentation about multi-version prediction :param dispatch_key: See documentation about multi-version prediction :param context: Optional, Python dictionary of additional context information. The context information is logged, but not directly used. :param with_explanations: Optional, whether individual explanations should be computed for each record. The prediction endpoint must be compatible. If None, will use the value configured in the endpoint. :param explanation_method: Optional, method to compute explanations. Valid values are 'SHAPLEY' or 'ICE'. If None, will use the value configured in the endpoint. :param n_explanations: Optional, number of explanations to output per prediction. If None, will use the value configured in the endpoint. :param n_explanations_mc_steps: Optional, precision parameter for SHAPLEY method, higher means more precise but slower (between 25 and 1000). If None, will use the value configured in the endpoint. :return: a Python dict of the API answer. The answer contains a "result" key (itself a dict) """ obj = { "features": features, "explanations": { "enabled": with_explanations, "method": explanation_method, "nExplanations": n_explanations, "nMonteCarloSteps": n_explanations_mc_steps } } self._set_dispatch(obj, forced_generation, dispatch_key) if context is not None: obj["context"] = context return self._perform_json("POST", "%s/predict" % endpoint_id, body = obj) def predict_records(self, endpoint_id, records, forced_generation=None, dispatch_key=None, with_explanations=None, explanation_method=None, n_explanations=None, n_explanations_mc_steps=None): """ Predicts a batch of records on a DSS API node endpoint (standard or custom prediction) :param str endpoint_id: Identifier of the endpoint to query :param records: Python list of records. Each record must be a Python dict. Each record must contain a "features" dict (see predict_record) and optionally a "context" dict. :param forced_generation: See documentation about multi-version prediction :param dispatch_key: See documentation about multi-version prediction :param with_explanations: Optional, whether individual explanations should be computed for each record. The prediction endpoint must be compatible. If None, will use the value configured in the endpoint. :param explanation_method: Optional, method to compute explanations. Valid values are 'SHAPLEY' or 'ICE'. If None, will use the value configured in the endpoint. :param n_explanations: Optional, number of explanations to output per prediction. If None, will use the value configured in the endpoint. :param n_explanations_mc_steps: Optional, precision parameter for SHAPLEY method, higher means more precise but slower (between 25 and 1000). If None, will use the value configured in the endpoint. :return: a Python dict of the API answer. The answer contains a "results" key (which is an array of result objects) """ for record in records: if not "features" in record: raise ValueError("Each record must contain a 'features' dict") obj = { "items": records, "explanations": { "enabled": with_explanations, "method": explanation_method, "nExplanations": n_explanations, "nMonteCarloSteps": n_explanations_mc_steps } } self._set_dispatch(obj, forced_generation, dispatch_key) return self._perform_json("POST", "%s/predict-multi" % endpoint_id, body = obj) def forecast(self, endpoint_id, records, forced_generation=None, dispatch_key=None): """ Forecast using a time series forecasting model on a DSS API node endpoint :param str endpoint_id: Identifier of the endpoint to query :param array records: List of time series data records to be used as an input for the time series forecasting model. Each record should be a dict where keys are feature names, and values feature values. Example: .. code-block:: python records = [ {'date': '2015-01-04T00:00:00.000Z', 'timeseries_id': 'A', 'target': 10.0}, {'date': '2015-01-04T00:00:00.000Z', 'timeseries_id': 'B', 'target': 4.5}, {'date': '2015-01-05T00:00:00.000Z', 'timeseries_id': 'A', 'target': 2.0}, ... {'date': '2015-03-20T00:00:00.000Z', 'timeseries_id': 'B', 'target': 1.3} ] :param forced_generation: See documentation about multi-version prediction :param dispatch_key: See documentation about multi-version prediction :return: a Python dict of the API answer. The answer contains a "results" key (which is an array of result objects, corresponding to the forecast records) Example: .. code-block:: python {'results': [ {'forecast': 12.57, 'ignored': False, 'quantiles': [0.0001, 0.5, 0.9999], 'quantilesValues': [3.0, 16.0, 16.0], 'time': '2015-03-21T00:00:00.000000Z', 'timeseriesIdentifier': {'timeseries_id': 'A'}}, {'forecast': 15.57, 'ignored': False, 'quantiles': [0.0001, 0.5, 0.9999], 'quantilesValues': [3.0, 18.0, 19.0], 'time': '2015-03-21T00:00:00.000000Z', 'timeseriesIdentifier': {'timeseries_id': 'B'}}, ...], ...} """ obj = {"items": records} self._set_dispatch(obj, forced_generation, dispatch_key) return self._perform_json("POST", "{}/forecast".format(endpoint_id), body=obj) def predict_effect(self, endpoint_id, features, forced_generation=None, dispatch_key=None): """ Predicts the treatment effect of a single record on a DSS API node endpoint (standard causal prediction) :param str endpoint_id: Identifier of the endpoint to query :param features: Python dictionary of features of the record :param forced_generation: See documentation about multi-version prediction :param dispatch_key: See documentation about multi-version prediction :return: a Python dict of the API answer. The answer contains a "result" key (itself a dict) """ obj = { "features": features, } self._set_dispatch(obj, forced_generation, dispatch_key) return self._perform_json("POST", "%s/predict-effect" % endpoint_id, body=obj) def predict_effects(self, endpoint_id, records, forced_generation=None, dispatch_key=None): """ Predicts the treatment effects on a batch of records on a DSS API node endpoint (standard causal prediction) :param str endpoint_id: Identifier of the endpoint to query :param records: Python list of records. Each record must be a Python dict. Each record must contain a "features" dict (see predict_record) and optionally a "context" dict. :param dispatch_key: See documentation about multi-version prediction :return: a Python dict of the API answer. The answer contains a "results" key (which is an array of result objects) """ for record in records: if not "features" in record: raise ValueError("Each record must contain a 'features' dict") obj = { "items": records, } self._set_dispatch(obj, forced_generation, dispatch_key) return self._perform_json("POST", "%s/predict-effect-multi" % endpoint_id, body = obj) def sql_query(self, endpoint_id, parameters): """ Queries a "SQL query" endpoint on a DSS API node :param str endpoint_id: Identifier of the endpoint to query :param parameters: Python dictionary of the named parameters for the SQL query endpoint :return: a Python dict of the API answer. The answer is the a dict with a columns field and a rows field (list of rows as list of strings) """ return self._perform_json("POST", "%s/query" % endpoint_id, body = parameters) def lookup_record(self, endpoint_id, record, context=None): """ Lookup a single record on a DSS API node endpoint of "dataset lookup" type :param str endpoint_id: Identifier of the endpoint to query :param record: Python dictionary of features of the record :param context: Optional, Python dictionary of additional context information. The context information is logged, but not directly used. :return: a Python dict of the API answer. The answer contains a "data" key (itself a dict) """ obj = { "data" :record } if context is not None: obj["context"] = context return self._perform_json("POST", "%s/lookup" % endpoint_id, body = obj).get("results", [])[0] def lookup_records(self, endpoint_id, records): """ Lookups a batch of records on a DSS API node endpoint of "dataset lookup" type :param str endpoint_id: Identifier of the endpoint to query :param records: Python list of records. Each record must be a Python dict, containing at least one entry called "data": a dict containing the input columns :return: a Python dict of the API answer. The answer contains a "results" key, which is an array of result objects. Each result contains a "data" dict which is the output """ for record in records: if not "data" in record: raise ValueError("Each record must contain a 'data' dict") obj = { "items" : records } return self._perform_json("POST", "%s/lookup-multi" % endpoint_id, body = obj) def run_function(self, endpoint_id, **kwargs): """ Calls a "Run function" endpoint on a DSS API node :param str endpoint_id: Identifier of the endpoint to query :param kwargs: Arguments of the function :return: The function result """ obj = {} for (k,v) in kwargs.items(): obj[k] = v return self._perform_json("POST", "%s/run" % endpoint_id, body = obj) dataiku-api-client-python-13.2.2/dataikuapi/auth.py000066400000000000000000000007501471211575100222200ustar00rootroot00000000000000from requests.auth import AuthBase class HTTPBearerAuth(AuthBase): """Attaches HTTP Bearer Authentication to the given Request object.""" def __init__(self, token): self.token = token def __eq__(self, other): return all([ self.token == getattr(other, 'token', None) ]) def __ne__(self, other): return not self == other def __call__(self, r): r.headers['Authorization'] = 'Bearer ' + self.token return r dataiku-api-client-python-13.2.2/dataikuapi/base_client.py000066400000000000000000000040331471211575100235250ustar00rootroot00000000000000import json from requests import Session, exceptions from requests import exceptions from requests.auth import HTTPBasicAuth from .auth import HTTPBearerAuth from .utils import DataikuException class DSSBaseClient(object): def __init__(self, base_uri, api_key=None, internal_ticket=None, bearer_token=None, insecure_tls=False): self.api_key = api_key self.bearer_token = bearer_token self.base_uri = base_uri self._session = Session() if insecure_tls: self._session.verify = False ######################################################## # Internal Request handling ######################################################## def _perform_http(self, method, path, params=None, body=None, stream=False): if body: body = json.dumps(body) headers = None auth = None if self.api_key: auth = HTTPBasicAuth(self.api_key, "") elif self.bearer_token: auth = HTTPBearerAuth(self.bearer_token) try: http_res = self._session.request( method, "%s/%s" % (self.base_uri, path), params=params, data=body, headers=headers, auth=auth, stream = stream) http_res.raise_for_status() return http_res except exceptions.HTTPError: ex = http_res.json() raise DataikuException("%s: %s" % (ex.get("errorType", "Unknown error"), ex.get("message", "No message"))) def _perform_empty(self, method, path, params=None, body=None): self._perform_http(method, path, params, body, False) def _perform_text(self, method, path, params=None, body=None): return self._perform_http(method, path, params, body, False).text def _perform_json(self, method, path, params=None, body=None): return self._perform_http(method, path, params, body, False).json() def _perform_raw(self, method, path, params=None, body=None): return self._perform_http(method, path, params, body, True) dataiku-api-client-python-13.2.2/dataikuapi/dss/000077500000000000000000000000001471211575100214745ustar00rootroot00000000000000dataiku-api-client-python-13.2.2/dataikuapi/dss/__init__.py000066400000000000000000000000001471211575100235730ustar00rootroot00000000000000dataiku-api-client-python-13.2.2/dataikuapi/dss/admin.py000066400000000000000000002602631471211575100231470ustar00rootroot00000000000000from .future import DSSFuture import json, warnings from datetime import datetime from ..utils import _timestamp_ms_to_zoned_datetime class DSSConnectionListItem(dict): """ An item in a list of connections. .. important:: Do not instantiate directly, use :meth:`dataikuapi.DSSClient.list_connections` instead. """ def __init__(self, client, data): super(DSSConnectionListItem, self).__init__(data) self.client = client def to_connection(self): """ Gets a handle corresponding to this item :rtype: :class:`DSSConnection` """ return DSSConnection(self.client, self["name"]) @property def name(self): """ Get the identifier of the connection. :rtype: string """ return self["id"] @property def type(self): """ Get the type of the connection. :return: a DSS connection type, like PostgreSQL, EC2, Azure, ... :rtype: string """ return self["label"] class DSSConnectionInfo(dict): """ A class holding read-only information about a connection. .. important:: Do not instantiate directly, use :meth:`DSSConnection.get_info` instead. The main use case of this class is to retrieve the decrypted credentials for a connection, if allowed by the connection permissions. Depending on the connection kind, the credential may be available using :meth:`get_basic_credential` or :meth:`get_aws_credential`. """ def __init__(self, data): super(DSSConnectionInfo, self).__init__(data) def get_type(self): """ Get the type of the connection :return: a connection type, for example Azure, Snowflake, GCS, ... :rtype: string """ return self["type"] def get_credential_mode(self): """ Get the credential mode of the connection :return: a connection mode :rtype: string """ return self["credentialsMode"] def get_params(self): """ Get the parameters of the connection, as a dict :return: the parameters, as a dict. Each connection type has different sets of fields. :rtype: dict """ return self["params"] def get_resolved_params(self): """ Get the resolved parameters of the connection, as a dict. May be null depending on the connection type. :return: the resolved parameters, as a dict. Each connection type has different sets of fields. :rtype: dict """ return self["resolvedParams"] def get_basic_credential(self): """ Get the basic credential (user/password pair) for this connection, if available :return: the credential, as a dict containing "user" and "password" :rtype: dict """ if not "resolvedBasicCredential" in self: raise ValueError("No basic credential available") return self["resolvedBasicCredential"] def get_aws_credential(self): """ Get the AWS credential for this connection, if available. The AWS credential can either be a keypair or a STS token triplet. :return: the credential, as a dict containing "accessKey", "secretKey", and "sessionToken" (only in the case of STS token) :rtype: dict """ if not "resolvedAWSCredential" in self: raise ValueError("No AWS credential available") return self["resolvedAWSCredential"] def get_oauth2_credential(self): """ Get the OAUTH2 credential for this connection, if available. :return: the credential, as a dict containing "accessToken" :rtype: dict """ if not "resolvedOAuth2Credential" in self: raise ValueError("No OAUTH2 credential available") return self["resolvedOAuth2Credential"] class DSSConnection(object): """ A connection on the DSS instance. .. important:: Do not instantiate directly, use :meth:`dataikuapi.DSSClient.get_connection` instead. """ def __init__(self, client, name): self.client = client self.name = name ######################################################## # Location info ######################################################## def get_location_info(self): """ Get information about this connection. .. caution:: Deprecated, use :meth:`~get_info()` """ warnings.warn("DSSConnection.get_location_info is deprecated, please use get_info", DeprecationWarning) return self.get_info() def get_info(self, contextual_project_key=None): """ Get information about this connection. .. note:: This call requires permissions to read connection details :param string contextual_project_key: (optional) project key to use to resolve variables :return: an object containing connection information :rtype: :class:`DSSConnectionInfo` """ additional_params = { "contextualProjectKey": contextual_project_key } if contextual_project_key is not None else None return DSSConnectionInfo(self.client._perform_json( "GET", "/connections/%s/info" % self.name, params=additional_params)) ######################################################## # Connection deletion ######################################################## def delete(self): """ Delete the connection """ return self.client._perform_empty( "DELETE", "/admin/connections/%s" % self.name) def get_settings(self): """ Get the settings of the connection. You must use :meth:`~DSSConnectionSettings.save()` on the returned object to make your changes effective on the connection. Usage example .. code-block:: python # make details of a connection accessible to some groups connection = client.get_connection("my_connection_name") settings = connection.get_settings() readability = settings.details_readability readability.set_readability(False, "group1", "group2") settings.save() :return: the settings of the connection :rtype: :class:`DSSConnectionSettings` """ settings = self.client._perform_json( "GET", "/admin/connections/%s" % self.name) return DSSConnectionSettings(self, settings) def get_definition(self): """ Get the connection's raw definition. .. caution:: Deprecated, use :meth:`get_settings()` instead. The exact structure of the returned dict is not documented and depends on the connection type. Create connections using the DSS UI and call :meth:`get_definition` to see the fields that are in it. .. note:: This method returns a dict with passwords and secrets in their encrypted form. If you need credentials, consider using :meth:`get_info()` and :meth:`dataikuapi.dss.admin.DSSConnectionInfo.get_basic_credential()`. :return: a connection definition, as a dict. See :meth:`DSSConnectionSettings.get_raw()` :rtype: dict """ return self.client._perform_json( "GET", "/admin/connections/%s" % self.name) def set_definition(self, definition): """ Set the connection's definition. .. caution:: Deprecated, use :meth:`get_settings()` then :meth:`DSSConnectionSettings.save()` instead. .. important:: You should only :meth:`set_definition` using an object that you obtained through :meth:`get_definition`, not create a new dict. Usage example .. code-block:: python # make details of a connection accessible to some groups connection = client.get_connection("my_connection_name") definition = connection.get_definition() definition['detailsReadability']['readableBy'] = 'ALLOWED' definition['detailsReadability']['allowedGroups'] = ['group1', 'group2'] connection.set_definition(definition) :param dict definition: the definition for the connection, as a dict. """ return self.client._perform_json( "PUT", "/admin/connections/%s" % self.name, body = definition) ######################################################## # Security ######################################################## def sync_root_acls(self): """ Resync root permissions on this connection path. This is only useful for HDFS connections when DSS has User Isolation activated with "DSS-managed HDFS ACL" :return: a handle to the task of resynchronizing the permissions :rtype: :class:`~dataikuapi.dss.future.DSSFuture` """ future_response = self.client._perform_json( "POST", "/admin/connections/%s/sync" % self.name, body = {'root':True}) return DSSFuture(self.client, future_response.get('jobId', None), future_response) def sync_datasets_acls(self): """ Resync permissions on datasets in this connection path. This is only useful for HDFS connections when DSS has User Isolation activated with "DSS-managed HDFS ACL" :return: a handle to the task of resynchronizing the permissions :rtype: :class:`~dataikuapi.dss.future.DSSFuture` """ future_response = self.client._perform_json( "POST", "/admin/connections/%s/sync" % self.name, body = {'root':True}) return DSSFuture(self.client, future_response.get('jobId', None), future_response) class DSSConnectionSettings(object): """ Settings of a DSS connection. .. important:: Do not instantiate directly, use :meth:`DSSConnection.get_settings` instead. Use :meth:`save` to save your changes """ def __init__(self, connection, settings): self.connection = connection self.settings = settings def get_raw(self): """ Get the raw settings of the connection. :return: a connection definition, as a dict. Notable fields are: * **type** : type of the connection (for example PostgreSQL, Azure, ...) * **params** : dict of the parameters specific to the connection type :rtype: dict """ return self.settings @property def type(self): """ Get the type of the connection. :return: a DSS connection type, like PostgreSQL, EC2, Azure, ... :rtype: string """ return self.settings['type'] @property def allow_managed_datasets(self): """ Whether managed datasets can use the connection. :rtype: boolean """ return self.settings['allowManagedDatasets'] @allow_managed_datasets.setter def allow_managed_datasets(self, new_value): self.settings["allowManagedDatasets"] = new_value @property def allow_managed_folders(self): """ Whether managed datasets can use the connection. :rtype: boolean """ return self.settings['allowManagedFolders'] @allow_managed_folders.setter def allow_managed_folders(self, new_value): self.settings["allowManagedFolders"] = new_value @property def allow_knowledge_banks(self): """ Whether Knowledge Banks can use the connection. :rtype: boolean """ return self.settings['allowKnowledgeBanks'] @allow_knowledge_banks.setter def allow_knowledge_banks(self, new_value): self.settings["allowKnowledgeBanks"] = new_value @property def allow_write(self): """ Whether data can be written to this connection. If not, the connection is read-only from DSS point of view. :rtype: boolean """ return self.settings['allowWrite'] @allow_write.setter def allow_write(self, new_value): self.settings["allowWrite"] = new_value @property def details_readability(self): """ Get the access control to connection details. :return: an handle on the access control definition. :rtype: :class:`DSSConnectionDetailsReadability` """ return DSSConnectionDetailsReadability(self.settings["detailsReadability"]) @property def usable_by(self): """ Get the mode of access control. This controls usage of the connection, that is, reading and/or writing data from/to the connection. :return: one ALL (anybody) or ALLOWED (ie. only users from groups in :meth:`usable_by_allowed_groups()`) :rtype: string """ return self.settings["usableBy"] @property def usable_by_allowed_groups(self): """ Get the groups allowed to use the connection Only applies if :meth:`usable_by()` is ALLOWED. :return: a list of group names :rtype: list[string] """ return self.settings["allowedGroups"] def set_usability(self, all, *groups): """ Set who can use the connection. :param boolean all: if True, anybody can use the connection :param \*string groups: a list of groups that can use the connection """ if all: self.settings["usableBy"] = 'ALL' else: self.settings["usableBy"] = 'ALLOWED' self.settings["allowedGroups"] = groups def save(self): """ Save the changes to the connection's settings """ self.connection.client._perform_json( "PUT", "/admin/connections/%s" % self.connection.name, body = self.settings) class DSSConnectionDetailsReadability(object): """ Handle on settings for access to connection details. Connection details mostly cover credentials, and giving access to the credentials is necessary to some workloads. Typically, having Spark processes access data directly implies giving credentials to these Spark processes, which in turn implies that the user can access the connection's details. """ def __init__(self, data): self._data = data @property def readable_by(self): """ Get the mode of access control. :return: one of NONE (nobody), ALL (anybody) or ALLOWED (ie. only users from groups in :meth:`allowed_groups()`) :rtype: string """ return self._data["readableBy"] @property def allowed_groups(self): """ Get the groups allowed to access connection details. Only applies if :meth:`readable_by()` is ALLOWED. :return: a list of group names :rtype: list[string] """ return self._data["allowedGroups"] def set_readability(self, all, *groups): """ Set who can get details from the connection. To make the details readable by nobody, pass all=False and no group. :param boolean all: if True, anybody can use the connection :param \*string groups: a list of groups that can use the connection """ if all: self._data["readableBy"] = 'ALL' elif groups is None or len(groups) == 0: self._data["readableBy"] = 'NONE' else: self._data["readableBy"] = 'ALLOWED' self._data["allowedGroups"] = groups class DSSUser(object): """ A handle for a user on the DSS instance. .. important:: Do not instantiate directly, use :meth:`dataikuapi.DSSClient.get_user` instead. """ def __init__(self, client, login): self.client = client self.login = login def delete(self): """ Deletes the user """ return self.client._perform_empty( "DELETE", "/admin/users/%s" % self.login) def get_settings(self): """ Get the settings of the user. You must use :meth:`~DSSUserSettings.save()` on the returned object to make your changes effective on the user. Usage example .. code-block:: python # disable some user user = client.get_user('the_user_login') settings = user.get_settings() settings.enabled = False settings.save() :return: the settings of the user :rtype: :class:`DSSUserSettings` """ raw = self.client._perform_json("GET", "/admin/users/%s" % self.login) return DSSUserSettings(self.client, self.login, raw) def get_activity(self): """ Gets the activity of the user. :return: the user's activity :rtype: :class:`DSSUserActivity` """ activity = self.client._perform_json("GET", "/admin/users/%s/activity" % self.login) return DSSUserActivity(self.client, self.login, activity) ######################################################## # Supplier interaction ######################################################## def start_resync_from_supplier(self): """ Starts a resync of the user from an external supplier (LDAP, Azure AD or custom auth) :return: a :class:`dataikuapi.dss.future.DSSFuture` representing the sync process :rtype: :class:`dataikuapi.dss.future.DSSFuture` """ future_resp = self.client._perform_json("POST", "/admin/users/%s/actions/resync" % self.login) return DSSFuture.from_resp(self.client, future_resp) ######################################################## # Legacy ######################################################## def get_definition(self): """ Get the definition of the user .. caution:: Deprecated, use :meth:`get_settings` instead :return: the user's definition, as a dict. Notable fields are * **login** : identifier of the user, can't be modified * **enabled** : whether the user can log into DSS * **groups** : list of group names this user belongs to :rtype: dict """ warnings.warn("DSSUser.get_definition is deprecated, please use get_settings", DeprecationWarning) return self.client._perform_json("GET", "/admin/users/%s" % self.login) def set_definition(self, definition): """ Set the user's definition. .. caution:: Deprecated, use :meth:`dataikuapi.dss.admin.DSSUserSettings.save()` instead .. important:: You should only use :meth:`set_definition` with an object that you obtained through :meth:`get_definition`, not create a new dict. .. note:: This call requires an API key with admin rights The fields that may be changed in a user definition are: * email * displayName * enabled * groups * userProfile * password (not returned by :meth:`get_definition()` but can be set) * userProperties * adminProperties * secrets * credentials :param dict definition: the definition for the user, as a dict """ warnings.warn("DSSUser.set_definition is deprecated, please use get_settings", DeprecationWarning) return self.client._perform_json("PUT", "/admin/users/%s" % self.login, body = definition) def get_client_as(self): """ Get an API client that has the permissions of this user. This allows administrators to impersonate actions on behalf of other users, in order to perform actions on their behalf. :return: a client through which calls will be run as the user :rtype: :class:`dataikuapi.DSSClient` """ from dataikuapi.dssclient import DSSClient if self.client.api_key is not None: return DSSClient(self.client.host, self.client.api_key, extra_headers={"X-DKU-ProxyUser": self.login}, insecure_tls=not self.client._session.verify) elif self.client.internal_ticket is not None: return DSSClient(self.client.host, internal_ticket = self.client.internal_ticket, extra_headers={"X-DKU-ProxyUser": self.login}, insecure_tls=not self.client._session.verify) else: raise ValueError("Don't know how to proxy this client") class DSSOwnUser(object): """ A handle to interact with your own user .. important:: Do not instantiate directly, use :meth:`dataikuapi.DSSClient.get_own_user` instead. """ def __init__(self, client): self.client = client def get_settings(self): """ Get your own settings You must use :meth:`~DSSOwnUserSettings.save()` on the returned object to make your changes effective on the user. :rtype: :class:`DSSOwnUserSettings` """ raw = self.client._perform_json("GET", "/current-user") return DSSOwnUserSettings(self.client, raw) class DSSUserSettingsBase(object): """ Settings for a DSS user. .. important:: Do not instantiate directly, use :meth:`DSSUser.get_settings` or :meth:`DSSOwnUser.get_settings` instead. """ def __init__(self, settings): self.settings = settings def get_raw(self): """ Get the the raw settings of the user Modifications made to the returned object are reflected when saving. :return: the dict of the settings (not a copy). Notable fields are: * **login** : identifier of the user, can't be modified * **enabled** : whether the user can log into DSS * **groups** : list of group names this user belongs to :rtype: dict """ return self.settings def add_secret(self, name, value): """ Add a user secret. If there was already a secret with the same name, it is replaced :param string name: name of the secret :param string value: name of the value """ self.remove_secret(name) return self.settings["secrets"].append({"name": name, "value": value, "secret": True}) def remove_secret(self, name): """ Remove a user secret based on its name If no secret of the given name exists, the method does nothing. :param string name: name of the secret """ self.settings["secrets"] = [x for x in self.settings["secrets"] if x["name"] != name] @property def user_properties(self): """ Get the user properties for this user. .. important:: Do not set this property, modify the dict in place User properties can be seen and modified by the user themselves. A contrario admin properties are for administrators' eyes only. :rtype: dict """ return self.settings["userProperties"] def set_basic_connection_credential(self, connection, login, password): """ Set per-user-credentials for a connection that takes a user/password pair. :param string connection: name of the connection :param string login: login of the credentials :param string password: password of the credentials """ self.settings["credentials"][connection] = { "type": "BASIC", "user": login, "password": password } def remove_connection_credential(self,connection): """ Remove per-user-credentials for a connection If no credentials for the givent connection exists, this method does nothing :param string connection: name of the connection """ if connection in self.settings["credentials"]: del self.settings["credentials"][connection] def set_basic_plugin_credential(self, plugin_id, param_set_id, preset_id, param_name, login, password): """ Set per-user-credentials for a plugin preset that takes a user/password pair :param string plugin_id: identifier of the plugin :param string param_set_id: identifier of the parameter set to which the preset belongs :param string preset_id: identifier of the preset :param string param_name: name of the credentials parameter in the preset :param string login: login of the credentials :param string password: password of the credentials """ name = json.dumps(["PLUGIN", plugin_id, param_set_id, preset_id, param_name])[1:-1] self.settings["credentials"][name] = { "type": "BASIC", "user": login, "password": password } def set_oauth2_plugin_credential(self, plugin_id, param_set_id, preset_id, param_name, refresh_token): """ Set per-user-credentials for a plugin preset that takes a OAuth refresh token :param string plugin_id: identifier of the plugin :param string param_set_id: identifier of the parameter set to which the preset belongs :param string preset_id: identifier of the preset :param string param_name: name of the credentials parameter in the preset :param string refresh_token: value of the refresh token """ name = json.dumps(["PLUGIN", plugin_id, param_set_id, preset_id, param_name])[1:-1] self.settings["credentials"][name] = { "type": "OAUTH_REFRESH_TOKEN", "refreshToken": refresh_token } def remove_plugin_credential(self, plugin_id, param_set_id, preset_id, param_name): """ Remove per-user-credentials for a plugin preset :param string plugin_id: identifier of the plugin :param string param_set_id: identifier of the parameter set to which the preset belongs :param string preset_id: identifier of the preset :param string param_name: name of the credentials parameter in the preset """ name = json.dumps(["PLUGIN", plugin_id, param_set_id, preset_id, param_name])[1:-1] if name in self.settings["credentials"]: del self.settings["credentials"][name] class DSSUserSettings(DSSUserSettingsBase): """ Settings for a DSS user. .. important:: Do not instantiate directly, use :meth:`DSSUser.get_settings` instead. """ def __init__(self, client, login, settings): super(DSSUserSettings, self).__init__(settings) self.client = client self.login = login @property def admin_properties(self): """ Get the admin properties for this user. .. important:: Do not set this property, modify the dict in place Admin properties can be seen and modified only by administrators, not by the user themselves. :rtype: dict """ return self.settings["adminProperties"] @property def enabled(self): """ Whether this user is enabled. :rtype: boolean """ return self.settings["enabled"] @enabled.setter def enabled(self, new_value): self.settings["enabled"] = new_value @property def creation_date(self): """ Get the timestamp of when the user was created :return: the creation date :rtype: :class:`datetime.datetime` or None """ timestamp = self.settings["creationDate"] if "creationDate" in self.settings else None return _timestamp_ms_to_zoned_datetime(timestamp) def save(self): """ Saves the settings """ self.client._perform_json("PUT", "/admin/users/%s" % self.login, body = self.settings) class DSSOwnUserSettings(DSSUserSettingsBase): """ Settings for the current DSS user. .. important:: Do not instantiate directly, use :meth:`DSSOwnUser.get_settings()` instead. """ def __init__(self, client, settings): super(DSSOwnUserSettings, self).__init__(settings) self.client = client def save(self): """ Saves the settings """ self.client._perform_empty("PUT", "/current-user", body = self.settings) class DSSUserActivity(object): """ Activity for a DSS user. .. important:: Do not instantiate directly, use :meth:`DSSUser.get_activity` or :meth:`dataikuapi.DSSClient.list_users_activity()` instead. """ def __init__(self, client, login, activity): self.client = client self.login = login self.activity = activity def get_raw(self): """ Get the raw activity of the user as a dict. :return: the raw activity. Fields are * **login** : the login of the user for this activity * **lastSuccessfulLogin** : timestamp in milliseconds of the last time the user logged into DSS * **lastFailedLogin** : timestamp in milliseconds of the last time DSS recorded a login failure for this user * **lastSessionActivity** : timestamp in milliseconds of the last time the user opened a tab :rtype: dict """ return self.activity @property def last_successful_login(self): """ Get the last successful login of the user Returns None if there was no successful login for this user. :return: the last successful login :rtype: :class:`datetime.datetime` or None """ timestamp = self.activity["lastSuccessfulLogin"] return _timestamp_ms_to_zoned_datetime(timestamp) @property def last_failed_login(self): """ Get the last failed login of the user Returns None if there were no failed login for this user. :return: the last failed login :rtype: :class:`datetime.datetime` or None """ timestamp = self.activity["lastFailedLogin"] return _timestamp_ms_to_zoned_datetime(timestamp) @property def last_session_activity(self): """ Get the last session activity of the user The last session activity is the last time the user opened a new DSS tab or refreshed his session. Returns None if there is no session activity yet. :return: the last session activity :rtype: :class:`datetime.datetime` or None """ timestamp = self.activity["lastSessionActivity"] return _timestamp_ms_to_zoned_datetime(timestamp) class DSSAuthorizationMatrix(object): """ The authorization matrix of all groups and enabled users of the DSS instance. .. important:: Do not instantiate directly, use :meth:`dataikuapi.DSSClient.get_authorization_matrix` instead. """ def __init__(self, authorization_matrix): self.authorization_matrix = authorization_matrix @property def raw(self): """ Get the raw authorization matrix as a dict :return: the authorization matrix. There are 2 parts in the matrix, each as a top-level field and with similar structures, **perUser** and **perGroup**. :rtype: dict """ return self.authorization_matrix class DSSGroup(object): """ A group on the DSS instance. .. important:: Do not instantiate directly, use :meth:`dataikuapi.DSSClient.get_group` instead. """ def __init__(self, client, name): self.client = client self.name = name ######################################################## # Group deletion ######################################################## def delete(self): """ Deletes the group """ return self.client._perform_empty( "DELETE", "/admin/groups/%s" % self.name) def get_definition(self): """ Get the group's definition (name, description, admin abilities, type, ldap name mapping) :return: the group's definition. Top-level fields are: * **name** : name of the group * **sourceType** : type of group. Possible values: LOCAL, LDAP :rtype: dict """ return self.client._perform_json( "GET", "/admin/groups/%s" % self.name) def set_definition(self, definition): """ Set the group's definition. .. important:: You should only use :meth:`set_definition` with an object that you obtained through :meth:`get_definition`, not create a new dict. :param dict definition: the definition for the group, as a dict """ return self.client._perform_json( "PUT", "/admin/groups/%s" % self.name, body = definition) class DSSGeneralSettings(object): """ The general settings of the DSS instance. .. important:: Do not instantiate directly, use :meth:`dataikuapi.DSSClient.get_general_settings` instead. """ def __init__(self, client): self.client = client self.settings = self.client._perform_json("GET", "/admin/general-settings") ######################################################## # Update settings on instance ######################################################## def save(self): """ Save the changes that were made to the settings on the DSS instance .. note:: This call requires an API key with admin rights """ return self.client._perform_empty("PUT", "/admin/general-settings", body = self.settings) ######################################################## # Value accessors ######################################################## def get_raw(self): """ Get the settings as a dictionary :return: the settings :rtype: dict """ return self.settings def add_impersonation_rule(self, rule, is_user_rule=True): """ Add a rule to the impersonation settings :param object rule: an impersonation rule, either a :class:`dataikuapi.dss.admin.DSSUserImpersonationRule` or a :class:`dataikuapi.dss.admin.DSSGroupImpersonationRule`, or a plain dict :param boolean is_user_rule: when the rule parameter is a dict, whether the rule is for users or groups """ rule_raw = rule if isinstance(rule, DSSUserImpersonationRule): rule_raw = rule.raw is_user_rule = True elif isinstance(rule, DSSGroupImpersonationRule): rule_raw = rule.raw is_user_rule = False impersonation = self.settings['impersonation'] if is_user_rule: impersonation['userRules'].append(rule_raw) else: impersonation['groupRules'].append(rule_raw) def get_impersonation_rules(self, dss_user=None, dss_group=None, unix_user=None, hadoop_user=None, project_key=None, scope=None, rule_type=None, is_user=None, rule_from=None): """ Retrieve the user or group impersonation rules that match the parameters :param string dss_user: a DSS user name :param string dss_group: a DSS group name :param string rule_from: a regex (which will be applied to user or group names) :param string unix_user: a name to match the target UNIX user :param string hadoop_user: a name to match the target Hadoop user :param string project_key: a project key :param string scope: project-scoped ('PROJECT') or global ('GLOBAL') :param string type: the rule user or group matching method ('IDENTITY', 'SINGLE_MAPPING', 'REGEXP_RULE') :param boolean is_user: True if only user-level rules should be considered, False for only group-level rules, None to consider both """ user_matches = self.settings['impersonation']['userRules'] if is_user == None or is_user == True else [] if dss_user is not None: user_matches = [m for m in user_matches if dss_user == m.get('dssUser', None)] if rule_from is not None: user_matches = [m for m in user_matches if rule_from == m.get('ruleFrom', None)] if unix_user is not None: user_matches = [m for m in user_matches if unix_user == m.get('targetUnix', None)] if hadoop_user is not None: user_matches = [m for m in user_matches if hadoop_user == m.get('targetHadoop', None)] if project_key is not None: user_matches = [m for m in user_matches if project_key == m.get('projectKey', None)] if rule_type is not None: user_matches = [m for m in user_matches if rule_type == m.get('type', None)] if scope is not None: user_matches = [m for m in user_matches if scope == m.get('scope', None)] group_matches = self.settings['impersonation']['groupRules'] if is_user == None or is_user == False else [] if dss_group is not None: group_matches = [m for m in group_matches if dss_group == m.get('dssGroup', None)] if rule_from is not None: group_matches = [m for m in group_matches if rule_from == m.get('ruleFrom', None)] if unix_user is not None: group_matches = [m for m in group_matches if unix_user == m.get('targetUnix', None)] if hadoop_user is not None: group_matches = [m for m in group_matches if hadoop_user == m.get('targetHadoop', None)] if rule_type is not None: group_matches = [m for m in group_matches if rule_type == m.get('type', None)] all_matches = [] for m in user_matches: all_matches.append(DSSUserImpersonationRule(m)) for m in group_matches: all_matches.append(DSSGroupImpersonationRule(m)) return all_matches def remove_impersonation_rules(self, dss_user=None, dss_group=None, unix_user=None, hadoop_user=None, project_key=None, scope=None, rule_type=None, is_user=None, rule_from=None): """ Remove the user or group impersonation rules that matches the parameters from the settings :param string dss_user: a DSS user name :param string dss_group: a DSS group name :param string rule_from: a regex (which will be applied to user or group names) :param string unix_user: a name to match the target UNIX user :param string hadoop_user: a name to match the target Hadoop user :param string project_key: a project key :param string scope: project-scoped ('PROJECT') or global ('GLOBAL') :param string type: the rule user or group matching method ('IDENTITY', 'SINGLE_MAPPING', 'REGEXP_RULE') :param boolean is_user: True if only user-level rules should be considered, False for only group-level rules, None to consider both """ for m in self.get_impersonation_rules(dss_user, dss_group, unix_user, hadoop_user, project_key, scope, rule_type, is_user, rule_from): if isinstance(m, DSSUserImpersonationRule): self.settings['impersonation']['userRules'].remove(m.raw) elif isinstance(m, DSSGroupImpersonationRule): self.settings['impersonation']['groupRules'].remove(m.raw) ######################################################## # Admin actions ######################################################## def push_container_exec_base_images(self): """ Push the container exec base images to their repository """ resp = self.client._perform_json("POST", "/admin/container-exec/actions/push-base-images") if resp is None: raise Exception('Container exec base image push returned no data') if resp.get('messages', {}).get('error', False): raise Exception('Container exec base image push failed : %s' % (json.dumps(resp.get('messages', {}).get('messages', {})))) return resp class DSSUserImpersonationRule(object): """ An user-level rule items for the impersonation settings """ def __init__(self, raw=None): self.raw = raw if raw is not None else {'scope':'GLOBAL','type':'IDENTITY'} def scope_global(self): """ Make the rule apply to all projects """ self.raw['scope'] = 'GLOBAL' return self def scope_project(self, project_key): """ Make the rule apply to a given project :param string project_key: the project this rule applies to """ self.raw['scope'] = 'PROJECT' self.raw['projectKey'] = project_key return self def user_identity(self): """ Make the rule map each DSS user to a UNIX user of the same name """ self.raw['type'] = 'IDENTITY' return self def user_single(self, dss_user, unix_user, hadoop_user=None): """ Make the rule map a given DSS user to a given UNIX user :param string dss_user: a DSS user :param string unix_user: a UNIX user :param string hadoop_user: a hadoop user (optional, defaults to unix_user) """ self.raw['type'] = 'SINGLE_MAPPING' self.raw['dssUser'] = dss_user self.raw['targetUnix'] = unix_user self.raw['targetHadoop'] = hadoop_user return self def user_regexp(self, regexp, unix_user, hadoop_user=None): """ Make the rule map a DSS users matching a given regular expression to a given UNIX user :param string regexp: a regular expression to match DSS user names :param string unix_user: a UNIX user :param string hadoop_user: a hadoop user (optional, defaults to unix_user) """ self.raw['type'] = 'REGEXP_RULE' self.raw['ruleFrom'] = regexp self.raw['targetUnix'] = unix_user self.raw['targetHadoop'] = hadoop_user return self class DSSGroupImpersonationRule(object): """ A group-level rule items for the impersonation settings """ def __init__(self, raw=None): self.raw = raw if raw is not None else {'type':'IDENTITY'} def group_identity(self): """ Make the rule map each DSS user to a UNIX user of the same name """ self.raw['type'] = 'IDENTITY' return self def group_single(self, dss_group, unix_user, hadoop_user=None): """ Make the rule map a given DSS user to a given UNIX user :param string dss_group: a DSS group :param string unix_user: a UNIX user :param string hadoop_user: a hadoop user (optional, defaults to unix_user) """ self.raw['type'] = 'SINGLE_MAPPING' self.raw['dssGroup'] = dss_group self.raw['targetUnix'] = unix_user self.raw['targetHadoop'] = hadoop_user return self def group_regexp(self, regexp, unix_user, hadoop_user=None): """ Make the rule map a DSS users matching a given regular expression to a given UNIX user :param string regexp: a regular expression to match DSS groups :param string unix_user: a UNIX user :param string hadoop_user: a hadoop user (optional, defaults to unix_user) """ self.raw['type'] = 'REGEXP_RULE' self.raw['ruleFrom'] = regexp self.raw['targetUnix'] = unix_user self.raw['targetHadoop'] = hadoop_user return self class DSSCodeEnv(object): """ A code env on the DSS instance. .. important:: Do not instantiate directly, use :meth:`dataikuapi.DSSClient.get_code_env` instead. """ def __init__(self, client, env_lang, env_name): self.client = client self.env_lang = env_lang self.env_name = env_name ######################################################## # Env deletion ######################################################## def delete(self): """ Delete the code env .. note:: This call requires an API key with admin rights """ resp = self.client._perform_json( "DELETE", "/admin/code-envs/%s/%s" % (self.env_lang, self.env_name)) if resp is None: raise Exception('Env deletion returned no data') if resp.get('messages', {}).get('error', False): raise Exception('Env deletion failed : %s' % (json.dumps(resp.get('messages', {}).get('messages', {})))) return resp ######################################################## # Code env description ######################################################## def get_definition(self): """ Get the code env's definition .. caution:: Deprecated, use :meth:`get_settings` instead .. note:: This call requires an API key with admin rights :return: the code env definition :rtype: dict """ return self.client._perform_json( "GET", "/admin/code-envs/%s/%s" % (self.env_lang, self.env_name)) def set_definition(self, env): """ Set the code env's definition. The definition should come from a call to :meth:`get_definition` .. caution:: Deprecated, use :meth:`get_settings` then :meth:`DSSDesignCodeEnvSettings.save()` or :meth:`DSSAutomationCodeEnvSettings.save()` instead Fields that can be updated in design node: * env.permissions, env.usableByAll, env.desc.owner * env.specCondaEnvironment, env.specPackageList, env.externalCondaEnvName, env.desc.installCorePackages, env.desc.corePackagesSet, env.desc.installJupyterSupport, env.desc.yarnPythonBin, env.desc.yarnRBin env.desc.envSettings, env.desc.allContainerConfs, env.desc.containerConfs, env.desc.allSparkKubernetesConfs, env.desc.sparkKubernetesConfs Fields that can be updated in automation node (where {version} is the updated version): * env.permissions, env.usableByAll, env.owner, env.envSettings * env.{version}.specCondaEnvironment, env.{version}.specPackageList, env.{version}.externalCondaEnvName, env.{version}.desc.installCorePackages, env.{version}.corePackagesSet, env.{version}.desc.installJupyterSupport env.{version}.desc.yarnPythonBin, env.{version}.desc.yarnRBin, env.{version}.desc.allContainerConfs, env.{version}.desc.containerConfs, env.{version}.desc.allSparkKubernetesConfs, env.{version}.{version}.desc.sparkKubernetesConfs .. note:: This call requires an API key with admin rights .. important:: You should only :meth:`set_definition` using an object that you obtained through :meth:`get_definition`, not create a new dict. :param dict data: a code env definition :return: the updated code env definition :rtype: dict """ return self.client._perform_json( "PUT", "/admin/code-envs/%s/%s" % (self.env_lang, self.env_name), body=env) def get_version_for_project(self, project_key): """ Resolve the code env version for a given project .. note:: Version will only be non-empty for versioned code envs actually used by the project :param string project_key: project to get the version for :return: the code env version full reference for the version of the code env that the project use, as a dict. The dict should contains a **version** field holding the identifier of the version and a **bundleId** field for the identifier of the active bundle in the project. :rtype: dict """ return self.client._perform_json( "GET", "/admin/code-envs/%s/%s/%s/version" % (self.env_lang, self.env_name, project_key)) def get_settings(self): """ Get the settings of this code env. .. important:: You must use :meth:`DSSCodeEnvSettings.save()` on the returned object to make your changes effective on the code env. .. code-block:: python # Example: setting the required packagd codeenv = client.get_code_env("PYTHON", "code_env_name") settings = codeenv.get_settings() settings.set_required_packages("dash==2.0.0", "bokeh<2.0") settings.save() # then proceed to update_packages() :rtype: :class:`DSSDesignCodeEnvSettings` or :class:`DSSAutomationCodeEnvSettings` """ data = self.client._perform_json( "GET", "/admin/code-envs/%s/%s" % (self.env_lang, self.env_name)) # you can't just use deploymentMode to check if it's an automation code # env, because some modes are common to both types of nodes. So we rely # on a non-null field that only the automation code envs have if data.get("versions", None) is not None: return DSSAutomationCodeEnvSettings(self, data) else: return DSSDesignCodeEnvSettings(self, data) ######################################################## # Code env actions ######################################################## def set_jupyter_support(self, active): """ Update the code env jupyter support .. note:: This call requires an API key with admin rights :param boolean active: True to activate jupyter support, False to deactivate """ resp = self.client._perform_json( "POST", "/admin/code-envs/%s/%s/jupyter" % (self.env_lang, self.env_name), params = {'active':active}) if resp is None: raise Exception('Env update returned no data') if resp.get('messages', {}).get('error', False): raise Exception('Env update failed : %s' % (json.dumps(resp.get('messages', {}).get('messages', {})))) return resp def update_packages(self, force_rebuild_env=False): """ Update the code env packages so that it matches its spec .. note:: This call requires an API key with admin rights :param boolean force_rebuild_env: whether to rebuild the code env from scratch :return: list of messages collected during the operation. Fields are: * **anyMessage** : whether there is at least 1 message * **success**, **warning**, **error** and **fatal** : whether there is at least one message of the corresponding category * **messages** : list of messages. Each message is a dict, with at least **severity** and **message** sufields. :rtype: dict """ resp = self.client._perform_json( "POST", "/admin/code-envs/%s/%s/packages" % (self.env_lang, self.env_name), params={"forceRebuildEnv": force_rebuild_env}) if resp is None: raise Exception('Env update returned no data') if resp.get('messages', {}).get('error', False): raise Exception('Env update failed : %s' % (json.dumps(resp.get('messages', {}).get('messages', {})))) return resp def update_images(self, env_version=None): """ Rebuild the docker image of the code env .. note:: This call requires an API key with admin rights :param string env_version: (optional) version of the code env. Applies only to versioned code envs. :return: list of messages collected during the operation. Fields are: * **anyMessage** : whether there is at least 1 message * **success**, **warning**, **error** and **fatal** : whether there is at least one message of the corresponding category * **messages** : list of messages. Each message is a dict, with at least **severity** and **message** sufields. :rtype: dict """ resp = self.client._perform_json( "POST", "/admin/code-envs/%s/%s/images" % (self.env_lang, self.env_name), params={"envVersion": env_version}) if resp is None: raise Exception('Env image build returned no data') if resp.get('messages', {}).get('error', False): raise Exception('Env image build failed : %s' % (json.dumps(resp.get('messages', {}).get('messages', {})))) return resp def list_usages(self): """ List usages of the code env in the instance :return: a list of objects where the code env is used. Each usage has is a dict with at least: * **envUsage** : type of usage. Possible values: PROJECT, RECIPE, NOTEBOOK, PLUGIN, SCENARIO, SCENARIO_STEP, SCENARIO_TRIGGER, DATASET_METRIC, DATASET_CHECK, DATASET, WEBAPP, REPORT, API_SERVICE_ENDPOINT, SAVED_MODEL, MODEL, CODE_STUDIO_TEMPLATE * **projectKey** and **objectId** : identifier of the object where the code env is used :rtype: list[dict] """ return self.client._perform_json( "GET", "/admin/code-envs/%s/%s/usages" % (self.env_lang, self.env_name)) def list_logs(self): """ List logs of the code env in the instance :return: a list of log descriptions. Each log description as a dict with at least a **name** field for the name of the log file. :rtype: list[dict] """ return self.client._perform_json( "GET", "/admin/code-envs/%s/%s/logs" % (self.env_lang, self.env_name)) def get_log(self, log_name): """ Get the logs of the code env :param string log_name: name of the log to fetch :return: the raw log :rtype: string """ return self.client._perform_text( "GET", "/admin/code-envs/%s/%s/logs/%s" % (self.env_lang, self.env_name, log_name)) class DSSCodeEnvSettings(object): """ Base settings class for a DSS code env. .. important:: Do not instantiate directly, use :meth:`DSSCodeEnv.get_settings` instead. Use :meth:`save` to save your changes """ def __init__(self, codeenv, settings): self.codeenv = codeenv self.settings = settings @property def env_lang(self): """ Get the language of the code env :return: a language (possible values: PYTHON, R) :rtype: string """ return self.codeenv.env_lang @property def env_name(self): """ Get the name of the code env :rtype: string """ return self.codeenv.env_name def save(self): """ Save the changes to the code env's settings """ self.codeenv.client._perform_json( "PUT", "/admin/code-envs/%s/%s" % (self.env_lang, self.env_name), body=self.settings) class DSSCodeEnvPackageListBearer(object): def get_required_packages(self, as_list=False): """ Get the list of required packages, as a single string :param boolean as_list: if True, return the spec as a list of lines; if False, return as a single multiline string :return: a list of packages specifications :rtype: list[string] or string """ x = self.settings.get("specPackageList", "") return x.split('\n') if as_list else x def set_required_packages(self, *packages): """ Set the list of required packages :param list[string] packages: a list of packages specifications """ self.settings["specPackageList"] = '\n'.join(packages) def get_required_conda_spec(self, as_list=False): """ Get the list of required conda packages, as a single string :param boolean as_list: if True, return the spec as a list of lines; if False, return as a single multiline string :return: a list of packages specifications :rtype: list[string] or string """ x = self.settings.get("specCondaEnvironment", "") return x.split('\n') if as_list else x def set_required_conda_spec(self, *spec): """ Set the list of required conda packages :param list[string] spec: a list of packages specifications """ self.settings["specCondaEnvironment"] = '\n'.join(packages) class DSSCodeEnvContainerConfsBearer(object): def get_built_for_all_container_confs(self): """ Whether the code env creates an image for each container config :rtype: boolean """ return self.settings.get("allContainerConfs", False) def get_built_container_confs(self): """ Get the list of container configs for which the code env builds an image (if not all) :return: a list of container configuration names :rtype: list[string] """ return self.settings.get("containerConfs", []) def set_built_container_confs(self, *configs, **kwargs): """ Set the list of container configs for which the code env builds an image :param boolean all: if True, an image is built for each config :param list[string] configs: list of configuration names to build images for """ all = kwargs.get("all", False) self.settings['allContainerConfs'] = all if not all: self.settings['containerConfs'] = configs def built_for_all_spark_kubernetes_confs(self): """ Whether the code env creates an image for each managed Spark over Kubernetes config """ return self.settings.get("allSparkKubernetesConfs", False) def get_built_spark_kubernetes_confs(self): """ Get the list of managed Spark over Kubernetes configs for which the code env builds an image (if not all) :return: a list of spark configuration names :rtype: list[string] """ return self.settings.get("sparkKubernetesConfs", []) def set_built_spark_kubernetes_confs(self, *configs, **kwargs): """ Set the list of managed Spark over Kubernetes configs for which the code env builds an image :param boolean all: if True, an image is built for each config :param list[string] configs: list of configuration names to build images for """ all = kwargs.get("all", False) self.settings['allSparkKubernetesConfs'] = all if not all: self.settings['sparkKubernetesConfs'] = configs class DSSDesignCodeEnvSettings(DSSCodeEnvSettings, DSSCodeEnvPackageListBearer, DSSCodeEnvContainerConfsBearer): """ Base settings class for a DSS code env on a design node. .. important:: Do not instantiate directly, use :meth:`DSSCodeEnv.get_settings` instead. Use :meth:`save` to save your changes """ def __init__(self, codeenv, settings): super(DSSDesignCodeEnvSettings, self).__init__(codeenv, settings) def get_raw(self): """ Get the raw code env settings The structure depends on the type of code env. The data consists of the definition as it is persisted on disk, the lists of requested packages and resource script (if relevant). :return: code env settings :rtype: dict """ return self.settings class DSSAutomationCodeEnvSettings(DSSCodeEnvSettings, DSSCodeEnvContainerConfsBearer): """ Base settings class for a DSS code env on an automation node. .. important:: Do not instantiate directly, use :meth:`DSSCodeEnv.get_settings` instead. Use :meth:`save` to save your changes """ def __init__(self, codeenv, settings): super(DSSAutomationCodeEnvSettings, self).__init__(codeenv, settings) def get_raw(self): """ Get the raw code env settings The structure depends on the type of code env. The data consists of the definition as it is persisted on disk, and the identifiers of the versions of the code env, or the spec of the unique version if the code env is not versioned. To access the lists of requested packages or resource scripts. Use :meth:`get_version()`. :return: code env settings :rtype: dict """ return self.settings def get_version(self, version_id=None): """ Get a specific code env version (for versioned envs) or the single version :param string version_id: for versioned code env, identifier of the desired version :return: the settings of a code env version :rtype: :class:`DSSAutomationCodeEnvVersionSettings` """ deployment_mode = self.settings.get("deploymentMode", None) if deployment_mode in ['AUTOMATION_SINGLE']: return DSSAutomationCodeEnvVersionSettings(self.codeenv, self.settings.get('currentVersion', {})) elif deployment_mode in ['AUTOMATION_VERSIONED']: versions = self.settings.get("versions", []) version_ids = [v.get('versionId') for v in versions] if version_id is None: raise Exception("A version id is required in a versioned code env. Existing ids: %s" % ', '.join(version_ids)) for version in versions: if version_id == version.get("versionId"): return DSSAutomationCodeEnvVersionSettings(self.codeenv, version) raise Exception("Version %s not found in : %s" % (version_id, ', '.join(version_ids))) elif deployment_mode in ['PLUGIN_NON_MANAGED', 'PLUGIN_MANAGED', 'AUTOMATION_NON_MANAGED_PATH', 'EXTERNAL_CONDA_NAMED']: return DSSAutomationCodeEnvVersionSettings(self.codeenv, self.settings.get('noVersion', {})) else: raise Exception("Unexpected deployment mode %s for an automation node code env. Alter the settings directly with get_raw()", deployment_mode) class DSSAutomationCodeEnvVersionSettings(DSSCodeEnvPackageListBearer): """ Base settings class for a DSS code env version on an automation node. .. important:: Do not instantiate directly, use :meth:`DSSAutomationCodeEnvSettings.get_version` instead. Use :meth:`save` on the :class:`DSSAutomationCodeEnvSettings` to save your changes """ def __init__(self, codeenv_settings, version_settings): self.codeenv_settings = codeenv_settings self.settings = version_settings def get_raw(self): """ Get the raw code env version settings The structure depends on the type of code env. The dict contains a **versionId** field, and the definition of the code env requirements. :return: code env settings :rtype: dict """ return self.settings class DSSGlobalApiKey(object): """ A global API key on the DSS instance """ def __init__(self, client, key, id_): self.client = client self.key = key self.id_ = id_ ######################################################## # Key deletion ######################################################## def delete(self): """ Delete the api key .. note:: This call requires an API key with admin rights """ return self.client._perform_empty( "DELETE", "/admin/global-api-keys/%s" % self.id_) ######################################################## # Key description ######################################################## def get_definition(self): """ Get the API key's definition .. note:: This call requires an API key with admin rights .. note:: If the secure API keys feature is enabled, the secret key of this API key will not be present in the returned dict :return: the API key definition, as a dict. The dict additionally contains the definition of the permissions attached to the key. :rtype: dict """ return self.client._perform_json( "GET", "/admin/global-api-keys/%s" % self.id_) def set_definition(self, definition): """ Set the API key's definition .. note:: This call requires an API key with admin rights .. important:: You should only :meth:`set_definition` using an object that you obtained through :meth:`get_definition`, not create a new dict. You may not use this method to update the 'key' field. Usage example .. code-block:: python # make an API key able to create projects key = client.get_global_api_key('my_api_key_secret') definition = key.get_definition() definition["globalPermissions"]["mayCreateProjects"] = True key.set_definition(definition) :param dict definition: the definition for the API key """ return self.client._perform_empty( "PUT", "/admin/global-api-keys/%s" % self.id_, body = definition) class DSSGlobalApiKeyListItem(dict): """ An item in a list of global API keys. .. important:: Do not instantiate directly, use :meth:`dataikuapi.DSSClient.list_global_api_keys` instead. """ def __init__(self, client, data): super(DSSGlobalApiKeyListItem, self).__init__(data) self.client = client def to_global_api_key(self): """ Gets a handle corresponding to this item :rtype: :class:`DSSGlobalApiKey` """ return DSSGlobalApiKey(self.client, self["key"], self["id"]) @property def id(self): """ Get the identifier of the API key :rtype: string """ return self["id"] @property def user_for_impersonation(self): """ Get the user associated to the API key :rtype: string """ return self.get("dssUserForImpersonation") @property def key(self): """ Get the API key If the secure API keys feature is enabled, this key field will not be available :rtype: string """ return self["key"] @property def label(self): """ Get the label of the API key :rtype: string """ return self["label"] @property def description(self): """ Get the description of the API key :rtype: string """ return self.get("description") @property def created_on(self): """ Get the timestamp of when the API key was created :rtype: :class:`datetime.datetime` """ timestamp = self["createdOn"] return _timestamp_ms_to_zoned_datetime(timestamp) @property def created_by(self): """ Get the login of the user who created the API key :rtype: string """ return self["createdBy"] class DSSPersonalApiKey(object): """ A personal API key on the DSS instance. .. important:: Do not instantiate directly, use :meth:`dataikuapi.DSSClient.get_personal_api_key` instead. """ def __init__(self, client, key, id_): self.client = client self.key = key self.id_ = id_ ######################################################## # Key description ######################################################## def get_definition(self): """ Get the API key's definition :return: the personal API key definition, as a dict. The login of the user of this personal key is in a **user** field. :rtype: dict """ return self.client._perform_json( "GET", "/personal-api-keys/%s" % (self.id_)) def set_definition(self, definition): """ Set the API key's definition .. note:: Only the label and description of the key can be updated. .. important:: You should only :meth:`set_definition` using an object that you obtained through :meth:`get_definition`, not create a new dict. You may not use this method to update the 'key' field. Usage example .. code-block:: python # update an API key label key = client.get_personal_api_key('my_api_key_id') definition = key.get_definition() definition["label"] = "My New Label" key.set_definition(definition) :param dict definition: the definition for the API key """ return self.client._perform_empty( "PUT", "/personal-api-keys/%s" % self.id_, body=definition) ######################################################## # Key deletion ######################################################## def delete(self): """ Delete the API key """ return self.client._perform_empty( "DELETE", "/personal-api-keys/%s" % self.id_) class DSSPersonalApiKeyListItem(dict): """ An item in a list of personal API key. .. important:: Do not instantiate directly, use :meth:`dataikuapi.DSSClient.list_personal_api_keys` or :meth:`dataikuapi.DSSClient.list_all_personal_api_keys` instead. """ def __init__(self, client, data): super(DSSPersonalApiKeyListItem, self).__init__(data) self.client = client def to_personal_api_key(self): """ Gets a handle corresponding to this item :rtype: :class:`DSSPersonalApiKey` """ return DSSPersonalApiKey(self.client, self.get("key", ""), self["id"]) @property def id(self): """ Get the identifier of the API key :rtype: string """ return self["id"] @property def user(self): """ Get the user associated to the API key :rtype: string """ return self["user"] @property def key(self): """ Get the API key If the secure API keys feature is enabled, this key field will not be available :rtype: string """ return self["key"] @property def label(self): """ Get the label of the API key :rtype: string """ return self["label"] @property def description(self): """ Get the description of the API key :rtype: string """ return self["description"] @property def created_on(self): """ Get the timestamp of when the API key was created :rtype: :class:`datetime.datetime` """ timestamp = self["createdOn"] return _timestamp_ms_to_zoned_datetime(timestamp) @property def created_by(self): """ Get the login of the user who created the API key :rtype: string """ return self["createdBy"] class DSSCluster(object): """ A handle to interact with a cluster on the DSS instance. .. important:: Do not instantiate directly, use :meth:`dataikuapi.DSSClient.get_cluster` instead. """ def __init__(self, client, cluster_id): self.client = client self.cluster_id = cluster_id ######################################################## # Cluster deletion ######################################################## def delete(self): """ Deletes the cluster. .. important:: This does not previously stop it. """ self.client._perform_empty( "DELETE", "/admin/clusters/%s" % (self.cluster_id)) ######################################################## # Cluster description ######################################################## def get_settings(self): """ Get the cluster's settings. This includes opaque data for the cluster if this is a started managed cluster. The returned object can be used to save settings. :return: a :class:`DSSClusterSettings` object to interact with cluster settings :rtype: :class:`DSSClusterSettings` """ settings = self.client._perform_json( "GET", "/admin/clusters/%s" % (self.cluster_id)) return DSSClusterSettings(self.client, self.cluster_id, settings) def get_definition(self): """ Get the cluster's definition. This includes opaque data for the cluster if this is a started managed cluster. .. caution:: Deprecated, use :meth:`get_settings()` :return: the definition of the cluster as a dict. For clusters from plugin components, the settings of the cluster are in a **params** field. :rtype: dict """ return self.client._perform_json( "GET", "/admin/clusters/%s" % (self.cluster_id)) def set_definition(self, cluster): """ Set the cluster's definition. The definition should come from a call to the get_definition() method. .. caution:: Deprecated, use :meth:`DSSClusterSettings.save()` .. important:: You should only :meth:`set_definition` using an object that you obtained through :meth:`get_definition`, not create a new dict. :param dict cluster: a cluster definition :return: the updated cluster definition :rtype: dict """ return self.client._perform_json( "PUT", "/admin/clusters/%s" % (self.cluster_id), body=cluster) def get_status(self): """ Get the cluster's status and usage :return: The cluster status, as a :class:`DSSClusterStatus` object :rtype: :class:`DSSClusterStatus` """ status = self.client._perform_json("GET", "/admin/clusters/%s/status" % (self.cluster_id)) return DSSClusterStatus(self.client, self.cluster_id, status) ######################################################## # Cluster actions ######################################################## def start(self): """ Starts or attaches the cluster .. caution:: This operation is only valid for a managed cluster. """ resp = self.client._perform_json( "POST", "/admin/clusters/%s/actions/start" % (self.cluster_id)) if resp is None: raise Exception('Cluster operation returned no data') if resp.get('messages', {}).get('error', False): raise Exception('Cluster operation failed : %s' % (json.dumps(resp.get('messages', {}).get('messages', {})))) return resp def stop(self, terminate=True, force_stop=False): """ Stops or detaches the cluster This operation is only valid for a managed cluster. :param boolean terminate: whether to delete the cluster after stopping it :param boolean force_stop: whether to try to force stop the cluster, useful if DSS expects the cluster to already be stopped """ resp = self.client._perform_json( "POST", "/admin/clusters/%s/actions/stop" % (self.cluster_id), params={'terminate': terminate, 'forceStop': force_stop}) if resp is None: raise Exception('Env update returned no data') if resp.get('messages', {}).get('error', False): raise Exception('Cluster operation failed : %s' % (json.dumps(resp.get('messages', {}).get('messages', {})))) return resp def run_kubectl(self, args): """ Runs an arbitrary kubectl command on the cluster. .. caution:: This operation is only valid for a Kubernetes cluster. .. note:: This call requires an API key with DSS instance admin rights :param string args: the arguments to pass to kubectl (without the "kubectl") :return: a dict containing the return value, standard output, and standard error of the command :rtype: dict """ return self.client._perform_json( "POST", "/admin/clusters/%s/k8s/actions/run-kubectl" % self.cluster_id, body={'args': args}) def delete_finished_jobs(self, delete_failed=False, namespace=None, label_filter=None, dry_run=False): """ Runs a kubectl command to delete finished jobs. .. caution:: This operation is only valid for a Kubernetes cluster. :param boolean delete_failed: if True, delete both completed and failed jobs, otherwise only delete completed jobs :param string namespace: the namespace in which to delete the jobs, if None, uses the namespace set in kubectl's current context :param string label_filter: delete only jobs matching a label filter :param boolean dry_run: if True, execute the command as a "dry run" :return: a dict containing whether the deletion succeeded, a list of deleted job names, and debug info for the underlying kubectl command :rtype: dict """ return self.client._perform_json( "POST", "/admin/clusters/%s/k8s/jobs/actions/delete-finished" % self.cluster_id, params={'deleteFailed': delete_failed, 'namespace': namespace, 'labelFilter': label_filter, 'dryRun': dry_run}) def delete_finished_pods(self, namespace=None, label_filter=None, dry_run=False): """ Runs a kubectl command to delete finished (succeeded and failed) pods. .. caution:: This operation is only valid for a Kubernetes cluster. :param string namespace: the namespace in which to delete the pods, if None, uses the namespace set in kubectl's current context :param string label_filter: delete only pods matching a label filter :param boolean dry_run: if True, execute the command as a "dry run" :return: a dict containing whether the deletion succeeded, a list of deleted pod names, and debug info for the underlying kubectl command :rtype: dict """ return self.client._perform_json( "POST", "/admin/clusters/%s/k8s/pods/actions/delete-finished" % self.cluster_id, params={'namespace': namespace, 'labelFilter': label_filter, 'dryRun': dry_run}) def delete_all_pods(self, namespace=None, label_filter=None, dry_run=False): """ Runs a kubectl command to delete all pods. .. caution:: This operation is only valid for a Kubernetes cluster. :param string namespace: the namespace in which to delete the pods, if None, uses the namespace set in kubectl's current context :param string label_filter: delete only pods matching a label filter :param boolean dry_run: if True, execute the command as a "dry run" :return: a dict containing whether the deletion succeeded, a list of deleted pod names, and debug info for the underlying kubectl command :rtype: dict """ return self.client._perform_json( "POST", "/admin/clusters/%s/k8s/pods/actions/delete-all" % self.cluster_id, params={'namespace': namespace, 'labelFilter': label_filter, 'dryRun': dry_run}) class DSSClusterSettings(object): """ The settings of a cluster. .. important:: Do not instantiate directly, use :meth:`DSSCluster.get_settings` instead. """ def __init__(self, client, cluster_id, settings): self.client = client self.cluster_id = cluster_id self.settings = settings def get_raw(self): """ Gets all settings as a raw dictionary. Changes made to the returned object will be reflected when saving. Fields that can be updated: * **permissions**, **usableByAll**, **owner** * **params** :return: reference to the raw settings, not a copy. See :meth:`DSSCluster.get_definition()` :rtype: dict """ return self.settings def get_plugin_data(self): """ Get the opaque data returned by the cluster's start. .. caution:: You should generally not modify this :return: the data stored by the plugin in the cluster, None if the cluster is not created by a plugin :rtype: dict """ return self.settings.get("data", None) def save(self): """ Saves back the settings to the cluster """ return self.client._perform_json( "PUT", "/admin/clusters/%s" % (self.cluster_id), body=self.settings) class DSSClusterStatus(object): """ The status of a cluster. .. important:: Do not instantiate directly, use :meth:`DSSCluster.get_status` instead. """ def __init__(self, client, cluster_id, status): self.client = client self.cluster_id = cluster_id self.status = status def get_raw(self): """ Gets the whole status as a raw dictionary. :return: status information, as a dict. The current state of the cluster is in a **state** field, with ossible values: NONE, STARTING, RUNNING, STOPPING :rtype: dict """ return self.status class DSSInstanceVariables(dict): """ Dict containing the instance variables. The variables can be modified directly in the dict and persisted using its :meth:`save` method. .. important:: Do not instantiate directly, use :meth:`dataikuapi.DSSClient.get_global_variables` instead. """ def __init__(self, client, variables): super(dict, self).__init__() self.update(variables) self.client = client def save(self): """ Save the changes made to the instance variables. .. note:: This call requires an API key with admin rights. """ return self.client._perform_empty("PUT", "/admin/variables/", body=self) class DSSGlobalUsageSummary(object): """ The summary of the usage of the DSS instance. .. important:: Do not instantiate directly, use :meth:`dataikuapi.DSSClient.get_global_usage_summary` instead. """ def __init__(self, data): self.data = data @property def raw(self): """ Get the usage summary structure The summary report has top-level fields per object type, like **projectSummaries** or **datasets**, each containing counts, usually a **all** global count, then several **XXXXByType** dict with counts by object sub-type (for example, for datasets the sub-type would be the type of the connection they're using) :rtype: dict """ return self.data @property def projects_count(self): """ Get the number of projects on the instance :rtype: int """ return self.data["projects"] @property def total_datasets_count(self): """ Get the number of datasets on the instance :rtype: int """ return self.data["datasets"]["all"] @property def total_recipes_count(self): """ Get the number of recipes on the instance :rtype: int """ return self.data["recipes"]["all"] @property def total_jupyter_notebooks_count(self): """ Get the number of code nobteooks on the instance :rtype: int """ return self.data["notebooks"]["nbJupyterNotebooks"] @property def total_sql_notebooks_count(self): """ Get the number of sql notebooks on the instance :rtype: int """ return self.data["notebooks"]["nbSqlNotebooks"] @property def total_scenarios_count(self): """ Get the number of scenarios on the instance :rtype: int """ return self.data["scenarios"]["all"] @property def total_active_with_trigger_scenarios_count(self): """ Get the number of active scenarios on the instance :rtype: int """ return self.data["scenarios"]["activeWithTriggers"] class DSSCodeStudioTemplateListItem(object): """ An item in a list of code studio templates. .. important:: Do not instantiate directly, use :meth:`dataikuapi.DSSClient.list_code_studio_templates` """ def __init__(self, client, data): self.client = client self._data = data def to_code_studio_template(self): """ Get the handle corresponding to this code studio template :rtype: :class:`DSSCodeStudioTemplate` """ return DSSCodeStudioTemplate(self.client, self._data["id"]) @property def label(self): """ Get the label of the template :rtype: string """ return self._data["label"] @property def id(self): """ Get the identifier of the template :rtype: string """ return self._data["id"] @property def build_for_configs(self): """ Get the list of container configurations this template is built for :return: a list of configuration name :rtype: list[string] """ return self._data.get("buildFor", []) @property def last_built(self): """ Get the timestamp of the last build of the template :return: a timestamp, or None if the template was never built :rtype: :class:`datetime.datetime` """ ts = self._data.get("lastBuilt", 0) return _timestamp_ms_to_zoned_datetime(ts) class DSSCodeStudioTemplate(object): """ A handle to interact with a code studio template on the DSS instance .. important:: Do not instantiate directly, use :meth:`dataikuapi.DSSClient.get_code_studio_template`. """ def __init__(self, client, template_id): self.client = client self.template_id = template_id ######################################################## # Template description ######################################################## def get_settings(self): """ Get the template's settings. :return: a :class:`DSSCodeStudioTemplateSettings` object to interact with code studio template settings :rtype: :class:`DSSCodeStudioTemplateSettings` """ settings = self.client._perform_json("GET", "/admin/code-studios/%s" % (self.template_id)) return DSSCodeStudioTemplateSettings(self.client, self.template_id, settings) ######################################################## # Building ######################################################## def build(self, disable_docker_cache=False): """ Build or rebuild the template. .. note:: This call needs an API key which has an user to impersonate set, or a personal API key. :param boolean disable_docker_cache: if True, the image is build with the option **--no-cache** (optional, defaults to False) :return: a handle to the task of building the image :rtype: :class:`~dataikuapi.dss.future.DSSFuture` """ future_response = self.client._perform_json( "POST", "/admin/code-studios/%s/build" % (self.template_id), params={"withNoCache": disable_docker_cache}) return DSSFuture(self.client, future_response.get('jobId', None), future_response) class DSSCodeStudioTemplateSettings(object): """ The settings of a code studio template .. important:: Do not instantiate directly, use :meth:`DSSCodeStudioTemplate.get_settings` """ def __init__(self, client, template_id, settings): self.client = client self.template_id = template_id self.settings = settings def get_raw(self): """ Gets all settings as a raw dictionary. :return: a reference to the raw settings, as a dict (not a copy). The dict contains a **type** field and the actual template settings in a **params** field. :rtype: dict """ return self.settings def get_built_for_all_container_confs(self): """ Whether the template an image for each container config :rtype: boolean """ return self.settings.get("allContainerConfs", False) def get_built_container_confs(self): """ Get the list of container configs for which the template builds an image (if not all) :return: a list of container configuration names :rtype: list[string] """ return self.settings.get("containerConfs", []) def set_built_container_confs(self, *configs, **kwargs): """ Set the list of container configs for which the template builds an image :param boolean all: if True, an image is built for each config :param list[string] configs: list of configuration names to build images for """ all = kwargs.get("all", False) self.settings['allContainerConfs'] = all if not all: self.settings['containerConfs'] = configs def save(self): """ Saves the settings of the code studio template """ self.client._perform_empty("PUT", "/admin/code-studios/%s" % (self.template_id), body=self.settings) dataiku-api-client-python-13.2.2/dataikuapi/dss/analysis.py000066400000000000000000000444461471211575100237050ustar00rootroot00000000000000from ..utils import DataikuException from ..utils import DataikuUTF8CSVReader from ..utils import DataikuStreamedHttpUTF8CSVReader import json import time from .metrics import ComputedMetrics from .ml import DSSMLTask from .utils import DSSDatasetSelectionBuilder class DSSAnalysisStepBuilder(object): def __init__(self, step_type=None, step_name=None): self.step = {'metaType':'PROCESSOR', 'type':step_type, 'name':step_name, 'params':{}} def build(self): """Returns the built step dict""" return self.step def with_type(self, step_type): """Sets the step's type""" self.step["type"] = step_type return self def with_name(self, step_name): """Sets the step's name""" self.step["name"] = step_name return self class DSSAnalysisDefinition(): """ Object to manipulate the definition of a visual analysis """ def __init__(self, analysis, acp): self.analysis = analysis self.acp = acp def get_raw(self): """ Gets the raw dictionary of the visual analysis definition """ return self.acp def get_raw_script(self): """ Gets the raw dictionary of visual analysis' script settings (including steps, sampling, ...) """ acp = self.get_raw() if not 'script' in acp: acp['script'] = {'steps':[]} return acp['script'] def get_raw_script_steps(self): """ Gets the raw dictionary of visual analysis' script steps """ script = self.get_raw_script() if not 'steps' in script: script['steps'] = [] return script['steps'] def get_raw_script_sampling(self): """ Gets the raw dictionary of visual analysis' script sampling """ script = self.get_raw_script() if not 'explorationSampling' in script: script['explorationSampling'] = {} return script['explorationSampling'] def save(self): """ Shortcut to :meth:`DSSAnalysis.set_definition()` """ self.analysis.set_definition(self) def add_step(self, step): """ Add a step to the script :param object selection: A :class:`DSSAnalysisStepBuilder` to build the settings of the step. """ steps = self.get_raw_script_steps() if isinstance(step, DSSAnalysisStepBuilder): steps.append(step.build()) else: steps.append(step) def set_script_sampling_selection(self, selection): """ Sets the sampling for the script :param object selection: A :class:`DSSDatasetSelectionBuilder` to build the settings of the extract of the dataset. """ sampling = self.get_raw_script_sampling() if isinstance(selection, DSSDatasetSelectionBuilder): sampling['selection'] = selection.build() else: sampling['selection'] = selection class DSSAnalysis(object): """A handle to interact with a DSS visual analysis""" def __init__(self, client, project_key, analysis_id): self.client = client self.project_key = project_key self.analysis_id = analysis_id ######################################################## # Analysis deletion ######################################################## def delete(self, drop_data=False): """ Delete the dataset :param bool drop_data: Should the data of the dataset be dropped """ return self.client._perform_empty("DELETE", "/projects/%s/lab/%s/" % (self.project_key, self.analysis_id)) ######################################################## # Analysis definition ######################################################## def get_definition(self): """ Get the definition of the analysis :return: a DSSAnalysisDefinition object to interact with the settings :rtype: :class:`dataikuapi.dss.analysis.DSSAnalysisDefinition` """ acp = self.client._perform_json("GET", "/projects/%s/lab/%s/" % (self.project_key, self.analysis_id)) return DSSAnalysisDefinition(self, acp) def set_definition(self, definition): """ Set the definition of the analysis Args: definition: the definition, as a JSON object or a :class:`dataikuapi.dss.analysis.DSSAnalysisDefinition`. You should only set a definition object that has been retrieved using the get_definition call. """ if isinstance(definition, DSSAnalysisDefinition): acp = definition.get_raw() else: acp = definition return self.client._perform_json("PUT", "/projects/%s/lab/%s/" % (self.project_key, self.analysis_id), body=acp) ######################################################## # ML ######################################################## def create_prediction_ml_task(self, target_variable, ml_backend_type="PY_MEMORY", guess_policy="DEFAULT", prediction_type=None, wait_guess_complete=True): """Creates a new prediction task in this visual analysis lab for a dataset. :param string target_variable: Variable to predict :param string ml_backend_type: ML backend to use, one of PY_MEMORY, MLLIB or H2O :param string guess_policy: Policy to use for setting the default parameters. Valid values are: DEFAULT, SIMPLE_FORMULA, DECISION_TREE, EXPLANATORY and PERFORMANCE :param string prediction_type: The type of prediction problem this is. If not provided the prediction type will be guessed. Valid values are: BINARY_CLASSIFICATION, REGRESSION, MULTICLASS :param boolean wait_guess_complete: if False, the returned ML task will be in 'guessing' state, i.e. analyzing the input dataset to determine feature handling and algorithms. You should wait for the guessing to be completed by calling ``wait_guess_complete`` on the returned object before doing anything else (in particular calling ``train`` or ``get_settings``) :return :class dataiku.dss.ml.DSSMLTask """ obj = { "taskType" : "PREDICTION", "targetVariable" : target_variable, "backendType": ml_backend_type, "guessPolicy": guess_policy } if prediction_type is not None: obj["predictionType"] = prediction_type ref = self.client._perform_json("POST", "/projects/%s/lab/%s/models/" % (self.project_key, self.analysis_id), body=obj) mltask = DSSMLTask(self.client, self.project_key, self.analysis_id, ref["mlTaskId"]) if wait_guess_complete: mltask.wait_guess_complete() return mltask def create_clustering_ml_task(self, ml_backend_type="PY_MEMORY", guess_policy="KMEANS", wait_guess_complete=True): """Creates a new clustering task in this visual analysis lab for a dataset. The returned ML task will be in 'guessing' state, i.e. analyzing the input dataset to determine feature handling and algorithms. You should wait for the guessing to be completed by calling ``wait_guess_complete`` on the returned object before doing anything else (in particular calling ``train`` or ``get_settings``) :param string ml_backend_type: ML backend to use, one of PY_MEMORY, MLLIB or H2O :param string guess_policy: Policy to use for setting the default parameters. Valid values are: KMEANS and ANOMALY_DETECTION :param boolean wait_guess_complete: if False, the returned ML task will be in 'guessing' state, i.e. analyzing the input dataset to determine feature handling and algorithms. You should wait for the guessing to be completed by calling ``wait_guess_complete`` on the returned object before doing anything else (in particular calling ``train`` or ``get_settings``) """ obj = { "taskType" : "CLUSTERING", "backendType": ml_backend_type, "guessPolicy": guess_policy } ref = self.client._perform_json("POST", "/projects/%s/lab/%s/models/" % (self.project_key, self.analysis_id), body=obj) mltask = DSSMLTask(self.client, self.project_key, self.analysis_id, ref["mlTaskId"]) if wait_guess_complete: mltask.wait_guess_complete() return mltask def create_timeseries_forecasting_ml_task(self, target_variable, time_variable, timeseries_identifiers=None, guess_policy="TIMESERIES_DEFAULT", wait_guess_complete=True): """Creates a new time series forecasting task in this visual analysis lab for a dataset. :param string target_variable: The variable to forecast :param string time_variable: Column to be used as time variable. Should be a Date (parsed) column. :param list timeseries_identifiers: List of columns to be used as time series identifiers (when the dataset has multiple series) :param string guess_policy: Policy to use for setting the default parameters. Valid values are: TIMESERIES_DEFAULT, TIMESERIES_STATISTICAL, and TIMESERIES_DEEP_LEARNING :param boolean wait_guess_complete: If False, the returned ML task will be in 'guessing' state, i.e. analyzing the input dataset to determine feature handling and algorithms. You should wait for the guessing to be completed by calling ``wait_guess_complete`` on the returned object before doing anything else (in particular calling ``train`` or ``get_settings``) :return :class dataiku.dss.ml.DSSMLTask """ obj = { "taskType": "PREDICTION", "targetVariable": target_variable, "timeVariable": time_variable, "timeseriesIdentifiers": timeseries_identifiers, "backendType": "PY_MEMORY", "guessPolicy": guess_policy, "predictionType": "TIMESERIES_FORECAST" } ref = self.client._perform_json( "POST", "/projects/{project_key}/lab/{analysis_id}/models/".format(project_key=self.project_key, analysis_id=self.analysis_id), body=obj ) mltask = DSSMLTask(self.client, self.project_key, self.analysis_id, ref["mlTaskId"]) if wait_guess_complete: mltask.wait_guess_complete() return mltask def create_causal_prediction_ml_task(self, outcome_variable, treatment_variable, prediction_type=None, wait_guess_complete=True): """Creates a new causal prediction task in this visual analysis lab for a dataset. :param string outcome_variable: Variable to predict :param string treatment_variable: Treatment variable :param string prediction_type: The type of prediction problem, guessed if not provided. Valid values: CAUSAL_BINARY_CLASSIFICATION, CAUSAL_REGRESSION :param boolean wait_guess_complete: if False, the returned ML task will be in 'guessing' state, i.e. analyzing the input dataset to determine feature handling and algorithms. You should wait for the guessing to be completed by calling ``wait_guess_complete`` on the returned object before doing anything else (in particular calling ``train`` or ``get_settings``) :return :class dataiku.dss.ml.DSSMLTask """ obj = { "taskType": "PREDICTION", "targetVariable": outcome_variable, "treatmentVariable": treatment_variable, "backendType": "PY_MEMORY", "guessPolicy": "CAUSAL_PREDICTION" } if prediction_type is not None: obj["predictionType"] = prediction_type ref = self.client._perform_json("POST", "/projects/%s/lab/%s/models/" % (self.project_key, self.analysis_id), body=obj) mltask = DSSMLTask(self.client, self.project_key, self.analysis_id, ref["mlTaskId"]) if wait_guess_complete: mltask.wait_guess_complete() return mltask def list_ml_tasks(self): """ List the ML tasks in this visual analysis Returns: the list of the ML tasks summaries, each one as a JSON object """ return self.client._perform_json("GET", "/projects/%s/lab/%s/models/" % (self.project_key, self.analysis_id)) def get_ml_task(self, mltask_id): """ Get a handle to interact with a specific ML task Args: mltask_id: the identifier of the desired ML task Returns: A :class:`dataikuapi.dss.ml.DSSMLTask` ML task handle """ return DSSMLTask(self.client, self.project_key, self.analysis_id, mltask_id) # some basic steps class DSSFormulaStepBuilder(DSSAnalysisStepBuilder): def __init__(self, step_name=None): super(DSSFormulaStepBuilder, self).__init__(step_type='CreateColumnWithGREL', step_name=step_name) def with_output_column(self, column_name): """Sets the step's output column's name""" self.step["params"]["column"] = column_name return self def with_error_column(self, column_name): """Sets the step's output column's name""" self.step["params"]["errorColumn"] = column_name return self def with_expression(self, expression): """Sets the step's expression""" self.step["params"]["expression"] = expression return self class AppliesToStepBuilder(DSSAnalysisStepBuilder): def __init__(self, step_type=None, step_name=None): super(AppliesToStepBuilder, self).__init__(step_type=step_type, step_name=step_name) self.step["params"]["appliesTo"] = 'SINGLE_COLUMN' def with_column_selection_mode(self, column_selection_mode): """Sets the step's column selection mode (SINGLE_COLUMN, COLUMNS, PATTERN, ALL)""" self.step["params"]["appliesTo"] = column_selection_mode return self def with_columns(self, *column_names): """Sets the step's selected columns""" self.step["params"]["columns"] = [c for c in column_names] return self def with_column_regex(self, regex): """Sets the step's column selection regular expression""" self.step["params"]["appliesToPattern"] = regex return self def with_single_column_selection(self, column_name): """Sets the step's as applying to a single column""" return self.with_column_selection_mode('SINGLE_COLUMN').with_columns(column_name) def with_multiple_column_selection(self, *column_names): """Sets the step's as applying to a single column""" return self.with_column_selection_mode('COLUMNS').with_columns(column_names) def with_regex_column_selection(self, regex): """Sets the step's as applying to a single column""" return self.with_column_selection_mode('PATTERN').with_column_regex(regex) def with_all_column_selection(self, column_name): """Sets the step's as applying to all columns""" return self.with_column_selection_mode('ALL') class FilterAndFlagStepBuilder(AppliesToStepBuilder): def __init__(self, step_type=None, step_name=None): super(FilterAndFlagStepBuilder, self).__init__(step_type=step_type, step_name=step_name) self.step["params"]["booleanMode"] = 'AND' self.step["params"]["action"] = 'REMOVE_ROW' def with_action(self, action): """Sets the step's action on match (KEEP_ROW, REMOVE_ROW, CLEAR_CELL, DONTCLEAR_CELL, FLAG)""" self.step["params"]["action"] = action return self def with_boolean_mode(self, boolean_mode): """Sets the step's mode for combining matches in different columns (AND, OR)""" self.step["params"]["booleanMode"] = boolean_mode return self def with_flag_column(self, column_name): """Sets the step's column for outputing the flag""" self.step["params"]["flagColumn"] = column_name return self class FilterOnValueStepBuilder(FilterAndFlagStepBuilder): def __init__(self, step_name=None): super(FilterOnValueStepBuilder, self).__init__(step_type='FlagOnValue', step_name=step_name) def with_values(self, *values): """Sets the step's flagged values""" self.step["params"]["values"] = [v for v in values] return self def with_matching_mode(self, matching_mode): """Sets the step's matching_mode (FULL_STRING, SUBSTRING, PATTERN)""" self.step["params"]["matchingMode"] = matching_mode return self def with_normalization_mode(self, normalization_mode): """Sets the step's normalization_mode (EXACT, LOWERCASE, NORMALIZED)""" self.step["params"]["normalizationMode"] = normalization_mode return self class FilterOnBadTypeStepBuilder(FilterAndFlagStepBuilder): def __init__(self, step_name=None): super(FilterOnBadTypeStepBuilder, self).__init__(step_type='FilterOnBadType', step_name=step_name) def with_meaning(self, meaning): """Sets the step's meaning to check""" self.step["params"]["type"] = meaning return self class RemoveRowsStepBuilder(AppliesToStepBuilder): def __init__(self, step_name=None): super(RemoveRowsStepBuilder, self).__init__(step_type='RemoveRowsOnEmpty', step_name=step_name) def with_meaning(self, keep): """Sets the step's behavior when an empty value is found : True=keep, False=drop (default)""" self.step["params"]["keep"] = keep return self dataiku-api-client-python-13.2.2/dataikuapi/dss/apideployer.py000066400000000000000000000660261471211575100243750ustar00rootroot00000000000000import json from .future import DSSFuture from ..utils import CallableStr class DSSAPIDeployer(object): """ Handle to interact with the API Deployer. Do not create this directly, use :meth:`dataikuapi.dss.DSSClient.get_apideployer` """ def __init__(self, client): self.client = client def list_deployments(self, as_objects=True): """ Lists deployments on the API Deployer :param boolean as_objects: if True, returns a list of :class:`DSSAPIDeployerDeployment`, else returns a list of dict. Each dict contains at least a field "id" indicating the identifier of this deployment :returns: a list - see as_objects for more information :rtype: list """ l = self.client._perform_json("GET", "/api-deployer/deployments") if as_objects: return [DSSAPIDeployerDeployment(self.client, x["deploymentBasicInfo"]["id"]) for x in l] else: return l def get_deployment(self, deployment_id): """ Returns a handle to interact with a single deployment, as a :class:`DSSAPIDeployerDeployment` :param str deployment_id: Identifier of the deployment to get :rtype: :class:`DSSAPIDeployerDeployment` """ return DSSAPIDeployerDeployment(self.client, deployment_id) def create_deployment(self, deployment_id, service_id, infra_id, version, endpoint_id=None, ignore_warnings=False): """ Creates a deployment and returns the handle to interact with it. The returned deployment is not yet started and you need to call :meth:`~DSSAPIDeployerDeployment.start_update` :param str deployment_id: Identifier of the deployment to create :param str service_id: Identifier of the API Service to target :param str infra_id: Identifier of the deployment infrastructure to use :param str version: Identifier of the API Service version to deploy :param str endpoint_id: Identifier of the endpoint to deploy if you use a Deploy Anywhere infra. Ignored otherwise :param boolean ignore_warnings: ignore warnings concerning the governance status of the model version(s) to deploy :rtype: :class:`DSSAPIDeployerDeployment` """ settings = { "deploymentId": deployment_id, "publishedServiceId": service_id, "infraId": infra_id, "version": version, "endpointId": endpoint_id } self.client._perform_json("POST", "/api-deployer/deployments", params={"ignoreWarnings": ignore_warnings}, body=settings) return self.get_deployment(deployment_id) def list_stages(self): """ Lists infrastructure stages of the API Deployer :rtype: list of dict. Each dict contains a field "id" for the stage identifier and "desc" for its description. :rtype: list """ return self.client._perform_json("GET", "/api-deployer/stages") def list_infras(self, as_objects=True): """ Lists deployment infrastructures on the API Deployer :param boolean as_objects: if True, returns a list of :class:`DSSAPIDeployerInfra`, else returns a list of dict. Each dict contains at least a field "id" indicating the identifier of this infra :returns: a list - see as_objects for more information :rtype: list """ l = self.client._perform_json("GET", "/api-deployer/infras") if as_objects: return [DSSAPIDeployerInfra(self.client, x["infraBasicInfo"]["id"]) for x in l] else: return l def create_infra(self, infra_id, stage, type, govern_check_policy="NO_CHECK"): """ Creates a new infrastructure on the API Deployer and returns the handle to interact with it. :param str infra_id: Unique Identifier of the infra to create :param str stage: Infrastructure stage. Stages are configurable on each API Deployer :param str type: STATIC, K8S, AZURE_ML, SAGEMAKER, SNOWPARK or VERTEX_AI :param str govern_check_policy: PREVENT, WARN, or NO_CHECK depending if the deployer will check whether the saved model versions deployed on this infrastructure has to be managed and approved in Dataiku Govern :rtype: :class:`DSSAPIDeployerInfra` """ settings = { "id": infra_id, "stage": stage, "type": type, "governCheckPolicy": govern_check_policy, } self.client._perform_json("POST", "/api-deployer/infras", body=settings) return self.get_infra(infra_id) def get_infra(self, infra_id): """ Returns a handle to interact with a single deployment infra, as a :class:`DSSAPIDeployerInfra` :param str infra_id: Identifier of the infra to get :rtype: :class:`DSSAPIDeployerInfra` """ return DSSAPIDeployerInfra(self.client, infra_id) def list_services(self, as_objects=True): """ Lists API services on the API Deployer :param boolean as_objects: if True, returns a list of :class:`DSSAPIDeployerService`, else returns a list of dict. Each dict contains at least a field "id" indicating the identifier of this Service :returns: a list - see as_objects for more information :rtype: list """ l = self.client._perform_json("GET", "/api-deployer/services") if as_objects: return [DSSAPIDeployerService(self.client, x["serviceBasicInfo"]["id"]) for x in l] else: return l def create_service(self, service_id): """ Creates a new API Service on the API Deployer and returns the handle to interact with it. :param str service_id: Identifier of the API Service to create :rtype: :class:`DSSAPIDeployerService` """ settings = { "publishedServiceId" : service_id } self.client._perform_json("POST", "/api-deployer/services", body=settings) return self.get_service(service_id) def get_service(self, service_id): """ Returns a handle to interact with a single service, as a :class:`DSSAPIDeployerService` :param str service_id: Identifier of the API service to get :rtype: :class:`DSSAPIDeployerService` """ return DSSAPIDeployerService(self.client, service_id) ############################################### # Infrastructures ############################################### class DSSAPIDeployerInfra(object): """ An API Deployer infrastructure. Do not create this directly, use :meth:`~dataikuapi.dss.apideployer.DSSAPIDeployer.get_infra` """ def __init__(self, client, infra_id): self.client = client self.infra_id = infra_id @property def id(self): return CallableStr(self.infra_id) def get_status(self): """ Returns status information about this infrastructure :rtype: :class:`dataikuapi.dss.apideployer.DSSAPIDeployerInfraStatus` """ light = self.client._perform_json("GET", "/api-deployer/infras/%s" % (self.infra_id)) return DSSAPIDeployerInfraStatus(self.client, self.infra_id, light) def get_settings(self): """ Gets the settings of this infra. If you want to modify the settings, you need to call :meth:`~dataikuapi.dss.apideployer.DSSAPIDeployerInfraSettings.save` on the returned object :returns: a :class:`dataikuapi.dss.apideployer.DSSAPIDeployerInfraSettings` """ settings = self.client._perform_json( "GET", "/api-deployer/infras/%s/settings" % (self.infra_id)) return DSSAPIDeployerInfraSettings(self.client, self.infra_id, settings) def delete(self): """ Deletes this infra You may only delete an infra if it has no deployments on it anymore. """ self.client._perform_empty( "DELETE", "/api-deployer/infras/%s" % (self.infra_id)) class DSSAPIDeployerInfraSettings(object): """ The settings of an API Deployer infrastructure Do not create this directly, use :meth:`~dataikuapi.dss.apideployer.DSSAPIDeployerInfra.get_settings` """ def __init__(self, client, infra_id, settings): self.client = client self.infra_id = infra_id self.settings = settings def get_type(self): """ Gets the type of this infra :returns: the type of this infra :rtype: string """ return self.settings["type"] def add_apinode(self, url, api_key, graphite_prefix=None): """ Adds an API node to the list of nodes of this infra. Only applicable to STATIC infrastructures :param str url: url of the API node that will be added to this infra :param str api_key: api key secret to connect to the API node :param str graphite_prefix: graphite prefix for metric reports if graphite is configured """ new_node = { "url": url, "adminAPIKey" : api_key, "graphitePrefix" : graphite_prefix } self.settings["apiNodes"].append(new_node) def remove_apinode(self, node_url): """ Removes a node from the list of nodes of this infra. Only applicable to STATIC infrastructures :param str node_url: URL of the node to remove """ api_nodes = list(self.settings["apiNodes"]) for node in api_nodes: if node.get("url") == node_url: self.settings["apiNodes"].remove(node) def get_raw(self): """ Gets the raw settings of this infra. This returns a reference to the raw settings, not a copy, so changes made to the returned object will be reflected when saving. :rtype: dict """ return self.settings def save(self): """Saves back these settings to the infra""" self.client._perform_empty( "PUT", "/api-deployer/infras/%s/settings" % (self.infra_id), body = self.settings) class DSSAPIDeployerInfraStatus(object): """ The status of an API Deployer infrastructure. Do not create this directly, use :meth:`~dataikuapi.dss.apideployer.DSSAPIDeployerInfra.get_status` """ def __init__(self, client, infra_id, light_status): self.client = client self.infra_id = infra_id self.light_status = light_status def get_deployments(self): """ Returns the deployments that are deployed on this infrastructure :returns: a list of deployments :rtype: list of :class:`dataikuapi.dss.apideployer.DSSAPIDeployerDeployment` """ return [DSSAPIDeployerDeployment(self.client, deployment["id"]) for deployment in self.light_status["deployments"]] def get_raw(self): """ Gets the raw status information. This returns a dictionary with various information about the infrastructure :rtype: dict """ return self.light_status ############################################### # Deployments ############################################### class DSSAPIDeployerDeployment(object): """ A Deployment on the API Deployer Do not create this directly, use :meth:`~dataikuapi.dss.apideployer.DSSAPIDeployer.get_deployment` """ def __init__(self, client, deployment_id): self.client = client self.deployment_id = deployment_id @property def id(self): return CallableStr(self.deployment_id) def get_status(self): """ Returns status information about this deployment :rtype: dataikuapi.dss.apideployer.DSSAPIDeployerDeploymentStatus """ light = self.client._perform_json("GET", "/api-deployer/deployments/%s" % (self.deployment_id)) heavy = self.client._perform_json("GET", "/api-deployer/deployments/%s/status" % (self.deployment_id)) return DSSAPIDeployerDeploymentStatus(self.client, self.deployment_id, light, heavy) def get_governance_status(self, version=""): """ Returns the governance status about this deployment if applicable It covers all the embedded model versions :param str version: (Optional) The specific package version of the published service to get status from. If empty, consider all the versions used in the deployment generation mapping. :rtype: dict InforMessages containing the governance status """ return self.client._perform_json("GET", "/api-deployer/deployments/%s/governance-status" % (self.deployment_id), params={ "version": version }) def get_settings(self): """ Gets the settings of this deployment. If you want to modify the settings, you need to call :meth:`~dataikuapi.dss.apideployer.DSSAPIDeployerDeploymentSettings.save` on the returned object :returns: a :class:`dataikuapi.dss.apideployer.DSSAPIDeployerDeploymentSettings` """ settings = self.client._perform_json( "GET", "/api-deployer/deployments/%s/settings" % (self.deployment_id)) return DSSAPIDeployerDeploymentSettings(self.client, self.deployment_id, settings) def start_update(self): """ Starts an asynchronous update of this deployment to try to match the actual state to the current settings :returns: a :class:`dataikuapi.dss.future.DSSFuture` tracking the progress of the update. Call :meth:`~dataikuapi.dss.future.DSSFuture.wait_for_result` on the returned object to wait for completion (or failure) """ future_response = self.client._perform_json( "POST", "/api-deployer/deployments/%s/actions/update" % (self.deployment_id)) return DSSFuture(self.client, future_response.get('jobId', None), future_response) def delete(self, disable_first=False, ignore_pre_delete_errors=False): """ Deletes this deployment. The disable_first flag automatically disables the deployment before its deletion. :param boolean disable_first: If True, automatically disables this deployment before deleting it. If False, will raise an Exception if this deployment is enabled. :param boolean ignore_pre_delete_errors: If True, any error occurred during the actions performed previously to delete the deployment will be ignored and the delete action will be performed anyway. """ # Check if the deployment is disabled is_enabled = self.get_status().light_status["deploymentBasicInfo"].get("enabled") if is_enabled and not disable_first: raise Exception("Deployment {} deletion failed: deployment must be disabled first.".format(self.deployment_id)) if is_enabled: settings = self.get_settings() settings.set_enabled(enabled=False) settings.save() self.client._perform_empty( "DELETE", "/api-deployer/deployments/%s" % (self.deployment_id), params = { "ignorePreDeleteErrors" : ignore_pre_delete_errors } ) def get_open_api(self): """ Gets the OpenAPI document of this deployment if it's available or raise a 404 error. :returns: a :class:`dataikuapi.dss.apideployer.DSSAPIDeployerDeploymentOpenApi` """ open_api = self.client._perform_text("GET", "/api-deployer/deployments/%s/get-open-api" % (self.deployment_id)) return DSSAPIDeployerDeploymentOpenApi(open_api) def run_test_queries(self, endpoint_id=None, test_queries=None): """ Runs test queries on a deployment and returns results as a dict :param str endpoint_id: Mandatory if the deployment has multiple endpoints :param list test_queries: Queries as str, formatted as [{"q": {"features": {"feat_1": "value", ...}}, {...}, ... ]. If left to None, the test queries of the current version of the service will be used. :rtype: dict Usage example .. code-block:: python import dataiku client = dataiku.api_client() deployer = client.get_apideployer() deployment = deployer.get_deployment('service14'); test_queries = [{'q': {'features': { 'Pclass': '200', 'Sex': 'male', 'Age': '22', 'Embarked': 'S' }}}] # run existing test queries on deployement endpoint (if unique, else error) test_queries_result = deployment.run_test_queries() # run specified test queries on deployement "survived" endpoint test_queries_result = deployment.run_test_queries(endpoint_id="survived", test_queries=test_queries) # run existing test queries on deployement "survived" endpoint test_queries_result = deployment.run_test_queries(endpoint_id="survived") # run specified test queries on deployement endpoint (if unique, else error) test_queries_result = deployment.run_test_queries(test_queries=test_queries) """ settings = {} if endpoint_id is not None: settings["endpointIdParam"] = endpoint_id if test_queries is not None: settings["testQueriesParam"] = json.dumps(test_queries) return self.client._perform_json("POST", "/api-deployer/deployments/%s/actions/run-test-queries" % self.deployment_id, params=settings) class DSSAPIDeployerDeploymentSettings(object): """ The settings of an API Deployer deployment. Do not create this directly, use :meth:`~dataikuapi.dss.apideployer.DSSAPIDeployerDeployment.get_settings` """ def __init__(self, client, deployment_id, settings): self.client = client self.deployment_id = deployment_id self.settings = settings def get_raw(self): """ Gets the raw settings of this deployment. This returns a reference to the raw settings, not a copy, so changes made to the returned object will be reflected when saving. :rtype: dict """ return self.settings def set_enabled(self, enabled): """ Enables or disables this deployment :param bool enabled: True/False to Enable/Disable this deployment """ self.settings["enabled"] = enabled def set_single_version(self, version): """ Sets this deployment to target a single version of the API service :param str version: Identifier of the version to set """ self.settings["generationsMapping"] = { "mode": "SINGLE_GENERATION", "generation": version } def save(self, ignore_warnings=False): """ Saves back these settings to the deployment :param boolean ignore_warnings: ignore warnings concerning the governance status of the model version(s) to deploy """ self.client._perform_empty( "PUT", "/api-deployer/deployments/%s/settings" % (self.deployment_id), params = { "ignoreWarnings" : ignore_warnings }, body = self.settings) class DSSAPIDeployerDeploymentStatus(object): """ The status of an API Deployer deployment. Do not create this directly, use :meth:`~dataikuapi.dss.apideployer.DSSAPIDeployerDeployment.get_status` """ def __init__(self, client, deployment_id, light_status, heavy_status): self.client = client self.deployment_id = deployment_id self.light_status = light_status self.heavy_status = heavy_status def get_light(self): """ Gets the 'light' (summary) status. This returns a dictionary with various information about the deployment, but not the actual health of the deployment :rtype: dict """ return self.light_status def get_heavy(self): """ Gets the 'heavy' (full) status. This returns a dictionary with various information about the deployment :rtype: dict """ return self.heavy_status def get_service_urls(self): """ Returns service-level URLs for this deployment (ie without the enpdoint-specific suffix) :returns: a list of service-level URLs for this deployment :rtype: list """ if "deployedServiceId" in self.light_status["deploymentBasicInfo"]: service_id = self.light_status["deploymentBasicInfo"]["deployedServiceId"] else: service_id = self.light_status["deploymentBasicInfo"]["publishedServiceId"] if "apiNodes" in self.heavy_status: return ["%s/public/api/v1/%s" % (x["url"], service_id) for x in self.heavy_status["apiNodes"]] elif "publicURL" in self.heavy_status: return ["%s/public/api/v1/%s" % (self.heavy_status["publicURL"], service_id)] else: raise ValueError("PublicURL not available for this deployment. It might still be initializing") def get_health(self): """ Returns the health of this deployment as a string :returns: HEALTHY if the deployment is working properly, various other status otherwise :rtype: string """ return self.heavy_status["health"] def get_health_messages(self): """Returns messages about the health of this deployment""" return self.heavy_status["healthMessages"] class DSSAPIDeployerDeploymentOpenApi(object): """ The OpenAPI document of an API Deployer deployment. Do not create this directly, use :meth:`~dataikuapi.dss.apideployer.DSSAPIDeployerDeployment.get_open_api` """ def __init__(self, open_api_doc_json): self.open_api_doc_json = open_api_doc_json def get(self): """ Gets the OpenAPI document as dict. :rtype: dict """ return json.loads(self.open_api_doc_json) def get_raw(self): """ Gets the OpenAPI document raw. :rtype: string """ return self.open_api_doc_json ############################################### # Published Service ############################################### class DSSAPIDeployerService(object): """ An API service on the API Deployer Do not create this directly, use :meth:`~dataikuapi.dss.apideployer.DSSAPIDeployer.get_service` """ def __init__(self, client, service_id): self.client = client self.service_id = service_id @property def id(self): return CallableStr(self.service_id) def get_status(self): """ Returns status information about this service. This is used mostly to get information about which versions are available and which deployments are exposing this service :rtype: dataikuapi.dss.apideployer.DSSAPIDeployerServiceStatus """ light = self.client._perform_json("GET", "/api-deployer/services/%s" % (self.service_id)) return DSSAPIDeployerServiceStatus(self.client, self.service_id, light) def import_version(self, fp): """ Imports a new version for an API service from a file-like object pointing to a version package Zip file :param string fp: A file-like object pointing to a version package Zip file """ return self.client._perform_empty("POST", "/api-deployer/services/%s/versions" % (self.service_id), files={"file":fp}) def get_settings(self): """ Gets the settings of this service. If you want to modify the settings, you need to call :meth:`~dataikuapi.dss.apideployer.DSSAPIDeployerServiceSettings.save` on the returned object. The main things that can be modified in a service settings are permissions :returns: a :class:`dataikuapi.dss.apideployer.DSSAPIDeployerServiceSettings` """ settings = self.client._perform_json( "GET", "/api-deployer/services/%s/settings" % (self.service_id)) return DSSAPIDeployerServiceSettings(self.client, self.service_id, settings) def delete_version(self, version): """ Deletes a version from this service :param string version: The version to delete """ self.client._perform_empty( "DELETE", "/api-deployer/services/%s/versions/%s" % (self.service_id, version)) def delete(self): """ Deletes this service You may only delete a service if it has no deployments on it anymore. """ self.client._perform_empty( "DELETE", "/api-deployer/services/%s" % (self.service_id)) class DSSAPIDeployerServiceSettings(object): """ The settings of an API Deployer Service. Do not create this directly, use :meth:`~dataikuapi.dss.apideployer.DSSAPIDeployerService.get_settings` """ def __init__(self, client, service_id, settings): self.client = client self.service_id = service_id self.settings = settings def get_raw(self): """ Gets the raw settings of this deployment. This returns a reference to the raw settings, not a copy, so changes made to the returned object will be reflected when saving. :rtype: dict """ return self.settings def save(self): """ Saves back these settings to the API service """ self.client._perform_empty( "PUT", "/api-deployer/services/%s/settings" % (self.service_id), body = self.settings) class DSSAPIDeployerServiceStatus(object): """ The status of an API Deployer Service. Do not create this directly, use :meth:`~dataikuapi.dss.apideployer.DSSAPIDeployerService.get_status` """ def __init__(self, client, service_id, light_status): self.client = client self.service_id = service_id self.light_status = light_status def get_deployments(self, infra_id=None): """ Returns the deployments that have been created from this published project :param str infra_id: Identifier of an infra, allows to only keep in the returned list the deployments on this infra. If not set, the list contains all the deployments using this published project, across every infra of the Project Deployer. :returns: a list of deployments :rtype: list of :class:`dataikuapi.dss.apideployer.DSSAPIDeployerDeployment` """ if infra_id is None: return [DSSAPIDeployerDeployment(self.client, deployment["id"]) for deployment in self.light_status["deployments"]] return [DSSAPIDeployerDeployment(self.client, deployment["id"]) for deployment in self.light_status["deployments"] if infra_id == deployment["infraId"]] def get_versions(self): """ Returns the versions of this service that have been published on the API Service Each version is a dict that contains at least a "id" field, which is the version identifier :returns: a list of versions, each as a dict containing a "id" field :rtype: list of dicts """ return self.light_status["packages"] def get_raw(self): """ Gets the raw status information. This returns a dictionary with various information about the service, :rtype: dict """ return self.light_status dataiku-api-client-python-13.2.2/dataikuapi/dss/apiservice.py000066400000000000000000000276071471211575100242140ustar00rootroot00000000000000class DSSAPIServiceListItem(dict): """ An item in a list of API services. .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.list_api_services()` """ def __init__(self, client, data): super(DSSAPIServiceListItem, self).__init__(data) self.client = client def to_api_service(self): """ Get a handle corresponding to this API service. :rtype: :class:`DSSAPIService` """ return DSSAPIService(self.client, self["projectKey"], self["id"]) @property def name(self): """ Get the name of the API service. :rtype: string """ return self["name"] @property def id(self): """ Get the identifier of the API service. :rtype: string """ return self["id"] @property def auth_method(self): """ Get the method used to authenticate on the API service. Usage example: .. code-block:: python # list all public API services for service in project.list_api_services(as_type="list_item"): if service.auth_method == 'PUBLIC': print("Service {} isn't authenticating requests".format(service.id)) :return: an authentication method. Possible values: PUBLIC (no authentication), API_KEYS, OAUTH2 :rtype: string """ return self["authMethod"] @property def endpoints(self): """ Get the endpoints in this API service. :return: a list of endpoints, each one a dict with fields: * **id** : identifier of the endpoint * **type** : type of endpoint. Possible values: STD_PREDICTION, STD_CLUSTERING, STD_FORECAST, STD_CAUSAL_PREDICTION, CUSTOM_PREDICTION, CUSTOM_R_PREDICTION, R_FUNCTION, PY_FUNCTION, DATASETS_LOOKUP, SQL_QUERY :rtype: list[dict] """ return self["endpoints"] class DSSAPIServiceSettings(object): """ The settings of an API Service in the API Designer. .. important:: Do not instantiate directly, use :meth:`DSSAPIService.get_settings`. """ def __init__(self, client, project_key, service_id, settings): self.client = client self.project_key = project_key self.service_id = service_id self.settings = settings def get_raw(self): """ Get the raw settings of this API Service. This returns a reference to the raw settings, not a copy, so changes made to the returned object will be reflected when saving. :return: the settings of the API service, as a dict. The definitions of the endpoints are inside the **endpoints** field, itself a list of dict. :rtype: dict """ return self.settings @property def auth_method(self): """ Get the method used to authenticate on the API service :return: an authentication method. Possible values: PUBLIC (no authentication), API_KEYS, OAUTH2 :rtype: string """ return self.settings["authMethod"] @property def endpoints(self): """ Get the list of endpoints of this API service :return: ist of endpoints, each one a dict. Endpoint have different fields depending on their type, but always have at least: * **id** : identifier of the endpoint * **type** : type of endpoint. Possible values: STD_PREDICTION, STD_CLUSTERING, STD_FORECAST, STD_CAUSAL_PREDICTION, CUSTOM_PREDICTION, CUSTOM_R_PREDICTION, R_FUNCTION, PY_FUNCTION, DATASETS_LOOKUP, SQL_QUERY :rtype: list[dict] """ return self.settings["endpoints"] def add_prediction_endpoint(self, endpoint_id, saved_model_id): """ Add a new "visual prediction" endpoint to this API service. :param string endpoint_id: identifier of the new endpoint to create :param string saved_model_id: identifier of the saved model (that is currently deployed to the Flow) to use """ self.settings["endpoints"].append({ "id" : endpoint_id, "type" : "STD_PREDICTION", "modelRef": saved_model_id }) def add_clustering_endpoint(self, endpoint_id, saved_model_id): """ Add a new "visual clustering" endpoint to this API service. :param string endpoint_id: identifier of the new endpoint to create :param string saved_model_id: identifier of the saved model (that is currently deployed to the Flow) to use """ self.settings["endpoints"].append({ "id" : endpoint_id, "type" : "STD_CLUSTERING", "modelRef": saved_model_id }) def add_forecasting_endpoint(self, endpoint_id, saved_model_id): """ Add a new "visual time series forecasting" endpoint to this API service. :param string endpoint_id: identifier of the new endpoint to create :param string saved_model_id: identifier of the saved model (that is currently deployed to the Flow) to use """ self.settings["endpoints"].append({ "id" : endpoint_id, "type" : "STD_FORECAST", "modelRef": saved_model_id }) def add_causal_prediction_endpoint(self, endpoint_id, saved_model_id, compute_propensity=False): """ Add a new "visual causal prediction" endpoint to this API service. :param string endpoint_id: identifier of the new endpoint to create :param string saved_model_id: identifier of the saved model (that is currently deployed to the Flow) to use :param bool compute_propensity: whether propensity should be computed, if True, the model must have a trained propensity model """ self.settings["endpoints"].append({ "id": endpoint_id, "type": "STD_CAUSAL_PREDICTION", "modelRef": saved_model_id, "computePropensity": compute_propensity, "useJava": False # Not supported for causal predictions }) def save(self): """ Save back these settings to the API Service. """ self.client._perform_empty( "PUT", "/projects/%s/apiservices/%s/settings" % (self.project_key, self.service_id), body = self.settings) class DSSAPIService(object): """ An API Service from the API Designer on the DSS instance. .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.get_api_service` """ def __init__(self, client, project_key, service_id): self.client = client self.project_key = project_key self.service_id = service_id @property def id(self): """ Get the API service's identifier :rtype: string """ return self.service_id def get_settings(self): """ Get the settings of this API Service. Usage example: .. code-block:: python # list all API services using a given model model_lookup = "my_saved_model_id" model = project.get_saved_model(model_lookup) model_name = model.get_settings().get_raw()["name"] for service in project.list_api_services(as_type='object'): settings = service.get_settings() endpoints_on_model = [e for e in settings.endpoints if e.get("modelRef", '') == model_lookup] if len(endpoints_on_model) > 0: print("Service {} uses model {}".format(service.id, model_name)) :return: a handle on the settings :rtype: :class:`DSSAPIServiceSettings` """ settings = self.client._perform_json( "GET", "/projects/%s/apiservices/%s/settings" % (self.project_key, self.service_id)) return DSSAPIServiceSettings(self.client, self.project_key, self.service_id, settings) def get_package_summary(self, package_id): """ Get summary of a package :param str package_id: version (identifier) of the package to get the summary for :rtype: dict """ return self.client._perform_json( "GET", "/projects/%s/apiservices/%s/packages/%s/summary" % (self.project_key, self.service_id, package_id), ) def list_packages(self): """ List the versions of this API service. :return: a list of packages, each one as a dict. Each dict has fields: * **id** : version (identifier) of the package * **createdOn** : timestamp in milliseconds of when the package was created :rtype: list[dict] """ return self.client._perform_json( "GET", "/projects/%s/apiservices/%s/packages" % (self.project_key, self.service_id)) def create_package(self, package_id, release_notes=None): """ Create a new version of this API service. :param string package_id: version (identifier) of the package to create :param str release_notes: important changes introduced in the package """ self.client._perform_empty( method="POST", path="/projects/%s/apiservices/%s/packages/%s" % (self.project_key, self.service_id, package_id), params={"releaseNotes": release_notes}, ) def delete_package(self, package_id): """ Delete a version of this API service. :param string package_id: version (identifier) of the package to delete """ self.client._perform_empty( "DELETE", "/projects/%s/apiservices/%s/packages/%s" % (self.project_key, self.service_id, package_id)) def download_package_stream(self, package_id): """ Download an archive of a package as a stream. The archive can then be deployed in a DSS API Node. .. warning:: This call will monopolize the DSSClient until the stream it returns is closed. :param string package_id: version (identifier) of the package to download :return: the package archive, as a HTTP stream :rtype: file-like """ return self.client._perform_raw( "GET", "/projects/%s/apiservices/%s/packages/%s/archive" % (self.project_key, self.service_id, package_id)).raw def download_package_to_file(self, package_id, path): """ Download an archive of a package to a local file. The archive can then be deployed in a DSS API Node. :param string package_id: version (identifier) of the package to download :param string path: absolute or relative path to a file in which the package is downloaded """ package_stream = self.client._perform_raw( "GET", "/projects/%s/apiservices/%s/packages/%s/archive" % (self.project_key, self.service_id, package_id)) with open(path, 'wb') as f: for chunk in package_stream.iter_content(chunk_size=10000): if chunk: f.write(chunk) f.flush() def publish_package(self, package_id, published_service_id=None): """ Publish a package on the API Deployer. :param string package_id: version (identifier) of the package to publish :param string published_service_id: identifier of the API service on the API Deployer in which the package will be published. A new published API service will be created if none matches the identifier. If the parameter is not set, the identifier from the current :class:`DSSAPIService` is used. """ params = None if published_service_id is not None: params = {"publishedServiceId": published_service_id} self.client._perform_empty("POST", "/projects/%s/apiservices/%s/packages/%s/publish" % (self.project_key, self.service_id, package_id), params=params) dataiku-api-client-python-13.2.2/dataikuapi/dss/app.py000066400000000000000000000213031471211575100226250ustar00rootroot00000000000000import random import re import string from .future import DSSFuture from .utils import DSSTaggableObjectListItem def random_string(length): return ''.join(random.choice(string.ascii_letters) for _ in range(length)) class DSSAppListItem(DSSTaggableObjectListItem): """ An item in a list of apps. .. important:: Do not instantiate this class, use :meth:`dataikuapi.DSSClient.list_apps` instead """ def __init__(self, client, data): super(DSSAppListItem, self).__init__(data) self.client = client def to_app(self): """ Get a handle corresponding to this app. :return: a handle to interact with the app :rtype: :class:`dataikuapi.dss.app.DSSApp` """ return DSSApp(self.client, self._data["appId"]) class DSSApp(object): """ A handle to interact with an application on the DSS instance. .. important:: Do not instantiate this class directly, instead use :meth:`dataikuapi.DSSClient.get_app` """ def __init__(self, client, app_id): self.client = client self.app_id = app_id ######################################################## # Instances ######################################################## def create_instance(self, instance_key, instance_name, wait=True): """ Create a new instance of this application. Each instance must have a globally unique instance key, separate from any other project key across the whole DSS instance. :param string instance_key: project key for the new created app instance :param string instance_name: name for the new created app instance :param boolean wait: if False, the method returns immediately with a :class:`dataikuapi.dss.future.DSSFuture` on which to wait for the app instance to be created :return: a handle to interact with the app instance :rtype: :class:`~DSSAppInstance` """ future_resp = self.client._perform_json( "POST", "/apps/%s/instances" % self.app_id, body={ "targetProjectKey": instance_key, "targetProjectName": instance_name }) future = DSSFuture(self.client, future_resp.get("jobId", None), future_resp) if wait: future.wait_for_result() return DSSAppInstance(self.client, instance_key) else: return future def make_random_project_key(self): """ Create a new project key based on this app name. This method suffixes the app's name with a random string to generate a unique app instance key. :return: a project key :rtype: string """ slugified_app_id = re.sub(r'[^A-Za-z_0-9]+', '_', self.app_id) return "%s_tmp_%s" % (slugified_app_id, random_string(10)) def create_temporary_instance(self): """ Create a new temporary instance of this application. The return value should be used as a Python context manager. Upon exit, the temporary app instance is deleted. :return: an app instance :rtype: :class:`TemporaryDSSAppInstance` """ key = self.make_random_project_key() self.create_instance(key, key, True) return TemporaryDSSAppInstance(self.client, key) def list_instance_keys(self): """ List the keys of the existing instances of this app. :return: a list of instance keys :rtype: list[string] """ return [x["projectKey"] for x in self.list_instances()] def list_instances(self): """ List the existing instances of this app. :return a list of instances, each as a dict containing at least a "projectKey" field :rtype: list """ return self.client._perform_json( "GET", "/apps/%s/instances/" % self.app_id) def get_instance(self, instance_key): """ Get a particular instance of this app. :return: an app instance :rtype: :class:`~DSSAppInstance` """ return DSSAppInstance(self.client, instance_key) def get_manifest(self): """ Get the manifest of this app. :return: an app manifest :rtype: :class:`~DSSAppManifest` """ raw_data = self.client._perform_json("GET", "/apps/%s/" % self.app_id) project_key = self.app_id[8:] if self.app_id.startswith('PROJECT_') else None return DSSAppManifest(self.client, raw_data, project_key) class DSSAppManifest(object): """ Handle on the manifest of an app or an app instance. .. important:: Do not instantiate this class directly, use :meth:`dataikuapi.dss.app.DSSApp.get_manifest()` or :meth:`dataikuapi.dss.app.DSSAppInstance.get_manifest()` """ def __init__(self, client, raw_data, project_key=None): self.client = client self.raw_data = raw_data self.project_key = project_key def get_raw(self): """ Get the raw definition of the manifest. Usage example: .. code-block:: python # list all app templates that anybody can instantiate for app in client.list_apps(as_type="objects"): manifest = app.get_manifest() if manifest.get_raw()["instantiationPermission"] == 'EVERYBODY': print(app.app_id) :return: the definition of the manifest, as a dict. The definitions of the tiles of the app are inside the **homepageSections** field, which is a list of the sections displayed in the app. When the app is an app-as-recipe, the field **useAsRecipeSettings** is defined and contains the recipe-specific settings. :rtype: dict """ return self.raw_data def get_all_actions(self): """ Get the flat list of all actions. :return: a list of action defintions, each one a dict. Each action has fields * **type** : the type of the action * **prompt** : label of the action in the form * **help** and **helpTitle** : metadata for showing a help button on the action * ... and additional fields depending on the type, to hold the action's setup :rtype: list """ return [x for section in self.raw_data["homepageSections"] for x in section["tiles"]] def get_runnable_scenarios(self): """ Get the scenario identifiers that are declared as actions for this app. :return: a list of scenario identifiers :rtype: list[string] """ return [x["scenarioId"] for x in self.get_all_actions() if x["type"] == "SCENARIO_RUN"] def save(self): """ Save the changes to this manifest object back to the template project. """ if self.project_key is None: raise Exception("This manifest object wasn't created from a project, cannot be saved back") self.client._perform_empty("PUT", "/projects/%s/app-manifest" % self.project_key, body=self.raw_data) class DSSAppInstance(object): """ Handle on an instance of an app. .. important:: Do not instantiate this class directly, use :meth:`dataikuapi.dss.app.DSSApp.get_instance()` """ def __init__(self, client, project_key): self.client = client self.project_key = project_key def get_as_project(self): """ Get a handle on the project corresponding to this application instance. :return: a handle on a DSS prohect :rtype: :class:`dataikuapi.dss.project.DSSProject` """ return self.client.get_project(self.project_key) def get_manifest(self): """ Get the manifest of this app instance. :return: an app manifest :rtype: :class:`~DSSAppManifest` """ raw_data = self.client._perform_json("GET", "/projects/%s/app-manifest" % self.project_key) return DSSAppManifest(self.client, raw_data) class TemporaryDSSAppInstance(DSSAppInstance): """ Variant of :class:`~DSSAppInstance` that can be used as a Python context. .. important:: Do not instantiate this class directly, use :meth:`dataikuapi.dss.app.DSSApp.create_temporary_instance()` """ def __init__(self, client, project_key): DSSAppInstance.__init__(self, client,project_key) def close(self): """ Delete the app instance. """ self.get_as_project().delete(clear_managed_datasets=True) def __enter__(self,): return self def __exit__(self, type, value, traceback): self.close() dataiku-api-client-python-13.2.2/dataikuapi/dss/codestudio.py000066400000000000000000000361121471211575100242130ustar00rootroot00000000000000from ..utils import DataikuException, _timestamp_ms_to_zoned_datetime import json from datetime import datetime from .future import DSSFuture class DSSCodeStudioObjectListItem(object): """ An item in a list of code studios. .. important:: Do not instantiate this class, use :meth:`dataikuapi.dss.project.DSSProject.list_code_studios` """ def __init__(self, client, project_key, data): self.client = client self.project_key = project_key self._data = data def to_code_studio(self): """ Get a handle to interact with this code studio :return: a handle on the code studio :rtype: :class:`dataikuapi.dss.codestudio.DSSCodeStudioObject` """ return DSSCodeStudioObject(self.client, self.project_key, self._data["id"]) @property def name(self): """ Get the name of the code studio .. note:: The name is user-provided and not necessarily unique. :rtype: string """ return self._data["name"] @property def id(self): """ Get the identifier of the code studio .. note:: The id is generated by DSS upon creation and random. :rtype: string """ return self._data["id"] @property def owner(self): """ Get the login of the owner of the code studio :rtype: string """ return self._data["owner"] @property def template_id(self): """ Get the identifier of the template that this code studio was created from :rtype: string """ return self._data["templateId"] @property def template_label(self): """ Get the label of the template that this code studio was created from :rtype: string """ return self._data.get('desc', {}).get('label', self._data['id']) @property def template_description(self): """ Get the description of the template that this code studio was created from :rtype: string """ return self._data.get('desc', {}).get('shortDesc', '') class DSSCodeStudioObject(object): """ A handle to manage a code studio in a project .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.get_code_studio` or :meth:`dataikuapi.dss.project.DSSProject.create_code_studio` instead """ def __init__(self, client, project_key, code_studio_id): self.client = client self.project_key = project_key self.code_studio_id = code_studio_id def delete(self): """ Delete the code studio """ self.client._perform_empty("DELETE", "/projects/%s/code-studios/%s" % (self.project_key, self.code_studio_id)) def get_settings(self): """ Get the code studio's definition Usage example .. code-block:: python # list code studios of some user for code_studio in project.list_code_studios(as_type="objects"): settings = code_studio.get_settings() if settings.owner == 'the_user_login': print("User owns code studio %s from template %s" % (settings.name, settings.template_id)) :return: a handle to inspect the code studio definition :rtype: :class:`dataikuapi.dss.codestudio.DSSCodeStudioObjectSettings` """ settings = self.client._perform_json("GET", "/projects/%s/code-studios/%s" % (self.project_key, self.code_studio_id)) return DSSCodeStudioObjectSettings(self.client, self.project_key, self.code_studio_id, settings) def get_status(self): """ Get the code studio's state Usage example .. code-block:: python # print list of currently running code studios for code_studio in project.list_code_studios(as_type="objects"): status = code_studio.get_status() if status.state == 'RUNNING': settings = code_studio.get_settings() print("Code studio %s from template %s is running" % (settings.name, settings.template_id)) :return: a handle to inspect the code studio state :rtype: :class:`dataikuapi.dss.codestudio.DSSCodeStudioObjectStatus` """ status = self.client._perform_json("GET", "/projects/%s/code-studios/%s/status" % (self.project_key, self.code_studio_id)) return DSSCodeStudioObjectStatus(self.client, self.project_key, self.code_studio_id, status) def stop(self): """ Stop a running code studio :return: a future to wait on the stop, or None if already stopped :rtype: :class:`dataikuapi.dss.future.DSSFuture` """ ret = self.client._perform_json("POST", "/projects/%s/code-studios/%s/stop" % (self.project_key, self.code_studio_id)) return DSSFuture.from_resp(self.client, ret) def restart(self): """ Start or restart a code studio :return: a future to wait on the start :rtype: :class:`dataikuapi.dss.future.DSSFuture` """ ret = self.client._perform_json("POST", "/projects/%s/code-studios/%s/restart" % (self.project_key, self.code_studio_id)) return DSSFuture.from_resp(self.client, ret) def check_conflicts(self, zone): """ Checks whether the files in a zone of the code studio have conflicting changes with what the DSS instance has. Usage example .. code-block:: python # stop a code studio if there's no conflict in any zone status = code_studio.get_status() conflict_count = 0 for zone in status.get_zones(as_type="names"): conflicts = code_studio.check_conflicts(zone) conflict_count += len(conflicts.added) conflict_count += len(conflicts.modified) conflict_count += len(conflicts.deleted) if conflict_count == 0: code_studio.stop().wait_for_result() :param string zone: name of the zone to check (see :meth:`dataikuapi.dss.codestudio.DSSCodeStudioObjectStatus.get_zones`) :return: a summary of the conflicts that were found. :rtype: :class:`DSSCodeStudioObjectConflicts` """ conflicts = self.client._perform_json("GET", "/projects/%s/code-studios/%s/conflicts/%s" % (self.project_key, self.code_studio_id, zone)) return DSSCodeStudioObjectConflicts(zone, conflicts) def pull_from_code_studio(self, zone): """ Copies the files from a zone of the code studio to the DSS instance :param str zone: name of the zone to pull (see :meth:`dataikuapi.dss.codestudio.DSSCodeStudioObjectStatus.get_zones`) """ self.client._perform_empty("GET", "/projects/%s/code-studios/%s/pull/%s" % (self.project_key, self.code_studio_id, zone)) def push_to_code_studio(self, zone): """ Copies the files from the DSS instance to a zone of the code studio :param str zone: name of the zone to push (see :meth:`dataikuapi.dss.codestudio.DSSCodeStudioObjectStatus.get_zones`) :return: a dictionary of {count: , size: } :rtype: dict """ return self.client._perform_json("GET", "/projects/%s/code-studios/%s/push/%s" % (self.project_key, self.code_studio_id, zone)) def change_owner(self, new_owner): """ Allows to change the owner of the Code Studio .. note:: You need to be either admin or owner of the Code Studio in order to change the owner. :param str new_owner: the id of the new owner :return: a handle on the updated code studio :rtype: :class:`dataikuapi.dss.codestudio.DSSCodeStudioObject` """ return self.client._perform_json("POST", "/projects/%s/code-studios/%s/change-owner?newOwner=%s" % (self.project_key, self.code_studio_id, new_owner)) class DSSCodeStudioObjectConflicts(dict): """ Summary of the conflicts on zones of a code studio. .. note:: Only conflicting files are listed, that is, files added, modified or deleted in the code studio for which the corresponding file in the DSS instance has been also added, modified or deleted. """ def __init__(self, zone, conflicts): super(DSSCodeStudioObjectConflicts, self).__init__(conflicts) self._zone = zone self._conflicts = conflicts def get_raw(self): """ Get the raw conflicts summary. :return: a summary of the conflicts that were found, as a dict. The top-level field is the zone checked, or 'error' if it wasn't found. The dict should contain summary information about the count of changes, and **commitFrom** and **commitTo** hashes to identify the state of the files on the DSS instance at the time of the last sync to the code studio pod and now. :rtype: dict """ return self._conflicts @property def is_error(self): """ Whether fetching the conflicts failed. Typically this can happen if the name of the zone(s) for which the conflicts where requested is invalid. :rtype: boolean """ return "error" in self._conflicts @property def authors(self): """ Get the authors of changes to the files. :return: a list of logins of users who modified the conflicting files in the DSS instance. :rtype: list[string] """ return self._conflicts.get(self._zone, {}).get("authors", []) @property def added(self): """ Get the list of files added in the code studio. :return: a list of paths to files that were added in the code studio and conflict with changes on the DSS instance. :rtype: list[string] """ return self._conflicts.get(self._zone, {}).get("added", []) @property def modified(self): """ Get the list of files modified in the code studio. :return: a list of paths to files that were modified in the code studio and conflict with changes on the DSS instance. :rtype: list[string] """ return self._conflicts.get(self._zone, {}).get("modified", []) @property def deleted(self): """ Get the list of files deleted in the code studio. :return: a list of paths to files that were deleted in the code studio and conflict with changes on the DSS instance. :rtype: list[string] """ return self._conflicts.get(self._zone, {}).get("deleted", []) class DSSCodeStudioObjectSettings(object): """ Settings for a code studio .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.codestudio.DSSCodeStudioObject.get_settings` instead """ def __init__(self, client, project_key, code_studio_id, settings): self.client = client self.project_key = project_key self.code_studio_id = code_studio_id self.settings = settings def get_raw(self): """ Gets all settings as a raw dictionary. :return: the settings, as a dict. The dict contains a **templateId** field indicating which code studio template was used to create this code studio. :rtype: dict """ return self.settings @property def id(self): """ Get the identifier of the code studio. .. note:: The id is generated by DSS upon creation and random. :rtype: string """ return self.settings["id"] @property def name(self): """ Get the name of the code studio. .. note:: The name is user-provided and not necessarily unique. :rtype: string """ return self.settings["name"] @property def template_id(self): """ Get the identifier of the template that the code studio was created from. :rtype: string """ return self.settings["templateId"] @property def lib_name(self): """ Get the name of the folder resource files of the code studio are stored in. The path to the resources files is then "/lib/code_studio///" :rtype: string """ return self.settings["libName"] @property def owner(self): """ Get the login of the owner of the code studio. Only the owner of a code studio can use it. Administrators of the project can merely stop/start code studios, not use them. :rtype: string """ return self.settings["owner"] class DSSCodeStudioObjectStatus(object): """ Handle to inspect the status of a code studio .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.codestudio.DSSCodeStudioObject.get_status()` instead """ def __init__(self, client, project_key, code_studio_id, status): self.client = client self.project_key = project_key self.code_studio_id = code_studio_id self.status = status def get_raw(self): """ Gets the status as a raw dict. .. note:: Some fields are only defined when the code studio is running. For instance, **exposed** and **syncedZones** are empty when the code studio is not running. :return: the dict contains a **state** field indicating whether the code studio is STOPPED, STARTING, RUNNING or STOPPING. If RUNNING, then the dict holds additional information about the zones that can be synchronized inside the pod, and the ports of the pod that are exposed. :rtype: dict """ return self.status @property def state(self): """ Get the current state of the code studio. Possible values are STOPPED, STARTING, RUNNING, STOPPING :rtype: string """ return self.status["state"] @property def last_state_change(self): """ Get the timestamp of the last change of the state. :return: a datetime, or None if the code studio was never started :rtype: `datetime.datetime` """ ts = self.status.get("lastStateChange", 0) return _timestamp_ms_to_zoned_datetime(ts) def get_zones(self, as_type="names"): """ Get the list of the zones synchronized inside the code studio :param string as_type: if set to "names", then return a list of zone identifiers; if set to "objects", then return a list of zone definitions :return: the list of zones, each one either a string (if `as_type` is "names), or a dict of with a **id** field. :rtype: list """ zones = self.status.get("syncedZones", []) if as_type == 'name' or as_type == 'names': return [z.get('id') for z in zones] if as_type == 'object' or as_type == 'objects': return zones else: raise Exception("Only 'names' or 'objects' is supported for as_type") dataiku-api-client-python-13.2.2/dataikuapi/dss/continuousactivity.py000066400000000000000000000057471471211575100260460ustar00rootroot00000000000000import time import sys from ..utils import DataikuException class DSSContinuousActivity(object): """ A handle to interact with the execution of a continuous recipe on the DSS instance. .. important:: Do not create this class directly, instead use :meth:`dataikuapi.dss.project.DSSProject.get_continuous_activity` """ def __init__(self, client, project_key, recipe_id): self.client = client self.recipe_id = recipe_id self.project_key = project_key def start(self, loop_params={}): """ Start the continuous activity :param dict loop_params: controls how the recipe is restarted after a failure, and the delay before the restarting. Default is to restart indefinitely without delay. Fields are: * **abortAfterCrashes** : when reaching this number of failures, the recipe isn't restarted anymore. Use -1 as 'no limit on number of failures' * **initialRestartDelayMS** : initial delay to wait before restarting after a failure * **restartDelayIncMS** : increase to the delay before restarting upon subsequent failures * **maxRestartDelayMS** : max delay before restarting after failure """ return self.client._perform_json( "POST", "/projects/%s/continuous-activities/%s/start" % (self.project_key, self.recipe_id), body=loop_params) def stop(self): """ Stop the continuous activity. """ self.client._perform_empty( "POST", "/projects/%s/continuous-activities/%s/stop" % (self.project_key, self.recipe_id)) def get_status(self): """ Get the current status of the continuous activity. Usage example: .. code-block:: python # stop a continuous activity via its future from dataikuapi.dss.future import DSSFuture activity = project.get_continuous_activity("my_continuous_recipe") status = activity.get_status() future = DSSFuture(a.client, status["mainLoopState"]['futureId'], status["mainLoopState"]['futureInfo']) future.abort() # this is equivalent to simply stop() activity.stop() :return: the state of the continuous activity. The state as requested by the use is stored in a **desiredState** field (values: 'STARTED' or 'STOPPED'), and the current effective state in a **mainLoopState** sub-dict. :rtype: dict """ return self.client._perform_json( "GET", "/projects/%s/continuous-activities/%s/" % (self.project_key, self.recipe_id)) def get_recipe(self): """ Get a handle on the associated recipe. :rtype: :class:`dataikuapi.dss.recipe.DSSRecipe` """ from .recipe import DSSRecipe return DSSRecipe(self.client, self.project_key, self.recipe_id) dataiku-api-client-python-13.2.2/dataikuapi/dss/dashboard.py000066400000000000000000000124571471211575100240060ustar00rootroot00000000000000import json import sys from .utils import DSSTaggableObjectListItem, DSSTaggableObjectSettings if sys.version_info >= (3,0): import urllib.parse dku_quote_fn = urllib.parse.quote else: import urllib dku_quote_fn = urllib.quote DASHBOARDS_URI_FORMAT = "/projects/%s/dashboards/" DASHBOARD_URI_FORMAT = DASHBOARDS_URI_FORMAT + "%s/" class DSSDashboard(object): """ A handle to interact with a dashboard on the DSS instance. .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.DSSProject.get_dashboard` instead """ def __init__(self, client, project_key, dashboard_id): self.client = client self.project_key = project_key self.dashboard_id = dashboard_id def delete(self): """ Delete the dashboard """ return self.client._perform_empty("DELETE", DASHBOARD_URI_FORMAT % (self.project_key, self.dashboard_id)) def get_settings(self): """ Get the dashboard's definition :return: a handle to inspect the dashboard definition :rtype: :class:`dataikuapi.dss.dashboard.DSSDashboardSettings` """ return DSSDashboardSettings(self.client, self.client._perform_json("GET", DASHBOARD_URI_FORMAT % (self.project_key, self.dashboard_id))) class DSSDashboardListItem(DSSTaggableObjectListItem): """ An item in a list of dashboards. .. important:: Do not instantiate this class, use :meth:`dataikuapi.dss.project.DSSProject.list_dashboards` """ def __init__(self, client, data): super(DSSDashboardListItem, self).__init__(data) self.client = client def to_dashboard(self): """ Get a handle to interact with this dashboard :return: a handle on the dashboard :rtype: :class:`dataikuapi.dss.dashboard.DSSDashboard` """ return DSSDashboard(self.client, self._data["projectKey"], self._data["id"]) @property def id(self): """ Get the identifier of the dashboard .. note:: The id is generated by DSS upon creation and random. :rtype: string """ return self._data["id"] @property def name(self): """ Get the name of the dashboard .. note:: The name is user-provided and not necessarily unique. :rtype: str """ return self._data['name'] @property def listed(self): """ Get the boolean indicating whether the dashboard is private or public (i.e. promoted) :rtype: bool """ return self._data['listed'] @property def owner(self): """ Get the login of the owner of the dashboard :rtype: str """ return self._data['owner'] @property def num_pages(self): """ Get the number of pages (i.e. slides) in the dashboard :rtype: int """ return self._data['numPages'] @property def num_tiles(self): """ Get the number of tiles in the dashboard :rtype: int """ return self._data['numTiles'] class DSSDashboardSettings(DSSTaggableObjectSettings): """ Settings for a dashboard .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.dashboard.DSSDashboard.get_settings` instead """ def __init__(self, client, settings): super(DSSDashboardSettings, self).__init__(settings) self.client = client self.project_key = settings['projectKey'] self.settings = settings def get_raw(self): """ Gets all settings as a raw dictionary. :return: the settings, as a dict. Fields are: * **projectKey** and **id** : identify the dashboard * **name** : name (label) of the dashboard * **owner** : login of the owner of the dashboard * **listed** : boolean indicating whether the dashboard is private or public (i.e. promoted) * **pages** : definition of the different slides * **versionTag**, **creationTag**, **checklists**, **tags**, **customFields** : common fields on DSS objects :rtype: dict """ return self.settings def save(self): """ Save the settings to the dashboard """ return self.client._perform_json("PUT", DASHBOARD_URI_FORMAT % (self.project_key, self.id), body = self.settings) @property def id(self): """ Get the identifier of the dashboard .. note:: The id is generated by DSS upon creation and random. :rtype: string """ return self.settings["id"] @property def name(self): """ Get the name of the dashboard .. note:: The name is user-provided and not necessarily unique. :rtype: str """ return self.settings['name'] @property def listed(self): """ Get the boolean indicating whether the dashboard is private or public (i.e. promoted) :rtype: bool """ return self.settings['listed'] @property def owner(self): """ Get the login of the owner of the dashboard :rtype: str """ return self.settings['owner'] dataiku-api-client-python-13.2.2/dataikuapi/dss/data_collection.py000066400000000000000000000226531471211575100252020ustar00rootroot00000000000000from datetime import datetime from ..utils import _timestamp_ms_to_zoned_datetime from dataikuapi.dss.dataset import DSSDataset class DSSDataCollectionListItem(object): """ An item in a list of Data Collections. Do not instantiate this class, use :meth:`dataikuapi.DSSClient.list_data_collections` """ def __init__(self, client, data): self.client = client self._data = data def get_raw(self): """ Get the raw representation of this :class:`DSSDataCollectionListItem` :rtype: :class:`dict` """ return self._data @property def id(self): return self._data["id"] @property def display_name(self): return self._data["displayName"] @property def description(self): return self._data["description"] @property def color(self): return self._data["color"] @property def tags(self): return self._data["tags"] @property def item_count(self): return self._data["itemCount"] @property def last_modified_on(self): ts = self._data.get("lastModifiedOn", 0) return _timestamp_ms_to_zoned_datetime(ts) def to_data_collection(self): """ Gets the :class:`DSSDataCollection` corresponding to this list item :return: handle of the Data Collection :rtype: :class:`DSSDataCollection` """ return DSSDataCollection(self.client, self._data["id"]) class DSSDataCollection(): """ A handle to interact with a Data Collection on the DSS instance. Do not create this class directly, instead use :meth:`dataikuapi.DSSClient.get_data_collection` or :meth:`.DSSDataCollectionListItem.to_data_collection` """ def __init__(self, client, id): self.client = client self.id = id def get_settings(self): """ Gets the settings of this Data Collection. :returns: a handle to read, modify and save the settings :rtype: :class:`DSSDataCollectionSettings` """ return DSSDataCollectionSettings(self, self.client._perform_json("GET", "/data-collections/%s" % self.id)) def list_objects(self, as_type='objects'): """ List the objects in this Data Collection :param str as_type: How to return the list. Supported values are "objects" and "dict" (defaults to **objects**). :returns: The list of objects :rtype: list of :class:`.DSSDataCollectionItem` if as_type is "objects", list of :class:`dict` if as_type is "dict" """ items = self.client._perform_json("GET", "/data-collections/%s/objects" % self.id) if as_type == "objects" or as_type == "object": return [DSSDataCollectionItem(self, item) for item in items] else: return items def add_object(self, obj): """ Add an object to this Data Collection. :param obj: object to add to the Data Collection. :type obj: :class:`~dataikuapi.dss.dataset.DSSDataset`, :class:`.DSSDataCollectionItem` or :class:`dict` """ if isinstance(obj, DSSDataset): data = ({ "type": "DATASET", "projectKey": obj.project_key, "id": obj.id, }) elif isinstance(obj, DSSDataCollectionItem): data = obj.get_raw() elif isinstance(obj, dict): data = obj else: raise ValueError("Unsupported object type") self.client._perform_json("POST", "/data-collections/%s/objects" % self.id, body=data) def delete(self): """ Delete this Data Collection This call requires Administrator rights on the Data Collection. """ return self.client._perform_empty("DELETE", "/data-collections/%s" % self.id) class DSSDataCollectionItem: """ A handle on an object inside a Data Collection Do not create this class directly, instead use :meth:`.DSSDataCollection.list_objects` """ def __init__(self, data_collection, data): self.data_collection = data_collection self.data = data def get_raw(self): """ Get the raw description of the Data Collection item. This returns a reference to the raw data, not a copy. :return: the Data Collection item raw description :rtype: :class:`dict` """ return self.data def get_as_dataset(self): """ Gets a handle on the corresponding dataset. .. attention:: The usability of this handle might be limited by the current user's authorizations, as seeing a dataset in a data-collection doesn't necessarily imply a lot of rights. :returns: a handle on a dataset :rtype: :class:`dataikuapi.dss.dataset.DSSDataset` """ return DSSDataset(self.data_collection.client, self.data["projectKey"], self.data["id"]) def remove(self): """ Remove this object from the Data Collection This call requires Contributor rights on the Data Collection. """ if self.data["type"] == "DATASET": self.data_collection.client._perform_empty( "DELETE", "/data-collections/%s/objects/dataset/%s/%s" % (self.data_collection.id, self.data['projectKey'], self.data['id'])) class DSSDataCollectionSettings: """ A handle on the settings of a Data Collection Do not create this class directly, instead use :meth:`.DSSDataCollection.get_settings` """ def __init__(self, data_collection, settings): self.data_collection = data_collection self.settings = settings def get_raw(self): """ Get the raw settings of the Data Collection. This returns a reference to the raw settings, not a copy, so changes made to the returned object will be reflected when saving. :return: the Data Collection raw settings :rtype: :class:`dict` """ return self.settings @property def id(self): """ The Data Collection id (read-only) :rtype: :class:`str` """ return self.settings['id'] @property def display_name(self): """ Get or set the name of the Data Collection :rtype: :class:`str` """ return self.settings['displayName'] @display_name.setter def display_name(self, value): self.settings['displayName'] = value @property def color(self): """ Get or set the background color of the Data Collection (using #RRGGBB syntax) :rtype: :class:`str` """ return self.settings['color'] @color.setter def color(self, value): self.settings['color'] = value @property def description(self): """ Get or set the description of the Data Collection :rtype: :class:`str` """ return self.settings['description'] @description.setter def description(self, value): self.settings['description'] = value @property def tags(self): """ Get or set the tags of the Data Collection :rtype: list of :class:`str` """ return self.settings['tags'] @tags.setter def tags(self, value): self.settings['tags'] = value @property def permissions(self): """ Get or set the permissions controlling who is a reader, contributor or admin of the Data Collection. If the user is not an admin of the data-collection, the permissions property will be redacted as None. :return: a list of the Data Collection permissions :rtype: list of :class:`dict` or :class:`None` """ return self.settings['permissions'] if 'permissions' in self.settings else None @permissions.setter def permissions(self, value): self.settings['permissions'] = value def save(self): """ Save the changes made on the settings This call requires Administrator rights on the Data Collection. """ self.data_collection.client._perform_empty( "PUT", "/data-collections/%s" % self.data_collection.id, body=self.settings) class DSSDataCollectionPermissionItem: @classmethod def admin_group(cls, group): """Creates a :class:`dict` representing an admin authorization for a group""" return {"group": group, "admin": True, "write": True, "read": True} @classmethod def contributor_group(cls, group): """Creates a :class:`dict` representing an contributor authorization for a group""" return {"group": group, "admin": False, "write": True, "read": True} @classmethod def reader_group(cls, group): """Creates a :class:`dict` representing an reader authorization for a group""" return {"group": group, "admin": False, "write": False, "read": True} @classmethod def admin_user(cls, user): """Creates a :class:`dict` representing an admin authorization for a user""" return {"user": user, "admin": True, "write": True, "read": True} @classmethod def contributor_user(cls, user): """Creates a :class:`dict` representing an contributor authorization for a user""" return {"user": user, "admin": False, "write": True, "read": True} @classmethod def reader_user(cls, user): """Creates a :class:`dict` representing an reader authorization for a user""" return {"user": user, "admin": False, "write": False, "read": True} dataiku-api-client-python-13.2.2/dataikuapi/dss/data_quality.py000066400000000000000000000275671471211575100245500ustar00rootroot00000000000000from dataikuapi.dss.future import DSSFuture class DSSDataQualityRuleSet(object): """ Base settings class for dataset data quality rules. .. caution:: Do not instantiate this class directly, use :meth:`dataikuapi.dss.dataset.DSSDataset.get_data_quality_rules` """ def __init__(self, project_key, dataset_name, client): self.ruleset = { "checks": [] } self.project_key = project_key self.dataset_name = dataset_name self.client = client def list_rules(self, as_type="objects"): """ Get the list of rules defined on the dataset. :param str as_type: How to return the rules. Possible values are "dict" and "objects" (defaults to **objects**) :returns: The rules defined on the dataset. :rtype: a list of :class:`DSSDataQualityRule` if as_type is "objects", a list of dict if as_type is "dict" """ self.ruleset = self.client._perform_json("GET", "/projects/%s/datasets/%s/data-quality/rules" % (self.project_key, self.dataset_name)) if as_type == "dict": return self.ruleset["checks"] elif as_type == "objects" or as_type == "object": return [DSSDataQualityRule(rule, self.dataset_name, self.project_key, self.client) for rule in self.ruleset["checks"]] else: raise ValueError("Unknown as_type") def create_rule(self, config=None): """ Create a data quality rule on the current dataset. :param object config: The config of the rule :returns: The created data quality rule :rtype: :class:`DSSDataQualityRule` """ if not config: raise ValueError("Config is required") rule = self.client._perform_json("POST", "/projects/%s/datasets/%s/data-quality/rules" % (self.project_key, self.dataset_name), body=config) self.ruleset["checks"].append(rule) return DSSDataQualityRule(rule, self.dataset_name, self.project_key, self.client) def get_partitions_status(self, partitions="NP"): """ Get the last computed status of the specified partition(s). :param partitions: The list of partitions name or the name of the partition to get the last status (or "ALL" to retrieve the whole dataset partition). If the dataset is not partitioned use "NP" or None. :return: the status of the specified partitions if they exists :rtype: object """ if not partitions: partitions = ["NP"] if isinstance(partitions, str): partitions = [partitions] return self.client._perform_json("POST", "/projects/%s/datasets/%s/data-quality/get-partitions-status" % (self.project_key, self.dataset_name), params={"partitions": partitions}) def compute_rules(self, partition="NP"): """ Compute all data quality enabled rules of the current dataset. :param str partition: If the dataset is partitioned, the name of the partition to compute (or "ALL" to compute on the whole dataset). If the dataset is not partitioned use "NP" or None. :returns: Job of the currently computed data quality rules. :rtype: :class:`dataikuapi.dss.future.DSSFuture` """ if not partition: partition = "NP" future_resp = self.client._perform_json("POST", "/projects/%s/datasets/%s/data-quality/actions/compute-rules" % (self.project_key, self.dataset_name), params={"partition": partition}) return DSSFuture(self.client, future_resp.get("jobId", None), future_resp) def get_status(self): """ Get the status of the dataset. For partitioned dataset this is the worst result of the last computed partitions. :returns: The status of the dataset. :rtype: str """ return self.client._perform_json("GET", "/projects/%s/datasets/%s/data-quality/status" % (self.project_key, self.dataset_name)) def get_status_by_partition(self, include_all_partitions=False): """ Return the status of a dataset detailed per partition used to compute it if any. If the dataset is not partitioned it will contain only one result. :param boolean include_all_partitions: Include all the partition having a data quality status or only the one relevant to the current status of the dataset. Default is False. :returns: The current status of each last built partitions of the dataset :rtype: dict """ return self.client._perform_json("GET", "/projects/%s/datasets/%s/data-quality/status-by-partition" % (self.project_key, self.dataset_name), params={ "includeAllPartitions": include_all_partitions}) def get_last_rules_results(self, partition="NP"): """ Return the last result of all the rules defined on the dataset on a specified partition. If the dataset is not partitioned it will get all the last rules results :param str partition: If the dataset is partitioned, the name of the partition to get the detailed rules results (or "ALL" to compute on the whole dataset). If the dataset is not partitioned use "NP" or None. :returns: The last result of each rule on the specified partition :rtype: a list of :class:`DSSDataQualityRuleResult` """ if not partition: partition = "NP" rule_results = self.client._perform_json("GET", "/projects/%s/datasets/%s/data-quality/last-rules-result" % (self.project_key, self.dataset_name), params={ "partition": partition}) return [DSSDataQualityRuleResult(result) for result in rule_results] def get_rules_history(self, min_timestamp=None, max_timestamp=None, results_per_page=10000, page=0, rule_ids=None): """ Get the history of computed rules. :param int min_timestamp: Timestamp representing the beginning of the timeframe. (included) :param int max_timestamp: Timestamp representing the end of the timeframe. (included) :param int results_per_page: The maximum number of records to be returned, default will be the last 10 000 records. :param int page: The page to be returned, default will be first page (page=0). :param list rule_ids: A list of rule ids to get the history from. Default is all the rules on the dataset. :returns: The detailed execution of data quality rules matching the filters set :rtype: a list of :class:`DSSDataQualityRuleResult` """ rule_results = self.client._perform_json("GET", "/projects/%s/datasets/%s/data-quality/rules-history" % (self.project_key, self.dataset_name), params={ "minTimestamp": min_timestamp, "maxTimestamp": max_timestamp, "resultsPerPage": results_per_page, "page": page, "ruleIds": rule_ids }) return [DSSDataQualityRuleResult(result) for result in rule_results] class DSSDataQualityRule(object): """ A rule defined on a dataset. .. caution:: Do not instantiate this class, use :meth:`DSSDataQualityRuleSet.list_rules` """ def __init__(self, rule, dataset_name, project_key, client): self.rule = rule self.dataset_name = dataset_name self.project_key = project_key self.client = client def get_raw(self): """ Get the raw representation of this :class:`DSSDataQualityRule` :rtype: :class:`dict` """ return self.rule @property def id(self): return self.rule["id"] @property def name(self): return self.rule["displayName"] def compute(self, partition="NP"): """ Compute the rule on a given partition or the full dataset. :param str partition: If the dataset is partitioned, the name of the partition to compute (or "ALL" to compute on the whole dataset). If the dataset is not partitioned use "NP" or None. :returns: A job of the computation of the rule. :rtype: :class:`dataikuapi.dss.future.DSSFuture` """ if not partition: partition = "NP" future_resp = self.client._perform_json("POST", "/projects/%s/datasets/%s/data-quality/actions/compute-rules" % (self.project_key, self.dataset_name), params={"partition": partition, "ruleId": self.rule["id"]}) return DSSFuture(self.client, future_resp.get("jobId", None), future_resp) def save(self): """ Save the settings of a rule. :returns: 'Success' :rtype: str """ self.client._perform_empty("PUT", "/projects/%s/datasets/%s/data-quality/rules/%s" % (self.project_key, self.dataset_name, self.id), body=self.rule) return "Success" def delete(self): """ Delete the rule from the dataset configuration. """ self.client._perform_empty("DELETE", "/projects/%s/datasets/%s/data-quality/rules/%s" % (self.project_key, self.dataset_name, self.id), params={ "ruleId": self.rule["id"] }) def get_last_result(self, partition="NP"): """ Return the last result of the rule on a specified dataset/partition. :param str partition: If the dataset is partitioned, the name of the partition to get the detailed rules results (or "ALL" to refer to the whole dataset). If the dataset is not partitioned use "NP" or None. :returns: The last result of the rule on the specified partition :rtype: :class:`DSSDataQualityRuleResult` """ if not partition: partition = "NP" result = self.client._perform_json("GET", "/projects/%s/datasets/%s/data-quality/last-rules-result" % (self.project_key, self.dataset_name), params={ "partition": partition, "ruleId": self.rule["id"]}) if len(result) == 0: # The rule has not been executed yet. return None return DSSDataQualityRuleResult(result[0]) def get_rule_history(self, min_timestamp=None, max_timestamp=None, results_per_page=10000, page=0): """ Get the history of the current rule. :param int min_timestamp: Timestamp representing the beginning of the timeframe. (included) :param int max_timestamp: Timestamp representing the end of the timeframe. (included) :param int results_per_page: The maximum number of records to be returned, default will be the last 10 000 records. :param int page: The page to be returned, default will be first page. :returns: The detailed execution of data quality rule matching the timeframe set :rtype: a list of :class:`DSSDataQualityRuleResult` """ rule_results = self.client._perform_json("GET", "/projects/%s/datasets/%s/data-quality/rules-history" % (self.project_key, self.dataset_name), params={ "minTimestamp": min_timestamp, "maxTimestamp": max_timestamp, "resultsPerPage": results_per_page, "page": page, "ruleId": self.rule["id"] }) return [DSSDataQualityRuleResult(result) for result in rule_results] class DSSDataQualityRuleResult(object): """ The result of a rule defined on a dataset .. caution:: Do not instantiate this class, use: :meth:`DSSDataQualityRuleSet.get_last_rules_results` or :meth:`DSSDataQualityRuleSet.get_rules_history` or :meth:`DSSDataQualityRule.get_last_result` or :meth:`DSSDataQualityRule.get_rule_history` """ def __init__(self, data): self.data = data def get_raw(self): """ Get the raw representation of this :class:`DSSDataQualityRuleResult` :rtype: :class:`dict` """ return self.data @property def id(self): return self.data["id"] @property def name(self): return self.data["name"] @property def outcome(self): return self.data["outcome"] @property def message(self): return self.data["message"] @property def compute_date(self): return self.data["computeDate"] @property def run_origin(self): return self.data["runOrigin"] @property def partition(self): return self.data["partition"] dataiku-api-client-python-13.2.2/dataikuapi/dss/dataset.py000066400000000000000000001524641471211575100235070ustar00rootroot00000000000000import datetime from ..utils import DataikuException, DataikuValueCaster from ..utils import DataikuStreamedHttpUTF8CSVReader from ..utils import _timestamp_ms_to_zoned_datetime import json, warnings from .utils import DSSTaggableObjectListItem, DSSTaggableObjectSettings from .future import DSSFuture from .metrics import ComputedMetrics from .discussion import DSSObjectDiscussions from .statistics import DSSStatisticsWorksheet from .data_quality import DSSDataQualityRuleSet from . import recipe import uuid try: basestring except NameError: basestring = str class DSSDatasetListItem(DSSTaggableObjectListItem): """ An item in a list of datasets. .. caution:: Do not instantiate this class, use :meth:`dataikuapi.dss.project.DSSProject.list_datasets` """ def __init__(self, client, data): super(DSSDatasetListItem, self).__init__(data) self.client = client def to_dataset(self): """ Gets a handle on the corresponding dataset. :returns: a handle on a dataset :rtype: :class:`DSSDataset` """ return DSSDataset(self.client, self._data["projectKey"], self._data["name"]) @property def name(self): """ Get the name of the dataset. :rtype: string """ return self._data["name"] @property def id(self): """ Get the identifier of the dataset. :rtype: string """ return self._data["name"] @property def type(self): """ Get the type of the dataset. :rtype: string """ return self._data["type"] @property def schema(self): """ Get the dataset schema as a dict. :returns: a dict object of the schema, with the list of columns. See :meth:`DSSDataset.get_schema()` :rtype: dict """ return self._data["schema"] @property def connection(self): """ Get the name of the connection on which this dataset is attached, or None if there is no connection for this dataset. :rtype: string """ if not "params" in self._data: return None return self._data["params"].get("connection", None) def get_column(self, column): """ Get a given column in the dataset schema by its name. :param str column: name of the column to find :returns: the column settings or None if column does not exist :rtype: dict """ matched = [col for col in self.schema["columns"] if col["name"] == column] return None if len(matched) == 0 else matched[0] class DSSDataset(object): """ A dataset on the DSS instance. Do not instantiate this class, use :meth:`dataikuapi.dss.project.DSSProject.get_dataset` """ def __init__(self, client, project_key, dataset_name): self.client = client self.project = client.get_project(project_key) self.project_key = project_key self.dataset_name = dataset_name @property def id(self): """ Get the dataset identifier. :rtype: string """ return self.dataset_name @property def name(self): """ Get the dataset name. :rtype: string """ return self.dataset_name ######################################################## # Dataset deletion ######################################################## def delete(self, drop_data=False): """ Delete the dataset. :param bool drop_data: Should the data of the dataset be dropped, defaults to False """ return self.client._perform_empty( "DELETE", "/projects/%s/datasets/%s" % (self.project_key, self.dataset_name), params = { "dropData" : drop_data }) ######################################################## # Dataset renaming ######################################################## def rename(self, new_name): """ Rename the dataset with the new specified name :param str new_name: the new name of the dataset """ if self.dataset_name == new_name: raise ValueError("Dataset name is already " + new_name) obj = { "oldName": self.dataset_name, "newName": new_name } self.client._perform_empty("POST", "/projects/%s/actions/renameDataset" % self.project_key, body=obj) self.dataset_name = new_name ######################################################## # Dataset definition ######################################################## def get_settings(self): """ Get the settings of this dataset as a :class:`DSSDatasetSettings`, or one of its subclasses. Know subclasses of :class:`DSSDatasetSettings` include :class:`FSLikeDatasetSettings` and :class:`SQLDatasetSettings` You must use :meth:`~DSSDatasetSettings.save()` on the returned object to make your changes effective on the dataset. .. code-block:: python # Example: activating discrete partitioning on a SQL dataset dataset = project.get_dataset("my_database_table") settings = dataset.get_settings() settings.add_discrete_partitioning_dimension("country") settings.save() :rtype: :class:`DSSDatasetSettings` """ data = self.client._perform_json("GET", "/projects/%s/datasets/%s" % (self.project_key, self.dataset_name)) if data["type"] in self.__class__._FS_TYPES: return FSLikeDatasetSettings(self, data) elif data["type"] in self.__class__._SQL_TYPES: return SQLDatasetSettings(self, data) else: return DSSDatasetSettings(self, data) def get_definition(self): """ Get the raw settings of the dataset as a dict. .. caution:: Deprecated. Use :meth:`get_settings` :rtype: dict """ warnings.warn("Dataset.get_definition is deprecated, please use get_settings", DeprecationWarning) return self.client._perform_json( "GET", "/projects/%s/datasets/%s" % (self.project_key, self.dataset_name)) def set_definition(self, definition): """ Set the definition of the dataset .. caution:: Deprecated. Use :meth:`get_settings` and :meth:`DSSDatasetSettings.save` :param dict definition: the definition, as a dict. You should only set a definition object that has been retrieved using the :meth:`get_definition` call. """ warnings.warn("Dataset.set_definition is deprecated, please use get_settings", DeprecationWarning) return self.client._perform_json( "PUT", "/projects/%s/datasets/%s" % (self.project_key, self.dataset_name), body=definition) def exists(self): """ Test if the dataset exists. :returns: whether this dataset exists :rtype: bool """ try: self.get_metadata() return True except Exception as e: return False ######################################################## # Dataset metadata ######################################################## def get_schema(self): """ Get the dataset schema. :returns: a dict object of the schema, with the list of columns. :rtype: dict """ return self.client._perform_json( "GET", "/projects/%s/datasets/%s/schema" % (self.project_key, self.dataset_name)) def set_schema(self, schema): """ Set the dataset schema. :param dict schema: the desired schema for the dataset, as a dict. All columns have to provide their name and type. """ return self.client._perform_json( "PUT", "/projects/%s/datasets/%s/schema" % (self.project_key, self.dataset_name), body=schema) def get_metadata(self): """ Get the metadata attached to this dataset. The metadata contains label, description checklists, tags and custom metadata of the dataset :returns: a dict object. For more information on available metadata, please see https://doc.dataiku.com/dss/api/latest/rest/ :rtype: dict """ return self.client._perform_json( "GET", "/projects/%s/datasets/%s/metadata" % (self.project_key, self.dataset_name)) def set_metadata(self, metadata): """ Set the metadata on this dataset. :param dict metadata: the new state of the metadata for the dataset. You should only set a metadata object that has been retrieved using the :meth:`get_metadata` call. """ return self.client._perform_json( "PUT", "/projects/%s/datasets/%s/metadata" % (self.project_key, self.dataset_name), body=metadata) ######################################################## # Dataset data ######################################################## def iter_rows(self, partitions=None): """ Get the dataset data as a row-by-row iterator. :param partitions: (optional) partition identifier, or list of partitions to include, if applicable. :type partitions: Union[string, list[string]] :returns: an iterator over the rows, each row being a list of values. The order of values in the list is the same as the order of columns in the schema returned by :meth:`get_schema` :rtype: generator[list] """ read_session_id = str(uuid.uuid4()) csv_stream = self.client._perform_raw( "GET" , "/projects/%s/datasets/%s/data/" %(self.project_key, self.dataset_name), params = { "format" : "tsv-excel-noheader", "partitions" : partitions, "readSessionId": read_session_id }) return DataikuStreamedHttpUTF8CSVReader(self.get_schema()["columns"], csv_stream, read_session_id=read_session_id, client=self.client, project_key=self.project_key, dataset_name=self.dataset_name).iter_rows() def list_partitions(self): """ Get the list of all partitions of this dataset. :returns: the list of partitions, as a list of strings. :rtype: list[string] """ return self.client._perform_json( "GET", "/projects/%s/datasets/%s/partitions" % (self.project_key, self.dataset_name)) def clear(self, partitions=None): """ Clear data in this dataset. :param partitions: (optional) partition identifier, or list of partitions to clear. When not provided, the entire dataset is cleared. :type partitions: Union[string, list[string]] :returns: a dict containing the method call status. :rtype: dict """ return self.client._perform_json( "DELETE", "/projects/%s/datasets/%s/data" % (self.project_key, self.dataset_name), params={"partitions" : partitions}) def copy_to(self, target, sync_schema=True, write_mode="OVERWRITE"): """ Copy the data of this dataset to another dataset. :param target: an object representing the target of this copy. :type target: :class:`dataikuapi.dss.dataset.DSSDataset` :param bool sync_schema: (optional) update the target dataset schema to make it match the sourece dataset schema. :param string write_mode: (optional) OVERWRITE (default) or APPEND. If OVERWRITE, the output dataset is cleared prior to writing the data. :returns: a DSSFuture representing the operation. :rtype: :class:`dataikuapi.dss.future.DSSFuture` """ dqr = { "targetProjectKey" : target.project_key, "targetDatasetName": target.dataset_name, "syncSchema": sync_schema, "writeMode" : write_mode } future_resp = self.client._perform_json("POST", "/projects/%s/datasets/%s/actions/copyTo" % (self.project_key, self.dataset_name), body=dqr) return DSSFuture(self.client, future_resp.get("jobId", None), future_resp) def search_data_elastic(self, query_string, start=0, size=128, sort_columns=None, partitions=None): """ .. caution:: Only for datasets on Elasticsearch connections Query the service with a search string to directly fetch data :param str query_string: Elasticsearch compatible query string :param int start: row to start fetching the data :param int size: number of results to return :param list sort_columns: list of {"column", "order"} dict, which is the order to fetch data. "order" is "asc" for ascending, "desc" for descending :param list partitions: if the dataset is partitioned, a list of partition ids to search :return: a dict containing "columns", "rows", "warnings", "found" (when start == 0) :rtype: dict """ params = { "queryString": query_string, "start": start, "size": size, "sortColumns": json.dumps(sort_columns), "partitions": json.dumps(partitions), } future_resp = self.client._perform_json("GET", "/projects/%s/datasets/%s/search-data-elastic" % (self.project_key, self.dataset_name), params=params) result = DSSFuture(self.client, future_resp.get("jobId", None), future_resp).wait_for_result() value_caster = DataikuValueCaster(result["columns"]) result["rows"] = [value_caster.cast_values(row) for row in result["rows"]] return result ######################################################## # Dataset actions ######################################################## def build(self, job_type="NON_RECURSIVE_FORCED_BUILD", partitions=None, wait=True, no_fail=False): """ Start a new job to build this dataset and wait for it to complete. Raises if the job failed. .. code-block:: python job = dataset.build() print("Job %s done" % job.id) :param job_type: the job type. One of RECURSIVE_BUILD, NON_RECURSIVE_FORCED_BUILD or RECURSIVE_FORCED_BUILD :param partitions: if the dataset is partitioned, a list of partition ids to build :param bool wait: whether to wait for the job completion before returning the job handle, defaults to True :param no_fail: if True, does not raise if the job failed. :returns: the :class:`dataikuapi.dss.job.DSSJob` job handle corresponding to the built job :rtype: :class:`dataikuapi.dss.job.DSSJob` """ jd = self.project.new_job(job_type) jd.with_output(self.dataset_name, partition=partitions) if wait: return jd.start_and_wait() else: return jd.start() def synchronize_hive_metastore(self): """ Synchronize this dataset with the Hive metastore """ self.client._perform_empty( "POST" , "/projects/%s/datasets/%s/actions/synchronizeHiveMetastore" %(self.project_key, self.dataset_name)) def update_from_hive(self): """ Resynchronize this dataset from its Hive definition """ self.client._perform_empty( "POST", "/projects/%s/datasets/%s/actions/updateFromHive" %(self.project_key, self.dataset_name)) def compute_metrics(self, partition='', metric_ids=None, probes=None): """ Compute metrics on a partition of this dataset. If neither metric ids nor custom probes set are specified, the metrics setup on the dataset are used. :param partition: (optional) partition identifier, use ALL to compute metrics on all data. :type partition: string :param list[string] metric_ids: (optional) ids of the metrics to build :returns: a metric computation report, as a dict :rtype: dict """ url = "/projects/%s/datasets/%s/actions" % (self.project_key, self.dataset_name) if metric_ids is not None: return self.client._perform_json( "POST" , "%s/computeMetricsFromIds" % url, params={'partition':partition}, body={"metricIds" : metric_ids}) elif probes is not None: return self.client._perform_json( "POST" , "%s/computeMetrics" % url, params={'partition':partition}, body=probes) else: return self.client._perform_json( "POST" , "%s/computeMetrics" % url, params={'partition':partition}) def run_checks(self, partition='', checks=None): """ Run checks on a partition of this dataset. If the checks are not specified, the checks setup on the dataset are used. .. caution:: Deprecated. Use :meth:`dataikuapi.dss.data_quality.DSSDataQualityRuleSet.compute_rules` instead :param str partition: (optional) partition identifier, use ALL to run checks on all data. :param list[string] checks: (optional) ids of the checks to run. :returns: a checks computation report, as a dict. :rtype: dict """ if checks is None: return self.client._perform_json( "POST" , "/projects/%s/datasets/%s/actions/runChecks" %(self.project_key, self.dataset_name), params={'partition':partition}) else: return self.client._perform_json( "POST" , "/projects/%s/datasets/%s/actions/runChecks" %(self.project_key, self.dataset_name), params={'partition':partition}, body=checks) def uploaded_add_file(self, fp, filename): """ Add a file to an "uploaded files" dataset :param file fp: A file-like object that represents the file to upload :param str filename: The filename for the file to upload """ self.client._perform_empty("POST", "/projects/%s/datasets/%s/uploaded/files" % (self.project_key, self.dataset_name), files={"file":(filename, fp)}) def uploaded_list_files(self): """ List the files in an "uploaded files" dataset. :returns: uploaded files metadata as a list of dicts, with one dict per file. :rtype: list[dict] """ return self.client._perform_json("GET", "/projects/%s/datasets/%s/uploaded/files" % (self.project_key, self.dataset_name)) ######################################################## # Lab and ML # Don't forget to synchronize with DSSProject.* ######################################################## def create_prediction_ml_task(self, target_variable, ml_backend_type="PY_MEMORY", guess_policy="DEFAULT", prediction_type=None, wait_guess_complete=True): """Creates a new prediction task in a new visual analysis lab for a dataset. :param str target_variable: the variable to predict :param str ml_backend_type: ML backend to use, one of PY_MEMORY, MLLIB or H2O (defaults to **PY_MEMORY**) :param str guess_policy: Policy to use for setting the default parameters. Valid values are: DEFAULT, SIMPLE_FORMULA, DECISION_TREE, EXPLANATORY and PERFORMANCE (defaults to **DEFAULT**) :param str prediction_type: The type of prediction problem this is. If not provided the prediction type will be guessed. Valid values are: BINARY_CLASSIFICATION, REGRESSION, MULTICLASS (defaults to **None**) :param boolean wait_guess_complete: if False, the returned ML task will be in 'guessing' state, i.e. analyzing the input dataset to determine feature handling and algorithms (defaults to **True**). You should wait for the guessing to be completed by calling **wait_guess_complete** on the returned object before doing anything else (in particular calling **train** or **get_settings**) :returns: A ML task handle of type 'PREDICTION' :rtype: :class:`dataikuapi.dss.ml.DSSMLTask` """ return self.project.create_prediction_ml_task(self.dataset_name, target_variable = target_variable, ml_backend_type = ml_backend_type, guess_policy = guess_policy, prediction_type = prediction_type, wait_guess_complete = wait_guess_complete) def create_clustering_ml_task(self, input_dataset, ml_backend_type="PY_MEMORY", guess_policy="KMEANS", wait_guess_complete=True): """Creates a new clustering task in a new visual analysis lab for a dataset. The returned ML task will be in 'guessing' state, i.e. analyzing the input dataset to determine feature handling and algorithms. You should wait for the guessing to be completed by calling **wait_guess_complete** on the returned object before doing anything else (in particular calling **train** or **get_settings**) :param string input_dataset: The dataset to use for training/testing the model :param str ml_backend_type: ML backend to use, one of PY_MEMORY, MLLIB or H2O (defaults to **PY_MEMORY**) :param str guess_policy: Policy to use for setting the default parameters. Valid values are: KMEANS and ANOMALY_DETECTION (defaults to **KMEANS**) :param boolean wait_guess_complete: if False, the returned ML task will be in 'guessing' state, i.e. analyzing the input dataset to determine feature handling and algorithms (defaults to **True**). You should wait for the guessing to be completed by calling **wait_guess_complete** on the returned object before doing anything else (in particular calling **train** or **get_settings**) :returns: A ML task handle of type 'CLUSTERING' :rtype: :class:`dataikuapi.dss.ml.DSSMLTask` """ return self.project.create_clustering_ml_task(self.dataset_name, ml_backend_type=ml_backend_type, guess_policy=guess_policy, wait_guess_complete=wait_guess_complete) def create_timeseries_forecasting_ml_task(self, target_variable, time_variable, timeseries_identifiers=None, guess_policy="TIMESERIES_DEFAULT", wait_guess_complete=True): """Creates a new time series forecasting task in a new visual analysis lab for a dataset. :param string target_variable: The variable to forecast :param string time_variable: Column to be used as time variable. Should be a Date (parsed) column. :param list timeseries_identifiers: List of columns to be used as time series identifiers (when the dataset has multiple series) :param string guess_policy: Policy to use for setting the default parameters. Valid values are: TIMESERIES_DEFAULT, TIMESERIES_STATISTICAL, and TIMESERIES_DEEP_LEARNING :param boolean wait_guess_complete: If False, the returned ML task will be in 'guessing' state, i.e. analyzing the input dataset to determine feature handling and algorithms. You should wait for the guessing to be completed by calling ``wait_guess_complete`` on the returned object before doing anything else (in particular calling ``train`` or ``get_settings``) :returns: A ML task handle of type 'PREDICTION' :rtype: :class:`dataikuapi.dss.ml.DSSMLTask` """ return self.project.create_timeseries_forecasting_ml_task(self.dataset_name, target_variable=target_variable, time_variable=time_variable, timeseries_identifiers=timeseries_identifiers, guess_policy=guess_policy, wait_guess_complete=wait_guess_complete) def create_causal_prediction_ml_task(self, outcome_variable, treatment_variable, prediction_type=None, wait_guess_complete=True): """Creates a new causal prediction task in a new visual analysis lab for a dataset. :param string outcome_variable: The outcome variable to predict. :param string treatment_variable: The treatment variable. :param string or None prediction_type: Valid values are: "CAUSAL_BINARY_CLASSIFICATION", "CAUSAL_REGRESSION" or None (in this case prediction_type will be set by the Guesser) :param boolean wait_guess_complete: If False, the returned ML task will be in 'guessing' state, i.e. analyzing the input dataset to determine feature handling and algorithms. You should wait for the guessing to be completed by calling ``wait_guess_complete`` on the returned object before doing anything else (in particular calling ``train`` or ``get_settings``) :returns: A ML task handle of type 'PREDICTION' :rtype: :class:`dataikuapi.dss.ml.DSSMLTask` """ return self.project.create_causal_prediction_ml_task(self.dataset_name, outcome_variable=outcome_variable, treatment_variable=treatment_variable, prediction_type=prediction_type, wait_guess_complete=wait_guess_complete) def create_analysis(self): """ Create a new visual analysis lab for the dataset. :returns: A visual analysis handle :rtype: :class:`dataikuapi.dss.analysis.DSSAnalysis` """ return self.project.create_analysis(self.dataset_name) def list_analyses(self, as_type="listitems"): """ List the visual analyses on this dataset :param str as_type: How to return the list. Supported values are "listitems" and "objects", defaults to "listitems" :returns: The list of the analyses. If "as_type" is "listitems", each one as a dict, If "as_type" is "objects", each one as a :class:`dataikuapi.dss.analysis.DSSAnalysis` :rtype: list """ analysis_list = [al for al in self.project.list_analyses() if self.dataset_name == al.get('inputDataset')] if as_type == "listitems" or as_type == "listitem": return analysis_list elif as_type == "objects" or as_type == "object": return [self.project.get_analysis(item["analysisId"])for item in analysis_list] else: raise ValueError("Unknown as_type") def delete_analyses(self, drop_data=False): """ Delete all analyses that have this dataset as input dataset. Also deletes ML tasks that are part of the analysis :param bool drop_data: whether to drop data for all ML tasks in the analysis, defaults to False """ [analysis.delete(drop_data=drop_data) for analysis in self.list_analyses(as_type="objects")] ######################################################## # Statistics worksheets ######################################################## def list_statistics_worksheets(self, as_objects=True): """ List the statistics worksheets associated to this dataset. :param bool as_objects: if true, returns the statistics worksheets as :class:`dataikuapi.dss.statistics.DSSStatisticsWorksheet`, else as a list of dicts :rtype: list of :class:`dataikuapi.dss.statistics.DSSStatisticsWorksheet` """ worksheets = self.client._perform_json( "GET", "/projects/%s/datasets/%s/statistics/worksheets/" % (self.project_key, self.dataset_name)) if as_objects: return [self.get_statistics_worksheet(worksheet['id']) for worksheet in worksheets] else: return worksheets def create_statistics_worksheet(self, name="My worksheet"): """ Create a new worksheet in the dataset, and return a handle to interact with it. :param string name: name of the worksheet :returns: a statistic worksheet handle :rtype: :class:`dataikuapi.dss.statistics.DSSStatisticsWorksheet` """ worksheet_definition = { "projectKey": self.project_key, "name": name, "dataSpec": { "inputDatasetSmartName": self.dataset_name, "datasetSelection": { "partitionSelectionMethod": "ALL", "maxRecords": 30000, "samplingMethod": "FULL" } } } created_worksheet = self.client._perform_json( "POST", "/projects/%s/datasets/%s/statistics/worksheets/" % (self.project_key, self.dataset_name), body=worksheet_definition ) return self.get_statistics_worksheet(created_worksheet['id']) def get_statistics_worksheet(self, worksheet_id): """ Get a handle to interact with a statistics worksheet :param string worksheet_id: the ID of the desired worksheet :returns: a statistic worksheet handle :rtype: :class:`dataikuapi.dss.statistics.DSSStatisticsWorksheet` """ return DSSStatisticsWorksheet(self.client, self.project_key, self.dataset_name, worksheet_id) ######################################################## # Metrics ######################################################## def get_last_metric_values(self, partition=''): """ Get the last values of the metrics on this dataset :param partition: (optional) partition identifier, use ALL to retrieve metric values on all data. :type partition: string :returns: a list of metric objects and their value :rtype: :class:`dataikuapi.dss.metrics.ComputedMetrics` """ return ComputedMetrics(self.client._perform_json( "GET", "/projects/%s/datasets/%s/metrics/last/%s" % (self.project_key, self.dataset_name, 'NP' if len(partition) == 0 else partition))) def get_metric_history(self, metric, partition=''): """ Get the history of the values of the metric on this dataset :param string metric: id of the metric to get :param partition: (optional) partition identifier, use ALL to retrieve metric history on all data. :type partition: string :returns: a dict containing the values of the metric, cast to the appropriate type (double, boolean,...) :rtype: dict """ return self.client._perform_json( "GET", "/projects/%s/datasets/%s/metrics/history/%s" % (self.project_key, self.dataset_name, 'NP' if len(partition) == 0 else partition), params={'metricLookup' : metric if isinstance(metric, str) or isinstance(metric, unicode) else json.dumps(metric)}) def get_info(self): """ Retrieve all the information about a dataset :returns: a :class:`DSSDatasetInfo` containing all the information about a dataset. :rtype: :class:`DSSDatasetInfo` """ data = self.client._perform_json("GET", "/projects/%s/datasets/%s/info" % (self.project_key, self.dataset_name)) return DSSDatasetInfo(self, data) ######################################################## # Misc ######################################################## def get_zone(self): """ Get the flow zone of this dataset :rtype: :class:`dataikuapi.dss.flow.DSSFlowZone` """ return self.project.get_flow().get_zone_of_object(self) def move_to_zone(self, zone): """ Move this object to a flow zone :param object zone: a :class:`dataikuapi.dss.flow.DSSFlowZone` where to move the object """ if isinstance(zone, basestring): zone = self.project.get_flow().get_zone(zone) zone.add_item(self) def share_to_zone(self, zone): """ Share this object to a flow zone :param object zone: a :class:`dataikuapi.dss.flow.DSSFlowZone` where to share the object """ if isinstance(zone, basestring): zone = self.project.get_flow().get_zone(zone) zone.add_shared(self) def unshare_from_zone(self, zone): """ Unshare this object from a flow zone :param object zone: a :class:`dataikuapi.dss.flow.DSSFlowZone` from where to unshare the object """ if isinstance(zone, basestring): zone = self.project.get_flow().get_zone(zone) zone.remove_shared(self) def get_usages(self): """ Get the recipes or analyses referencing this dataset :returns: a list of usages :rtype: list[dict] """ return self.client._perform_json("GET", "/projects/%s/datasets/%s/usages" % (self.project_key, self.dataset_name)) def get_object_discussions(self): """ Get a handle to manage discussions on the dataset :returns: the handle to manage discussions :rtype: :class:`dataikuapi.dss.discussion.DSSObjectDiscussions` """ return DSSObjectDiscussions(self.client, self.project_key, "DATASET", self.dataset_name) ######################################################## # Test / Autofill ######################################################## _FS_TYPES = ["Filesystem", "UploadedFiles", "FilesInFolder", "HDFS", "S3", "Azure", "GCS", "FTP", "SCP", "SFTP"] # HTTP is FSLike but not FS _SQL_TYPES = ["JDBC", "PostgreSQL", "MySQL", "Vertica", "Snowflake", "Redshift", "Greenplum", "Teradata", "Oracle", "SQLServer", "SAPHANA", "Netezza", "BigQuery", "Athena", "hiveserver2", "Synapse", "Databricks"] def test_and_detect(self, infer_storage_types=False): """Used internally by :meth:`autodetect_settings` It is not usually required to call this method :param bool infer_storage_types: whether to infer storage types """ settings = self.get_settings() if settings.type in self.__class__._FS_TYPES: future_resp = self.client._perform_json("POST", "/projects/%s/datasets/%s/actions/testAndDetectSettings/fsLike"% (self.project_key, self.dataset_name), body = {"detectPossibleFormats" : True, "inferStorageTypes" : infer_storage_types }) return DSSFuture(self.client, future_resp.get('jobId', None), future_resp) elif settings.type in self.__class__._SQL_TYPES: return self.client._perform_json("POST", "/projects/%s/datasets/%s/actions/testAndDetectSettings/externalSQL"% (self.project_key, self.dataset_name)) elif settings.type == "ElasticSearch": return self.client._perform_json("POST", "/projects/%s/datasets/%s/actions/testAndDetectSettings/elasticsearch"% (self.project_key, self.dataset_name)) else: raise ValueError("don't know how to test/detect on dataset type:%s" % settings.type) def autodetect_settings(self, infer_storage_types=False): """ Detect appropriate settings for this dataset using Dataiku detection engine :param bool infer_storage_types: whether to infer storage types :returns: new suggested settings that you can :meth:`DSSDatasetSettings.save` :rtype: :class:`DSSDatasetSettings` or a subclass """ settings = self.get_settings() if settings.type in self.__class__._FS_TYPES: future = self.test_and_detect(infer_storage_types) result = future.wait_for_result() if not "format" in result or not result["format"]["ok"]: raise DataikuException("Format detection failed, complete response is " + json.dumps(result)) settings.get_raw()["formatType"] = result["format"]["type"] settings.get_raw()["formatParams"] = result["format"]["params"] settings.get_raw()["schema"] = result["format"]["schemaDetection"]["newSchema"] return settings elif settings.type in self.__class__._SQL_TYPES: result = self.test_and_detect() if not "schemaDetection" in result: raise DataikuException("Format detection failed, complete response is " + json.dumps(result)) settings.get_raw()["schema"] = result["schemaDetection"]["newSchema"] return settings elif settings.type == "ElasticSearch": result = self.test_and_detect() if not "schemaDetection" in result: raise DataikuException("Format detection failed, complete response is " + json.dumps(result)) settings.get_raw()["schema"] = result["schemaDetection"]["newSchema"] return settings else: raise ValueError("don't know how to test/detect on dataset type:%s" % settings.type) def get_as_core_dataset(self): """ Get the :class:`dataiku.Dataset` object corresponding to this dataset :rtype: :class:`dataiku.Dataset` """ import dataiku return dataiku.Dataset("%s.%s" % (self.project_key, self.dataset_name)) ######################################################## # Creation of recipes ######################################################## def new_code_recipe(self, type, code=None, recipe_name=None): """ Start the creation of a new code recipe taking this dataset as input. :param str type: type of the recipe ('python', 'r', 'pyspark', 'sparkr', 'sql', 'sparksql', 'hive', ...). :param str code: the code of the recipe. :param str recipe_name: (optional) base name for the new recipe. :returns: a handle to the new recipe's creator object. :rtype: Union[:class:`dataikuapi.dss.recipe.CodeRecipeCreator`, :class:`dataikuapi.dss.recipe.PythonRecipeCreator`] """ if type == "python": builder = recipe.PythonRecipeCreator(recipe_name, self.project) else: builder = recipe.CodeRecipeCreator(recipe_name, type, self.project) builder.with_input(self.dataset_name) if code is not None: builder.with_script(code) return builder def new_recipe(self, type, recipe_name=None): """ Start the creation of a new recipe taking this dataset as input. For more details, please see :meth:`dataikuapi.dss.project.DSSProject.new_recipe` :param str type: type of the recipe ('python', 'r', 'pyspark', 'sparkr', 'sql', 'sparksql', 'hive', ...). :param str recipe_name: (optional) base name for the new recipe. """ builder = self.project.new_recipe(type=type, name=recipe_name) builder.with_input(self.dataset_name) return builder ######################################################## # Data Quality ######################################################## def get_data_quality_rules(self): """ Get a handle to interact with the data quality rules of the dataset. :returns: A handle to the data quality rules of the dataset. :rtype: :class:`dataikuapi.dss.data_quality.DSSDataQualityRuleSet` """ return DSSDataQualityRuleSet(self.project_key, self.dataset_name, self.client) ######################################################## # Column Lineage ######################################################## def get_column_lineage(self, column, max_dataset_count=None): """ Get the full lineage (auto-computed and manual) information of a column in this dataset. Column relations with datasets from both local and foreign projects will be included in the result. :param str column: name of the column to retrieve the lineage on. :param integer max_dataset_count: (optional) the maximum number of datasets to query for. If none, then the max hard limit is used. :returns: the full column lineage (auto-computed and manual) as a list of relations. :rtype: list of dict """ if max_dataset_count is not None and max_dataset_count <= 0: raise ValueError("Invalid value, max_dataset_count must be a positive integer.") return self.client._perform_json( "GET", "/projects/%s/datasets/%s/column-lineage" % (self.project_key, self.dataset_name), params={ "columnName": column, "maxDatasetCount": max_dataset_count, } ) class DSSDatasetSettings(DSSTaggableObjectSettings): """ Base settings class for a DSS dataset. .. caution:: Do not instantiate this class directly, use :meth:`DSSDataset.get_settings` Use :meth:`save` to save your changes """ def __init__(self, dataset, settings): super(DSSDatasetSettings, self).__init__(settings) self.dataset = dataset self.settings = settings def get_raw(self): """Get the raw dataset settings as a dict. :rtype: dict """ return self.settings def get_raw_params(self): """Get the type-specific params, as a raw dict. :rtype: dict """ return self.settings["params"] @property def type(self): """Returns the settings type as a string. :rtype: string """ return self.settings["type"] @property def schema_columns(self): """ Get the schema columns settings. :returns: a list of dicts with column settings. :rtype: list[dict] """ return self.settings["schema"]["columns"] def remove_partitioning(self): """ Reset partitioning settings to those of a non-partitioned dataset. """ self.settings["partitioning"] = {"dimensions" : []} def add_discrete_partitioning_dimension(self, dim_name): """ Add a discrete partitioning dimension to settings. :param string dim_name: name of the partition to add. """ self.settings["partitioning"]["dimensions"].append({"name": dim_name, "type": "value"}) def add_time_partitioning_dimension(self, dim_name, period="DAY"): """ Add a time partitioning dimension to settings. :param string dim_name: name of the partition to add. :param string period: (optional) time granularity of the created partition. Can be YEAR, MONTH, DAY, HOUR. """ self.settings["partitioning"]["dimensions"].append({"name": dim_name, "type": "time", "params":{"period": period}}) def add_raw_schema_column(self, column): """ Add a column to the schema settings. :param dict column: column settings to add. """ self.settings["schema"]["columns"].append(column) @property def is_feature_group(self): """ Indicates whether the Dataset is defined as a Feature Group, available in the Feature Store. :rtype: bool """ return self.settings["featureGroup"] def set_feature_group(self, status): """ (Un)sets the dataset as a Feature Group, available in the Feature Store. Changes of this property will be applied when calling :meth:`save` and require the "Manage Feature Store" permission. :param status: whether the dataset should be defined as a feature group :type status: bool """ self.settings["featureGroup"] = status def save(self): """ Save settings. """ self.dataset.client._perform_empty( "PUT", "/projects/%s/datasets/%s" % (self.dataset.project_key, self.dataset.dataset_name), body=self.settings) class FSLikeDatasetSettings(DSSDatasetSettings): """ Settings for a files-based dataset. This class inherits from :class:`DSSDatasetSettings`. .. caution:: Do not instantiate this class directly, use :meth:`DSSDataset.get_settings` Use :meth:`save` to save your changes """ def __init__(self, dataset, settings): super(FSLikeDatasetSettings, self).__init__(dataset, settings) def set_connection_and_path(self, connection, path): """ Set connection and path parameters. :param string connection: connection to use. :param string path: path to use. """ self.settings["params"]["connection"] = connection self.settings["params"]["path"] = path def get_raw_format_params(self): """ Get the raw format parameters as a dict. :rtype: dict """ return self.settings["formatParams"] def set_format(self, format_type, format_params = None): """ Set format parameters. :param string format_type: format type to use. :param dict format_params: dict of parameters to assign to the formatParams settings section. """ if format_params is None: format_params = {} self.settings["formatType"] = format_type self.settings["formatParams"] = format_params def set_csv_format(self, separator=",", style="excel", skip_rows_before=0, header_row=True, skip_rows_after=0): """ Set format parameters for a csv-based dataset. :param string separator: (optional) separator to use, default is ','". :param string style: (optional) style to use, default is 'excel'. :param int skip_rows_before: (optional) number of rows to skip before header, default is 0. :param bool header_row: (optional) wheter or not the header row is parsed, default is true. :param int skip_rows_after: (optional) number of rows to skip before header, default is 0. """ format_params = { "style" : style, "separator": separator, "skipRowsBeforeHeader": skip_rows_before, "parseHeaderRow": header_row, "skipRowsAfterHeader": skip_rows_after } self.set_format("csv", format_params) def set_partitioning_file_pattern(self, pattern): """ Set the dataset partitionning file pattern. :param str pattern: pattern to set. """ self.settings["partitioning"]["filePathPattern"] = pattern class SQLDatasetSettings(DSSDatasetSettings): """ Settings for a SQL dataset. This class inherits from :class:`DSSDatasetSettings`. .. caution:: Do not instantiate this class directly, use :meth:`DSSDataset.get_settings` Use :meth:`save` to save your changes """ def __init__(self, dataset, settings): super(SQLDatasetSettings, self).__init__(dataset, settings) def set_table(self, connection, schema, table, catalog=None): """ Sets this SQL dataset in 'table' mode, targeting a particular table of a connection Leave catalog to None to target the default database associated with the connection """ self.settings["params"].update({ "connection": connection, "mode": "table", "schema": schema, "table": table, "catalog": catalog }) class DSSManagedDatasetCreationHelper(object): """Provide an helper to create partitioned dataset .. code-block:: python import dataiku client = dataiku.api_client() project_key = dataiku.default_project_key() project = client.get_project(project_key) #create the dataset builder = project.new_managed_dataset("py_generated") builder.with_store_into("filesystem_folders") dataset = builder.create(overwrite=True) #setup format & schema settings ds_settings = ds.get_settings() ds_settings.set_csv_format() ds_settings.add_raw_schema_column({'name':'id', 'type':'int'}) ds_settings.add_raw_schema_column({'name':'name', 'type':'string'}) ds_settings.save() #put some data data = ["foo", "bar"] with ds.get_as_core_dataset().get_writer() as writer: for idx, val in enumerate(data): writer.write_row_array((idx, val)) .. caution:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.new_managed_dataset` """ def __init__(self, project, dataset_name): self.project = project self.dataset_name = dataset_name self.creation_settings = { "specificSettings" : {} } def get_creation_settings(self): """ Get the dataset creation settings as a dict. :rtype: dict """ return self.creation_settings def with_store_into(self, connection, type_option_id = None, format_option_id = None): """ Sets the connection into which to store the new managed dataset :param str connection: Name of the connection to store into :param str type_option_id: If the connection accepts several types of datasets, the type :param str format_option_id: Optional identifier of a file format option :returns: self """ self.creation_settings["connectionId"] = connection if type_option_id is not None: self.creation_settings["typeOptionId"] = type_option_id if format_option_id is not None: self.creation_settings["specificSettings"]["formatOptionId"] = format_option_id return self def with_copy_partitioning_from(self, dataset_ref, object_type='DATASET'): """ Sets the new managed dataset to use the same partitioning as an existing dataset :param str dataset_ref: Name of the dataset to copy partitioning from :param str object_type: Type of the object to copy partitioning from, values can be DATASET or FOLDER :returns: self """ code = 'dataset' if object_type == 'DATASET' else 'folder' self.creation_settings["partitioningOptionId"] = "copy:%s:%s" % (code, dataset_ref) return self def create(self, overwrite=False): """ Executes the creation of the managed dataset according to the selected options :param overwrite: If the dataset being created already exists, delete it first (removing data), defaults to False :type overwrite: bool, optional :returns: the newly created dataset :rtype: :class:`DSSDataset` """ if overwrite and self.already_exists(): self.project.get_dataset(self.dataset_name).delete(drop_data = True) self.project.client._perform_json("POST", "/projects/%s/datasets/managed" % self.project.project_key, body = { "name": self.dataset_name, "creationSettings": self.creation_settings }) return DSSDataset(self.project.client, self.project.project_key, self.dataset_name) def already_exists(self): """ Check if dataset already exists. :returns: whether this managed dataset already exists :rtype: bool """ dataset = self.project.get_dataset(self.dataset_name) try: dataset.get_metadata() return True except Exception as e: return False class DSSDatasetInfo(object): """ Info class for a DSS dataset (Read-Only). .. caution:: Do not instantiate this class directly, use :meth:`DSSDataset.get_info` """ def __init__(self, dataset, info): self.dataset = dataset self.info = info def get_raw(self): """ Get the raw dataset full information as a dict :returns: the raw dataset full information :rtype: dict """ return self.info @property def last_build_start_time(self): """ The last build start time of the dataset as a :class:`datetime.datetime` or None if there is no last build information. :returns: the last build start time :rtype: :class:`datetime.datetime` or None """ last_build_info = self.info.get("lastBuild", dict()) timestamp = last_build_info.get("buildStartTime", None) return _timestamp_ms_to_zoned_datetime(timestamp) @property def last_build_end_time(self): """ The last build end time of the dataset as a :class:`datetime.datetime` or None if there is no last build information. :returns: the last build end time :rtype: :class:`datetime.datetime` or None """ last_build_info = self.info.get("lastBuild", dict()) timestamp = last_build_info.get("buildEndTime", None) return _timestamp_ms_to_zoned_datetime(timestamp) @property def is_last_build_successful(self): """ Get whether the last build of the dataset is successful. :returns: True if the last build is successful :rtype: bool """ last_build_info = self.info.get("lastBuild", dict()) success = last_build_info.get("buildSuccess", False) return success dataiku-api-client-python-13.2.2/dataikuapi/dss/discussion.py000066400000000000000000000162711471211575100242400ustar00rootroot00000000000000import json import sys if sys.version_info >= (3,0): import urllib.parse dku_quote_fn = urllib.parse.quote else: import urllib dku_quote_fn = urllib.quote class DSSObjectDiscussions(object): """ A handle to manage discussions on a DSS object. .. important:: Do not create this class directly, instead use :meth:`dataikuapi.DSSClient.get_object_discussions` on any commentable DSS object. """ def __init__(self, client, project_key, object_type, object_id): self.client = client self.project_key = project_key self.object_type = object_type self.object_id = object_id # encode in UTF-8 if its python2 and unicode if sys.version_info < (3,0) and isinstance(self.object_id, unicode): self.object_id = self.object_id.encode('utf-8') def list_discussions(self): """ Gets the list of discussions on the object. :returns: list of discussions on the object :rtype: list of :class:`dataikuapi.dss.discussion.DSSDiscussion` """ data = self.client._perform_json("GET", "/projects/%s/discussions/%s/%s/" % (self.project_key, self.object_type, self.object_id)) return [DSSDiscussion(self.client, self.project_key, self.object_type, self.object_id, discu_data['id'], discu_data, False) for discu_data in data] def create_discussion(self, topic, message): """ Creates a new discussion with one message. :param str topic: the discussion topic :param str message: the markdown formatted first message :returns: the newly created discussion :rtype: :class:`dataikuapi.dss.discussion.DSSDiscussion` """ creation_data = { "topic" : topic, "reply" : message } discu_data = self.client._perform_json("POST", "/projects/%s/discussions/%s/%s/" % (self.project_key, self.object_type, self.object_id), body=creation_data) return DSSDiscussion(self.client, self.project_key, self.object_type, self.object_id, discu_data['id'], discu_data, True) def get_discussion(self, discussion_id): """ Gets a specific discussion. :param str discussion_id: the discussion ID :returns: the discussion :rtype: :class:`dataikuapi.dss.discussion.DSSDiscussion` """ discu_data = self.client._perform_json("GET", "/projects/%s/discussions/%s/%s/%s" % (self.project_key, self.object_type, self.object_id, discussion_id)) return DSSDiscussion(self.client, self.project_key, self.object_type, self.object_id, discussion_id, discu_data, True) class DSSDiscussion(object): """ A handle to interact with a discussion. .. important:: Do not call directly, use :meth:`dataikuapi.dss.discussion.DSSObjectDiscussions.get_discussion`, :meth:`dataikuapi.dss.discussion.DSSObjectDiscussions.create_discussion` or :meth:`dataikuapi.dss.discussion.DSSObjectDiscussions.list_discussions`. """ def __init__(self, client, project_key, object_type, object_id, discussion_id, discussion_data, discussion_data_has_replies): """ :param :class:`dataikuapi.dssclient.DSSClient` client: an api client to connect to the DSS backend :param str project_key: identifier of the project to access :param str object_type: DSS object type :param str object_id: DSS object ID :param str discussion_id: identifier of the discussion :param dict discussion_data: the discussion data :param bool discussion_data_has_replies: a flag that indicates if the replies has been loaded """ self.client = client self.project_key = project_key self.object_type = object_type self.object_id = object_id self.discussion_id = discussion_id self.discussion_data = discussion_data self.discussion_data_has_replies = discussion_data_has_replies def _get_with_replies(self): """ Reloads the discussion data from the backend including the replies. """ self.discussion_data = self.client._perform_json("GET", "/projects/%s/discussions/%s/%s/%s" % (self.project_key, self.object_type, self.object_id, self.discussion_id)) self.discussion_data_has_replies = True def get_metadata(self): """ Gets the discussion metadata. :returns: the discussion metadata :rtype: dict """ metadata = dict(self.discussion_data) if "replies" in metadata: del metadata["replies"] return metadata def set_metadata(self, discussion_metadata): """ Updates the discussion metadata. :param dict discussion_metadata: the discussion metadata """ if not self.discussion_data_has_replies: self._get_with_replies() edited_metadata = dict(discussion_metadata) edited_metadata["replies"] = self.discussion_data["replies"] self.discussion_data = self.client._perform_json("PUT", "/projects/%s/discussions/%s/%s/%s" % (self.project_key, self.object_type, self.object_id, self.discussion_id), body=edited_metadata) self.discussion_data_has_replies = True def get_replies(self): """ Gets the list of replies in this discussion. :returns: a list of replies :rtype: list of :class:`dataikuapi.dss.discussion.DSSDiscussionReply` """ if not self.discussion_data_has_replies: self._get_with_replies() return [DSSDiscussionReply(reply_data) for reply_data in self.discussion_data["replies"]] def add_reply(self, text): """ Adds a reply to a discussion. :param str text: the markdown formatted text to reply """ reply_data = { "reply": text } self.discussion_data = self.client._perform_json("POST", "/projects/%s/discussions/%s/%s/%s/replies/" % (self.project_key, self.object_type, self.object_id, self.discussion_id), body=reply_data) self.discussion_data_has_replies = True class DSSDiscussionReply(object): """ A read-only handle to access a discussion reply. .. important:: Do not create this class directly, use :meth:`dataikuapi.dss.discussion.DSSDiscussion.get_replies` """ def __init__(self, reply_data): self.reply_data = reply_data def get_raw_data(self): """ Gets the reply raw data. :returns: the reply data :rtype: dict """ return self.reply_data def get_text(self): """ Gets the reply text. :returns: the reply text :rtype: str """ return self.reply_data["text"] def get_author(self): """ Gets the reply author. :returns: the author ID :rtype: str """ return self.reply_data["author"] def get_timestamp(self): """ Gets the reply timestamp. :returns: the reply timestamp :rtype: long """ return self.reply_data["time"] def get_edited_timestamp(self): """ Gets the last edition timestamp. :returns: the last edition timestamp :rtype: long """ return self.reply_data["editedOn"] dataiku-api-client-python-13.2.2/dataikuapi/dss/feature_store.py000066400000000000000000000032071471211575100247170ustar00rootroot00000000000000from dataikuapi.dss.dataset import DSSDataset class DSSFeatureGroupListItem(object): """ An item in a list of feature groups. .. important:: Do not instantiate this class, use :meth:'dataikuapi.dss.feature_store.DSSFeatureStore.list_feature_groups' """ def __init__(self, client, project_key, name): self.client = client self.project_key = project_key self.name = name @property def id(self): """ Get the identifier of the feature group. :rtype: string """ return self.project_key + "." + self.name def get_as_dataset(self): """ Get the feature group as a dataset. :returns: a handle on the dataset :rtype: :class:`dataikuapi.dss.dataset.DSSDataset` """ return DSSDataset(self.client, self.project_key, self.name) class DSSFeatureStore(object): """ A handle on the Feature Store. .. important:: Do not create this class directly, use :meth:`dataikuapi.DSSClient.get_feature_store` """ def __init__(self, client): self.client = client def list_feature_groups(self): """ List the feature groups on which the user has at least read permissions. :returns: the list of feature groups, each one as a :class:`dataikuapi.dss.feature_store.DSSFeatureGroupListItem` :rtype: list of :class:`dataikuapi.dss.feature_store.DSSFeatureGroupListItem` """ items = self.client._perform_json("GET", "/feature-store/feature-groups") return [DSSFeatureGroupListItem(self.client, item["projectKey"], item["name"]) for item in items] dataiku-api-client-python-13.2.2/dataikuapi/dss/flow.py000066400000000000000000001053501471211575100230210ustar00rootroot00000000000000from .labeling_task import DSSLabelingTask from ..utils import _write_response_content_to_file from .utils import AnyLoc from .dataset import DSSDataset from .managedfolder import DSSManagedFolder from .savedmodel import DSSSavedModel from .modelevaluationstore import DSSModelEvaluationStore from .recipe import DSSRecipe, DSSRecipeDefinitionAndPayload from .knowledgebank import DSSKnowledgeBank from .future import DSSFuture from .streaming_endpoint import DSSStreamingEndpoint import logging, json class DSSProjectFlow(object): def __init__(self, client, project): self.client = client self.project = project def get_graph(self): """ Get the flow graph. :return: A handle to use the flow graph :rtype: :class:`.DSSProjectFlowGraph` """ data = self.client._perform_json("GET", "/projects/%s/flow/graph/" % (self.project.project_key)) return DSSProjectFlowGraph(self, data) def create_zone(self, name, color="#2ab1ac"): """ Creates a new flow zone. :param str name: new flow zone name :param str color: new flow zone color, in hexadecimal format #RRGGBB (defaults to **#2ab1ac**) :returns: the newly created zone :rtype: :class:`DSSFlowZone` """ data = self.client._perform_json("POST", "/projects/%s/flow/zones" % (self.project.project_key), body={ "name": name, "color": color }) return DSSFlowZone(self, data) def get_zone(self, id): """ Gets a single Flow zone by id. :param str id: flow zone id :returns: A flow zone :rtype: :class:`DSSFlowZone` """ data = self.client._perform_json("GET", "/projects/%s/flow/zones/%s" % (self.project.project_key, id)) return DSSFlowZone(self, data) def get_default_zone(self): """ Returns the default zone of the Flow. :returns: A flow zone :rtype: :class:`DSSFlowZone` """ return self.get_zone("default") def list_zones(self): """ Lists all zones in the Flow. :returns: A list of flow zones :rtype: list of :class:`DSSFlowZone` """ data = self.client._perform_json("GET", "/projects/%s/flow/zones" % (self.project.project_key)) return [DSSFlowZone(self, z) for z in data] def get_zone_of_object(self, obj): """ Finds the zone to which this object belongs. If the object is not found in any specific zone, it belongs to the default zone, and the default zone is returned. :param obj: An object to search :type obj: :class:`.DSSDataset`, :class:`.DSSManagedFolder`, :class:`.DSSSavedModel`, :class:`.DSSRecipe` :class:`.DSSModelEvaluationStore` or :class:`.DSSStreamingEndpoint` :returns: The flow zone containing the object :rtype: :class:`DSSFlowZone` """ sr = self._to_smart_ref(obj) for zone in self.list_zones(): for item in zone._raw["items"]: if json.dumps(sr) == json.dumps(item): return zone return self.get_default_zone() def replace_input_computable(self, current_ref, new_ref, type="DATASET"): """ This method replaces all references to a "computable" (Dataset, Managed Folder or Saved Model) as input of recipes in the whole Flow by a reference to another computable. No specific checks are performed. It is your responsibility to ensure that the schema of the new dataset will be compatible with the previous one (in the case of datasets). If `new_ref` references an object in a foreign project, this method will automatically ensure that `new_ref` is exposed to the current project. :param str current_ref: Either a "simple" object id (dataset name or model/folder/model evaluation store/streaming endpoint id) or a foreign object reference in the form "FOREIGN_PROJECT_KEY.local_id" :param str new_ref: Either a "simple" object id (dataset name or model/folder/model evaluation store/streaming endpoint id) or a foreign object reference in the form "FOREIGN_PROJECT_KEY.local_id" :param str type: The type of object being replaced (DATASET, SAVED_MODEL, MANAGED_FOLDER, MODEL_EVALUATION_STORE, STREAMING_ENDPOINT) (defaults to **DATASET**) """ new_loc = AnyLoc.from_ref(self.project.project_key, new_ref) if new_loc.project_key != self.project.project_key: logging.info("New ref is in project %s, exposing it to project %s" % (new_loc.project_key, self.project.project_key)) new_ref_src_project = self.client.get_project(new_loc.project_key) settings = new_ref_src_project.get_settings() settings.add_exposed_object(type, new_loc.object_id, self.project.project_key) settings.save() for recipe in self.project.list_recipes(): recipe_handle = self.project.get_recipe(recipe["name"]) fake_rap = DSSRecipeDefinitionAndPayload(recipe_handle, {"recipe": recipe}) if fake_rap.has_input(current_ref): logging.info("Recipe %s has %s as input, performing the replacement by %s" % \ (recipe["name"], current_ref, new_ref)) recipe_obj = self.project.get_recipe(recipe["name"]) dap = recipe_obj.get_definition_and_payload() dap.replace_input(current_ref, new_ref) recipe_obj.set_definition_and_payload(dap) def generate_documentation(self, folder_id=None, path=None): """ Start the flow document generation from a template docx file in a managed folder, or from the default template if no folder id and path are specified. :param str folder_id: the id of the managed folder (defaults to **None**) :param str path: the path to the file from the root of the folder (defaults to **None**) :return: The flow document generation future :rtype: :class:`.DSSFuture` """ if bool(folder_id) != bool(path): raise ValueError("Both folder id and path arguments are required to use a template from folder. " + "Use without argument to generate the flow documentation using the default template") template_mode_url = "" if folder_id is None and path is None else "-with-template-in-folder" f = self.client._perform_json("POST", "/projects/%s/flow/documentation/generate%s" % (self.project.project_key, template_mode_url), params={"folderId": folder_id, "path": path}) return DSSFuture(self.client, f["jobId"]) def generate_documentation_from_custom_template(self, fp): """ Start the flow document generation from a docx template (as a file object). :param object fp: A file-like object pointing to a template docx file :return: The flow document generation future :rtype: :class:`.DSSFuture` """ files = {'file': fp} f = self.client._perform_json("POST", "/projects/%s/flow/documentation/generate-with-template" % self.project.project_key, files=files) return DSSFuture(self.client, f["jobId"]) def download_documentation_stream(self, export_id): """ Download a flow documentation, as a binary stream. .. warning :: You need to close the stream after download. Failure to do so will result in the DSSClient becoming unusable. :param str export_id: the id of the generated flow documentation returned as the result of the future :returns: the generated flow documentation file as a stream :rtype: :class:`requests.Response` """ return self.client._perform_raw("GET", "/projects/%s/flow/documentation/generated/%s" % (self.project.project_key, export_id)) def download_documentation_to_file(self, export_id, path): """ Download a flow documentation into the given output file. :param str export_id: the id of the generated flow documentation returned as the result of the future :param str path: the path where to download the flow documentation """ stream = self.download_documentation_stream(export_id) _write_response_content_to_file(stream, path) ######################################################## # Flow tools ######################################################## def start_tool(self, type, data={}): """ Start a tool or open a view in the flow. :param str type: one of {COPY, CHECK_CONSISTENCY, PROPAGATE_SCHEMA} (tools) or {TAGS, CUSTOM_FIELDS, CONNECTIONS, COUNT_OF_RECORDS, FILESIZE, FILEFORMATS, RECIPES_ENGINES, RECIPES_CODE_ENVS, IMPALA_WRITE_MODE, HIVE_MODE, SPARK_ENGINE, SPARK_CONFIG, SPARK_PIPELINES, SQL_PIPELINES, PARTITIONING, PARTITIONS, SCENARIOS, CREATION, LAST_MODIFICATION, LAST_BUILD, RECENT_ACTIVITY, WATCH} (views) :param dict data: initial data for the tool (defaults to **{}**) :returns: A handle to interact with the newly-created tool or view :rtype: :class:`.DSSFlowTool` """ tool_id = self.client._perform_text("POST", "/projects/%s/flow/tools/" % self.project.project_key, params={'type': type}, body=data) return DSSFlowTool(self.client, self.project.project_key, tool_id) def new_schema_propagation(self, dataset_name): """ Start an automatic schema propagation from a dataset. :param str dataset_name: name of a dataset to start propagating from :returns: A handle to set options and start the propagation :rtype: :class:`.DSSSchemaPropagationRunBuilder` """ return DSSSchemaPropagationRunBuilder(self.project, self.client, dataset_name) def _to_smart_ref(self, obj): if isinstance(obj, DSSDataset): ot = "DATASET" elif isinstance(obj, DSSManagedFolder): ot = "MANAGED_FOLDER" elif isinstance(obj, DSSSavedModel): ot = "SAVED_MODEL" elif isinstance(obj, DSSModelEvaluationStore): ot = "MODEL_EVALUATION_STORE" elif isinstance(obj, DSSRecipe): ot = "RECIPE" elif isinstance(obj, DSSStreamingEndpoint): ot = "STREAMING_ENDPOINT" elif isinstance(obj, DSSLabelingTask): ot = "LABELING_TASK" elif isinstance(obj, DSSKnowledgeBank): ot = "RETRIEVABLE_KNOWLEDGE" else: raise ValueError("Cannot transform to DSS object ref: %s" % obj) if obj.project_key == self.project.project_key: return { "objectId": obj.id, "objectType": ot } else: return { "projectKey": obj.project_key, "objectId": obj.id, "objectType": ot } class DSSSchemaPropagationRunBuilder(object): """ .. important :: Do not create this directly, use :meth:`DSSProjectFlow.new_schema_propagation`. """ def __init__(self, project, client, dataset_name): self.project = project self.client = client self.dataset_name = dataset_name self.settings = { 'recipeUpdateOptions': { "byType": {}, "byName": {} }, 'defaultPartitionValuesByDimension': {}, 'partitionsByComputable': {}, 'excludedRecipes': [], 'markAsOkRecipes': [], 'autoRebuild': True } def set_auto_rebuild(self, auto_rebuild): """ Sets whether to automatically rebuild datasets if needed while propagating. :param bool auto_rebuild: whether to automatically rebuild datasets if needed (defaults to **True**) """ self.settings["autoRebuild"] = auto_rebuild def set_default_partitioning_value(self, dimension, value): """ In the case of partitioned flows, sets the default partition value to use when rebuilding, for a specific dimension name. :param str dimension: a partitioning dimension name :param str value: a partitioning dimension value """ self.settings["defaultPartitionValuesByDimension"][dimension] = value def set_partition_for_computable(self, full_id, partition): """ In the case of partitioned flows, sets the partition id to use when building a particular computable. Overrides the default partitioning value per dimension. :param str full_id: Full name of the computable, in the form PROJECTKEY.id :param str partition: a full partition id (all dimensions) """ self.settings["partitionsByComputable"][full_id] = partition def stop_at(self, recipe_name): """ Sets the given recipe as a schema propagation stop mark. :param str recipe_name: the name of the recipe """ self.settings["excludedRecipes"].append(recipe_name) def mark_recipe_as_ok(self, name): """ Marks a recipe as always considered as OK during propagation. :param str name: recipe to mark as ok """ self.settings["markAsOkRecipes"].append(name) def set_grouping_update_options(self, recipe=None, remove_missing_aggregates=True, remove_missing_keys=True, new_aggregates={}): """ Sets update options for grouping recipes. :param str recipe: if None, applies to all grouping recipes. Else, applies only to this name (defaults to **None**) :param bool remove_missing_aggregates: whether to remove missing aggregates (defaults to **True**) :param bool remove_missing_keys: whether to remove missing keys (defaults to **True**) :param dict new_aggregates: new aggregates (defaults to **{}**) """ data = { "removeMissingAggregates": remove_missing_aggregates, "removeMissingKeys": remove_missing_keys, "newAggregates": new_aggregates } if recipe is None: self.settings["recipeUpdateOptions"]["byType"]["grouping"] = data else: self.settings["recipeUpdateOptions"]["byName"][recipe] = data def set_window_update_options(self, recipe=None, remove_missing_aggregates=True, remove_missing_in_window=True, new_aggregates={}): """ Sets update options for window recipes. :param str recipe: if None, applies to all window recipes. Else, applies only to this name (defaults to **None**) :param bool remove_missing_aggregates: whether to remove missing aggregates (defaults to **True**) :param bool remove_missing_in_window: whether to remove missing keys in windows (defaults to **True**) :param dict new_aggregates: new aggregates (defaults to **{}**) """ data = { "removeMissingAggregates": remove_missing_aggregates, "removeMissingInWindow": remove_missing_in_window, "newAggregates": new_aggregates } if recipe is None: self.settings["recipeUpdateOptions"]["byType"]["window"] = data else: self.settings["recipeUpdateOptions"]["byName"][recipe] = data def set_join_update_options(self, recipe=None, remove_missing_join_conditions=True, remove_missing_join_values=True, new_selected_columns={}): """ Sets update options for join recipes. :param str recipe: if None, applies to all join recipes. Else, applies only to this name (defaults to **None**) :param bool remove_missing_join_conditions: whether to remove missing join conditions (defaults to **True**) :param bool remove_missing_join_values: whether to remove missing join values (defaults to **True**) :param dict new_selected_columns: new selected columns (defaults to **{}**) """ data = { "removeMissingJoinConditions": remove_missing_join_conditions, "removeMissingJoinValues": remove_missing_join_values, "newSelectedColumns": new_selected_columns } if recipe is None: self.settings["recipeUpdateOptions"]["byType"]["join"] = data else: self.settings["recipeUpdateOptions"]["byName"][recipe] = data def start(self): """ Starts the actual propagation. Returns a future to wait for completion. :returns: A future representing the schema propagation job :rtype: :class:`.DSSFuture` """ loc = AnyLoc.from_ref(self.project.project_key, self.dataset_name) future_resp = self.client._perform_json("POST", "/projects/%s/flow/tools/propagate-schema/" % self.project.project_key, body={ "options": self.settings, "sources": [{"projectKey": loc.project_key, "id": loc.object_id}]}) return DSSFuture.from_resp(self.client, future_resp) class DSSFlowZone(object): """ A zone in the Flow. .. important :: Do not create this object manually, use :meth:`DSSProjectFlow.get_zone` or :meth:`DSSProjectFlow.list_zones` """ def __init__(self, flow, data): self.flow = flow self.client = flow.client self._raw = data @property def id(self): return self._raw["id"] @property def name(self): return self._raw["name"] @property def color(self): return self._raw["color"] def __repr__(self): return "" % (self.id, self.name) def get_settings(self): """Gets the settings of this zone in order to modify them. :returns: The settings of the flow zone :rtype: :class:`DSSFlowZoneSettings` """ return DSSFlowZoneSettings(self) def _to_native_obj(self, zone_item): if not "projectKey" in zone_item or zone_item["projectKey"] == self.flow.project.project_key: p = self.flow.project else: p = self.client.get_project(zone_item["projectKey"]) if zone_item["objectType"] == "DATASET": return p.get_dataset(zone_item["objectId"]) elif zone_item["objectType"] == "MANAGED_FOLDER": return p.get_managed_folder(zone_item["objectId"]) elif zone_item["objectType"] == "SAVED_MODEL": return p.get_saved_model(zone_item["objectId"]) elif zone_item["objectType"] == "RECIPE": return p.get_recipe(zone_item["objectId"]) elif zone_item["objectType"] == "STREAMING_ENDPOINT": return p.get_streaming_endpoint(zone_item["objectId"]) elif zone_item["objectType"] == "LABELING_TASK": return p.get_labeling_task(zone_item["objectId"]) elif zone_item["objectType"] == "MODEL_EVALUATION_STORE": return p.get_model_evaluation_store(zone_item["objectId"]) elif zone_item["objectType"] == "RETRIEVABLE_KNOWLEDGE": return p.get_knowledge_bank(zone_item["objectId"]) else: raise ValueError("Cannot transform to DSS object: %s" % zone_item) def add_item(self, obj): """ Adds an item to this zone. The item will automatically be moved from its existing zone. Additional items may be moved to this zone as a result of the operation (notably the recipe generating `obj`). :param obj: object to add to the zone :type obj: :class:`.DSSDataset`, :class:`.DSSManagedFolder`, :class:`.DSSSavedModel`, :class:`.DSSRecipe`, :class:`.DSSModelEvaluationStore` or :class:`.DSSStreamingEndpoint` """ self._raw = self.client._perform_json("POST", "/projects/%s/flow/zones/%s/items" % (self.flow.project.project_key, self.id), body=self.flow._to_smart_ref(obj)) def add_items(self, items): """ Adds items to this zone. The items will automatically be moved from their existing zones. Additional items may be moved to this zone as a result of the operations (notably the recipe generating the `items`). :param items: A list of objects to add to the zone :type items: list of :class:`.DSSDataset`, :class:`.DSSManagedFolder`, :class:`.DSSSavedModel`, :class:`.DSSRecipe`, :class:`.DSSModelEvaluationStore` or :class:`.DSSStreamingEndpoint` """ smart_refs = [] for item in items: smart_refs.append(self.flow._to_smart_ref(item)) self._raw = self.client._perform_json("POST", "/projects/%s/flow/zones/%s/add-items" % (self.flow.project.project_key, self.id), body=smart_refs) @property def items(self): """ The list of items explicitly belonging to this zone. This list is read-only. To add an object, use :meth:`add_item`. It will remove it from its current zone, if any. To remove an object from a zone without placing in another specific zone, add it to the default zone: ``flow.get_zone('default').add_item(item)`` .. note :: The "default" zone content is defined as all items that are not explicitly in another zone. It cannot directly be listed with the ``items`` property. To get the list of items including those in the default zone, use the :meth:`get_graph` method. :returns: the items in the zone :rtype: list of :class:`.DSSDataset`, :class:`.DSSManagedFolder`, :class:`.DSSSavedModel`, :class:`.DSSRecipe`, :class:`.DSSModelEvaluationStore` or :class:`.DSSStreamingEndpoint` """ return [self._to_native_obj(i) for i in self._raw["items"]] def add_shared(self, obj): """ Share an item to this zone. The item will not be automatically unshared from its existing zone. :param obj: object to share to the zone :type obj: :class:`.DSSDataset`, :class:`.DSSManagedFolder`, :class:`.DSSSavedModel`, :class:`.DSSRecipe`, :class:`.DSSModelEvaluationStore` or :class:`.DSSStreamingEndpoint` """ self._raw = self.client._perform_json("POST", "/projects/%s/flow/zones/%s/shared" % (self.flow.project.project_key, self.id), body=self.flow._to_smart_ref(obj)) def remove_shared(self, obj): """ Remove a shared item from this zone. :param obj: object to remove from the zone :type obj: :class:`.DSSDataset`, :class:`.DSSManagedFolder`, :class:`.DSSSavedModel`, :class:`.DSSModelEvaluationStore` or :class:`.DSSStreamingEndpoint` """ smartRef = self.flow._to_smart_ref(obj) self._raw = self.client._perform_json("DELETE", "/projects/%s/flow/zones/%s/shared/%s/%s" % ( self.flow.project.project_key, self.id, smartRef['objectType'], smartRef['objectId'])) @property def shared(self): """ The list of items that have been explicitly pre-shared to this zone. This list is read-only, to modify it, use :meth:`add_shared` and :meth:`remove_shared` :returns: the items shared to this zone :rtype: list of :class:`.DSSDataset`, :class:`.DSSManagedFolder`, :class:`.DSSSavedModel`, :class:`.DSSModelEvaluationStore` or :class:`.DSSStreamingEndpoint` """ return [self._to_native_obj(i) for i in self._raw["shared"]] def get_graph(self): """ Get the flow graph. :return: A handle to use the flow graph :rtype: :class:`DSSProjectFlowGraph` """ data = self.client._perform_json("GET", "/projects/%s/flow/zones/%s/graph" % (self.flow.project.project_key, self.id)) return DSSProjectFlowGraph(self.flow, data) def delete(self): """ Delete the zone, all items will be moved to the default zone. """ return self.client._perform_empty("DELETE", "/projects/%s/flow/zones/%s" % (self.flow.project.project_key, self.id)) class DSSFlowZoneSettings(object): """ The settings of a flow zone. .. important :: Do not create this directly, use :meth:`DSSFlowZone.get_settings`. """ def __init__(self, zone): self._zone = zone self._raw = zone._raw def get_raw(self): """ Gets the raw settings of the zone. .. note :: You cannot modify the `items` and `shared` elements through this class. Instead, use :meth:`DSSFlowZone.add_item` and others """ return self._raw @property def name(self): return self._raw["name"] @name.setter def name(self, new_name): self._raw["name"] = new_name @property def color(self): return self._raw["color"] @color.setter def color(self, new_color): self._raw["color"] = new_color def save(self): """Saves the settings of the zone""" self._zone.client._perform_empty("PUT", "/projects/%s/flow/zones/%s" % (self._zone.flow.project.project_key, self._zone.id), body=self._raw) class DSSProjectFlowGraph(object): def __init__(self, flow, data): self.flow = flow self.data = data self.nodes = data["nodes"] def get_source_computables(self, as_type="dict"): """ :param str as_type: How to return the source computables. Possible values are "dict" and "object" (defaults to **dict**) :returns: The list of source computables :rtype: If as_type=dict, each computable is returned as a dict containing at least "ref" and "type". If as_type=object, each computable is returned as a :class:`.DSSDataset`, :class:`.DSSManagedFolder`, :class:`.DSSSavedModel`, :class:`.DSSModelEvaluationStore` or :class:`.DSSStreamingEndpoint` """ ret = [] for node in self.nodes.values(): if len(node["predecessors"]) == 0 and node["type"].startswith("COMPUTABLE"): ret.append(node) return self._convert_nodes_list(ret, as_type) def get_source_recipes(self, as_type="dict"): """ :param str as_type: How to return the source recipes. Possible values are "dict" and "object" (defaults to **dict**) :returns: The list of source recipes :rtype: If as_type=dict, each recipe is returned as a dict containing at least "ref" and "type". If as_type=object, each recipe is returned as a :class:`.DSSRecipe`. """ ret = [] for node in self.nodes.values(): if len(node["predecessors"]) == 0 and node["type"] == "RUNNABLE_RECIPE": ret.append(node) return self._convert_nodes_list(ret, as_type) def get_source_datasets(self): """ :returns: The list of source datasets for this project :rtype: List of :class:`.DSSDataset` """ return [self._get_object_from_graph_node(x) for x in self.get_source_computables() if x["type"] == "COMPUTABLE_DATASET"] def get_successor_recipes(self, node, as_type="dict"): """ :param node: Either a name or a dataset object :type node: str or :class:`.DSSDataset` :param str as_type: How to return the successor recipes. Possible values are "dict" and "object" (defaults to **dict**) :returns: A list of recipes that are a successor of the given graph node :rtype: If as_type=dict, each recipe is returned as a dict containing at least "ref" and "type". If as_type=object, each recipe is returned as a :class:`.DSSRecipe`. """ if isinstance(node, DSSDataset): node = node.dataset_name computable = self.nodes.get(node, None) if computable is None: raise ValueError("Computable %s not found in Flow graph" % node) runnables = [self.nodes[x] for x in computable["successors"]] return self._convert_nodes_list(runnables, as_type) def get_successor_computables(self, node, as_type="dict"): """ :param node: Either a name or a recipe object :type node: str or :class:`.DSSRecipe` :param str as_type: How to return the successor computables. Possible values are "dict" and "object" (defaults to **dict**). :returns: A list of computables that are a successor of a given graph node :rtype: If as_type=dict, each computable is returned as a dict containing at least "ref" and "type". If as_type=object, each computable is returned as a :class:`.DSSDataset`. :class:`.DSSManagedFolder`, :class:`.DSSSavedModel`, :class:`.DSSModelEvaluationStore` or :class:`.DSSStreamingEndpoint` """ if isinstance(node, DSSRecipe): node = node.recipe_name runnable = self.nodes.get(node, None) if runnable is None: raise ValueError("Runnable %s not found in Flow graph" % node) computables = [self.nodes[x] for x in runnable["successors"]] return self._convert_nodes_list(computables, as_type) def _convert_nodes_list(self, nodes, as_type): actual_nodes = [node for node in nodes if node['type'] != 'RUNNABLE_IMPLICIT_RECIPE'] if as_type == "object" or as_type == "objects": return [self._get_object_from_graph_node(node) for node in actual_nodes] else: return nodes def _get_object_from_graph_node(self, node): if node["type"] == "COMPUTABLE_DATASET": return DSSDataset(self.flow.client, self.flow.project.project_key, node["ref"]) elif node["type"] == "RUNNABLE_RECIPE": return DSSRecipe(self.flow.client, self.flow.project.project_key, node["ref"]) elif node["type"] == "COMPUTABLE_FOLDER": return DSSManagedFolder(self.flow.client, self.flow.project.project_key, node["ref"]) elif node["type"] == "COMPUTABLE_SAVED_MODEL": return DSSSavedModel(self.flow.client, self.flow.project.project_key, node["ref"]) elif node["type"] == "COMPUTABLE_STREAMING_ENDPOINT": return DSSStreamingEndpoint(self.flow.client, self.flow.project.project_key, node["ref"]) elif node["type"] == "RUNNABLE_LABELING_TASK": return DSSLabelingTask(self.flow.client, self.flow.project.project_key, node["ref"]) elif node["type"] == "COMPUTABLE_RETRIEVABLE_KNOWLEDGE": return DSSKnowledgeBank(self.flow.client, self.flow.project.project_key, node["ref"]) else: # TODO add streaming elements raise Exception("unsupported node type: %s" % node["type"]) def get_items_in_traversal_order(self, as_type="dict"): """ Get the list of nodes in left to right order. :param str as_type: How to return the nodes. Possible values are "dict" and "object" (defaults to **dict**). :returns: A list of nodes :rtype: If as_type=dict, each item is returned as a dict containing at least "ref" and "type". If as_type=object, each item is returned as a :class:`.DSSDataset`. :class:`.DSSManagedFolder`, :class:`.DSSSavedModel`, :class:`.DSSModelEvaluationStore`, :class:`.DSSStreamingEndpoint` or :class:`.DSSRecipe` """ ret = [] def add_to_set(node): ret.append(node) def in_set(obj): for candidate in ret: if candidate["type"] == obj["type"] and candidate["ref"] == obj["ref"]: return True return False def add_from(graph_node): # To keep traversal order, we recurse to predecessors first for predecessor_ref in graph_node["predecessors"]: predecessor_node = self.nodes[predecessor_ref] if not in_set(predecessor_node): add_from(predecessor_node) # Then add ourselves if not in_set(graph_node): add_to_set(graph_node) # Then recurse to successors for successor_ref in graph_node["successors"]: successor_node = self.nodes[successor_ref] if not in_set(successor_node): add_from(successor_node) for source_computable in self.get_source_computables(): add_from(source_computable) for source_recipe in self.get_source_recipes(): add_from(source_recipe) return self._convert_nodes_list(ret, as_type) class DSSFlowTool(object): """ Handle to interact with a flow tool. """ def __init__(self, client, project_key, tool_id): self.client = client self.project_key = project_key self.tool_id = tool_id def stop(self): """ Stops the tool and releases the resources held by it. """ return self.client._perform_json("POST", "/projects/%s/flow/tools/%s/stop" % (self.project_key, self.tool_id)) def get_state(self, options={}): """ Get the current state of the tool or view. :param dict options: options (defaults to **{}**) :returns: the state :rtype: dict """ return self.client._perform_json("GET", "/projects/%s/flow/tools/%s/state" % (self.project_key, self.tool_id), body=options) def do(self, action): """ Perform a manual user action on the tool. :param dict action: the action to do. It must contain: * a string `name` (the object id) * a string `type` (`"MARK_DATASET_AS_REBUILT"` or `"MARK_RECIPE_AS_OK"`) If `type` is `"MARK_RECIPE_AS_OK"`, a boolean `updated` can be added to the dict, if `True`, marks the recipe as OK after the update. :returns: the current state :rtype: dict """ return self.client._perform_json("PUT", "/projects/%s/flow/tools/%s/action" % (self.project_key, self.tool_id), body=action) def update(self, options={}): """ (for tools only) Start updating the tool state. :params options dict: options for the update (defaults to **{}**) :returns: A handle to interact with task of performing the update :rtype: :class:`.DSSFuture` """ update_future = self.client._perform_json("POST", "/projects/%s/flow/tools/%s/update" % (self.project_key, self.tool_id), body=options) return DSSFuture(self.client, update_future.get('jobId', None), update_future) dataiku-api-client-python-13.2.2/dataikuapi/dss/future.py000066400000000000000000000117161471211575100233660ustar00rootroot00000000000000import time class DSSFuture(object): """ A future represents a long-running task on a DSS instance. It allows you to track the state of the task, retrieve its result when it is ready or abort it. Usage example: .. code-block:: python # In this example, 'folder' is a DSSManagedFolder future = folder.compute_metrics() # Waits until the metrics are computed metrics = future.wait_for_result() .. note:: This class does not need to be instantiated directly. A :class:`dataikuapi.dss.future.DSSFuture` is usually returned by the API calls that are initiating long running-tasks. """ def __init__(self, client, job_id, state=None, result_wrapper=lambda result: result): self.client = client self.job_id = job_id self.state = state self.state_is_peek = True self.result_wrapper = result_wrapper @staticmethod def from_resp(client, resp, result_wrapper=lambda result: result): """ Creates a :class:`dataikuapi.dss.future.DSSFuture` from the response of an endpoint that initiated a long-running task. :param client: An api client to connect to the DSS backend :type client: :class:`dataikuapi.dssclient.DSSClient` :param resp: The response of the API call that initiated a long-running task. :type resp: dict :param result_wrapper: (optional) a function to apply to the result of the future, before it is returned by `get_result` or `wait_for_result` methods. :type result_wrapper: callable :return: :class:`dataikuapi.dss.future.DSSFuture` """ return DSSFuture(client, resp.get('jobId', None), state=resp, result_wrapper=result_wrapper) @classmethod def get_result_wait_if_needed(cls, client, ret): """ :meta private: """ if 'jobId' in ret: future = DSSFuture(client, ret["jobId"], ret) future.wait_for_result() return future.get_result() else: return ret['result'] def abort(self): """ Aborts the long-running task. """ self.client._perform_empty("DELETE", "/futures/%s" % self.job_id) def get_state(self): """ Queries the state of the future, and fetches the result if it's ready. :return: the state of the future, including the result if it is ready. :rtype: dict """ self.state = self.client._perform_json( "GET", "/futures/%s" % self.job_id, params={'peek': False}) self.state_is_peek = False return self.state def peek_state(self): """ Queries the state of the future, without fetching the result. :return: the state of the future :rtype: dict """ self.state = self.client._perform_json( "GET", "/futures/%s" % self.job_id, params={'peek': True}) self.state_is_peek = True return self.state def get_result(self): """ Returns the future's result if it's ready. .. note:: if a custom result_wrapper was provided, it will be applied on the result that will be returned. :return: the result of the future :rtype: object :raises Exception: if the result is not ready (i.e. the task is still running, or it has failed) """ if self.state is None or not self.state.get('hasResult', False) or self.state_is_peek: self.get_state() if self.state.get('hasResult', False): return self.result_wrapper(self.state.get('result', None)) else: raise Exception("Result not ready") def has_result(self): """ Checks whether the task is completed successfully and has a result. :return: True if the task has successfully returned a result, False otherwise :rtype: bool """ if self.state is None or not self.state.get('hasResult', False): self.get_state() return self.state.get('hasResult', False) def wait_for_result(self): """ Waits for the completion of the long-running task, and returns its result. .. note:: if a custom result_wrapper was provided, it will be applied on the result that will be returned. :return: the result of the future :rtype: object :raises Exception: if the task failed """ if self.state is not None and self.state.get('hasResult', False): # no future created in backend, result already in the state return self.result_wrapper(self.state.get('result', None)) if self.state is None or not self.state.get('hasResult', False) or self.state_is_peek: self.get_state() while not self.state.get('hasResult', False): time.sleep(5) self.get_state() if self.state.get('hasResult', False): return self.result_wrapper(self.state.get('result', None)) else: raise Exception("No result") dataiku-api-client-python-13.2.2/dataikuapi/dss/insight.py000066400000000000000000000120211471211575100235070ustar00rootroot00000000000000import json import sys from .utils import DSSTaggableObjectListItem, DSSTaggableObjectSettings if sys.version_info >= (3,0): import urllib.parse dku_quote_fn = urllib.parse.quote else: import urllib dku_quote_fn = urllib.quote INSIGHTS_URI_FORMAT = "/projects/%s/insights/" INSIGHT_URI_FORMAT = INSIGHTS_URI_FORMAT + "%s/" class DSSInsight(object): """ A handle to interact with an insight on the DSS instance. .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.get_insight` instead """ def __init__(self, client, project_key, insight_id): self.client = client self.project_key = project_key self.insight_id = insight_id def delete(self): """ Delete the insight """ return self.client._perform_empty("DELETE", INSIGHT_URI_FORMAT % (self.project_key, self.insight_id)) def get_settings(self): """ Get the insight's definition :return: a handle to inspect the insight definition :rtype: :class:`dataikuapi.dss.insight.DSSInsightSettings` """ return DSSInsightSettings(self.client, self.client._perform_json("GET", INSIGHT_URI_FORMAT % (self.project_key, self.insight_id))) class DSSInsightListItem(DSSTaggableObjectListItem): """ An item in a list of insights. .. important:: Do not instantiate this class, use :meth:`dataikuapi.dss.project.DSSProject.list_insights` """ def __init__(self, client, data): super(DSSInsightListItem, self).__init__(data) self.client = client @property def id(self): """ Get the identifier of the insight .. note:: The id is generated by DSS upon creation and random. :rtype: string """ return self._data["id"] @property def name(self): """ Get the name of the insight .. note:: The name is user-provided and not necessarily unique. :rtype: str """ return self._data['name'] @property def type(self): """ Get the type of the insight (ex: "chart") :rtype: str """ return self._data['type'] @property def listed(self): """ Get the boolean indicating whether the insight is private or public (i.e. promoted) :rtype: bool """ return self._data['listed'] @property def owner(self): """ Get the login of the owner of the insight :rtype: str """ return self._data['owner'] def to_insight(self): """ Get a handle to interact with this insight :return: a handle on the insight :rtype: :class:`dataikuapi.dss.insight.DSSInsight` """ return DSSInsight(self.client, self._data["projectKey"], self._data["id"]) class DSSInsightSettings(DSSTaggableObjectSettings): """ Settings for an insight .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.insight.DSSInsight.get_settings` instead """ def __init__(self, client, settings): super(DSSInsightSettings, self).__init__(settings) self.client = client self.project_key = settings['projectKey'] self.settings = settings def get_raw(self): """ Gets all settings as a raw dictionary. :return: the settings, as a dict. Fields are: * **projectKey** and **id** : identify the insight * **name** : name (label) of the insight * **owner** : login of the owner of the insight * **versionTag**, **creationTag**, **checklists**, **tags**, **customFields** : common fields on DSS objects :rtype: dict """ return self.settings def save(self): """ Save the settings to the insight """ return self.client._perform_json("POST", INSIGHT_URI_FORMAT % (self.project_key, self.id), body = {'insight': self.settings}) @property def id(self): """ Get the identifier of the insight .. note:: The id is generated by DSS upon creation and random. :rtype: string """ return self.settings["id"] @property def name(self): """ Get the name of the insight .. note:: The name is user-provided and not necessarily unique. :rtype: str """ return self.settings['name'] @property def type(self): """ Get the type of the insight (ex: "chart") :rtype: str """ return self.settings['type'] @property def listed(self): """ Get the boolean indicating whether the insight is private or public (i.e. promoted) :rtype: bool """ return self.settings['listed'] @property def owner(self): """ Get the login of the owner of the insight :rtype: str """ return self.settings['owner'] dataiku-api-client-python-13.2.2/dataikuapi/dss/job.py000066400000000000000000000054521471211575100226260ustar00rootroot00000000000000import time import sys from ..utils import DataikuException class DSSJob(object): """ A job on the DSS instance. .. important:: Do not instantiate this class directly, instead use :meth:`dataikuapi.dss.project.DSSProject.get_job` or :meth:`dataikuapi.dss.project.DSSProject.start_job`. """ def __init__(self, client, project_key, id): self.client = client self.id = id self.project_key = project_key def abort(self): """ Aborts the job. :returns: a confirmation message for the request :rtype: dict """ return self.client._perform_json( "POST", "/projects/%s/jobs/%s/abort" % (self.project_key, self.id)) def get_status(self): """ Gets the current status of the job. :returns: the state of the job :rtype: dict """ return self.client._perform_json( "GET", "/projects/%s/jobs/%s/" % (self.project_key, self.id)) def get_log(self, activity=None): """ Gets the logs of the job. If an activity is passed in the parameters the logs will be scoped to that activity. :param string activity: (optional) the name of the activity in the job whose log is requested (defaults to **None**) :returns: the job logs :rtype: string """ return self.client._perform_text( "GET", "/projects/%s/jobs/%s/log" % (self.project_key, self.id), params={ "activity" : activity }) class DSSJobWaiter(object): """ Creates a helper to wait for the completion of a job. :param job: The job to wait for :type job: :class:`dataikuapi.dss.job.DSSJob` """ def __init__(self, job): self.job = job def wait(self, no_fail=False): """ Waits for the job to finish. If the job fails or is aborted, an exception is raised unless the `no_fail` parameter is set to True. :param boolean no_fail: (optional) should an error be raised if the job finished with another status than `DONE` (defaults to **False**) :raises DataikuException: when the job does not complete successfully :returns: the job state :rtype: dict """ job_state = self.job.get_status().get("baseStatus", {}).get("state", "") sleep_time = 2 while job_state not in ["DONE", "ABORTED", "FAILED"]: sleep_time = 60 if sleep_time >= 60 else sleep_time * 1.2 time.sleep(int(sleep_time)) job_state = self.job.get_status().get("baseStatus", {}).get("state", "") if no_fail or (job_state == "DONE"): return job_state raise DataikuException("Job run did not finish. Status: %s" % (job_state)) dataiku-api-client-python-13.2.2/dataikuapi/dss/jupyternotebook.py000066400000000000000000000236731471211575100253240ustar00rootroot00000000000000from .discussion import DSSObjectDiscussions from .utils import DSSTaggableObjectListItem class DSSJupyterNotebookListItem(DSSTaggableObjectListItem): """ An item in a list of Jupyter notebooks. .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.list_jupyter_notebooks` """ def __init__(self, client, data): super(DSSJupyterNotebookListItem, self).__init__(data) self.client = client def to_notebook(self): """ Get a handle on the corresponding notebook :rtype: :class:`DSSJupyterNotebook` """ return DSSJupyterNotebook(self.client, self._data["projectKey"], self._data["name"]) @property def name(self): """ Get the name of the notebook. Used as identifier. """ return self._data["name"] @property def language(self): """ Get the language of the notebook. Possible values are 'python', 'R', 'toree' (scala) :rtype: string """ return self._data["language"] @property def kernel_spec(self): """ Get the raw kernel spec of the notebook. The kernel spec is internal data for Jupyter, that identifies the kernel. :return: the Jupyter spec of a Jupyter kernel. Top-level fields are: * **name** : identifier of the kernel in Jupyter, for example 'python2' or 'python3' * **display_name** : name of the kernel as shown in the Jupyter UI * **language** : language of the notebook (informative field) :rtype: dict """ return self._data["kernelSpec"] class DSSJupyterNotebook(object): """ A handle on a Python/R/scala notebook. .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.get_jupyter_notebook()` or :meth:`dataikuapi.dss.project.DSSProject.create_jupyter_notebook()`. """ def __init__(self, client, project_key, notebook_name): self.client = client self.project_key = project_key self.notebook_name = notebook_name def unload(self, session_id=None): """ Stop this Jupyter notebook and release its resources. """ sessions = self.get_sessions() if sessions is None: raise Exception("Notebook isn't running") if len(sessions) == 0: raise Exception("Notebook isn't running") if session_id is None: if len(sessions) > 1: raise Exception("Several sessions of the notebook are running, choose one") else: session_id = sessions[0].get('sessionId', None) return self.client._perform_json("DELETE", "/projects/%s/jupyter-notebooks/%s/sessions/%s" % (self.project_key, self.notebook_name, session_id)) def get_sessions(self, as_objects=False): """ Get the list of running sessions of this Jupyter notebook. Usage example: .. code-block:: python # list the running notebooks in a project for notebook in project.list_jupyter_notebooks(as_type="object"): if len(notebook.get_sessions()) > 0: print("Notebook %s is running" % notebook.notebook_name) :param boolean as_objects: if True, each returned item will be a :class:`dataikuapi.dss.jupyternotebook.DSSNotebookSession` :return: a list of :class:`dataikuapi.dss.jupyternotebook.DSSNotebookSession` if **as_objects** is True, a list of dict otherwise. The dict holds information about the kernels currently running the notebook, notably a **sessionId** for the Jupyter session id, and a **kernelId** for the Jupyter kernel id. :rtype: list """ sessions = self.client._perform_json("GET", "/projects/%s/jupyter-notebooks/%s/sessions" % (self.project_key, self.notebook_name)) if as_objects: return [DSSNotebookSession(self.client, session) for session in sessions] else: return sessions def get_content(self): """ Get the full contents of this Jupyter notebook. The content comprises metadata, cells, notebook format info. Usage example: .. code-block:: python # collect all the source code of a notebook lines = [] for cell in notebook.get_content().get_cells(): if cell["cell_type"] != 'code': continue lines = lines + cell["source"] print('\\n'.join(lines)) :rtype: :class:`dataikuapi.dss.jupyternotebook.DSSNotebookContent` """ raw_content = self.client._perform_json("GET", "/projects/%s/jupyter-notebooks/%s" % (self.project_key, self.notebook_name)) return DSSNotebookContent(self.client, self.project_key, self.notebook_name, raw_content) def delete(self): """ Delete this Jupyter notebook and stop all of its active sessions. """ return self.client._perform_json("DELETE", "/projects/%s/jupyter-notebooks/%s" % (self.project_key, self.notebook_name)) ######################################################## # Discussions ######################################################## def get_object_discussions(self): """ Get a handle to manage discussions on the notebook. :return: the handle to manage discussions :rtype: :class:`dataikuapi.dss.discussion.DSSObjectDiscussions` """ return DSSObjectDiscussions(self.client, self.project_key, "JUPYTER_NOTEBOOK", self.notebook_name) class DSSNotebookContent(object): """ The content of a Jupyter Notebook. This is the actual notebook data, see the `nbformat doc `_ . .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.jupyternotebook.DSSJupyterNotebook.get_content` """ def __init__(self, client, project_key, notebook_name, content): self.client = client self.project_key = project_key self.notebook_name = notebook_name self.content = content def get_raw(self): """ Get the raw content of this Jupyter notebook. The content comprises metadata, cells, notebook format info. :return: a dict containing the full content of a notebook. Notable fields are: * **metadata** : the metadata of the notebook, as a dict (see :meth:`get_metadata()`) * **nbformat** and **nbformat_minor** : the version of the notebook format * **cells** : list of cells, each one a dict (see :meth:`get_cells()`) :rtype: dict """ return self.content def get_metadata(self): """ Get the metadata associated to this Jupyter notebook. :return: the Jupyter metadata of the notebook, with fields: * **kernelspec** : identification of the kernel used to run the notebook * **creator** : name of the user that created the notebook * **createdOn** : timestamp of creation, in milliseconds * **modifiedBy** : name of last modifier * **language_info** : information on the language of the notebook :rtype: dict """ return self.content["metadata"] def get_cells(self): """ Get the cells of this Jupyter notebook. :return: a list of cells, as defined by Jupyter. Each cell is a dict, with notable fields: * **cell_type** : type of cell, for example 'code' * **metadata** : notebook metadata in the cell * **executionCount** : index of the last run of this cell * **source** : content of the cell, as a list of string * **output** : list of outputs of the last run of the cell, as a list of dict with fields **output_type**, **name** and **text**. Exact contents and meaning depend on the cell type. :rtype: list[dict] """ return self.content["cells"] def save(self): """ Save the content of this Jupyter notebook. """ return self.client._perform_json("PUT", "/projects/%s/jupyter-notebooks/%s" % (self.project_key, self.notebook_name), body=self.content) class DSSNotebookSession(object): """ Metadata associated to the session of a Jupyter Notebook. .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.jupyternotebook.DSSJupyterNotebook.get_sessions()` """ def __init__(self, client, session): self.client = client self.project_key = session.get("projectKey") self.notebook_name = session.get("notebookName") self.session_creator = session.get("sessionCreator") self.session_creator_display_name = session.get("sessionCreatorDisplayName") self.session_unix_owner = session.get("sessionUnixOwner") self.session_id = session.get("sessionId") self.kernel_id = session.get("kernelId") self.kernel_pid = session.get("kernelPid") self.kernel_connections = session.get("kernelConnections") self.kernel_last_activity_time = session.get("kernelLastActivityTime") self.kernel_execution_state = session.get("kernelExecutionState") self.session_start_time = session.get("sessionStartTime") def unload(self): """ Stop this Jupyter notebook and release its resources. """ return self.client._perform_json("DELETE", "/projects/%s/jupyter-notebooks/%s/sessions/%s" % (self.project_key, self.notebook_name, self.session_id)) dataiku-api-client-python-13.2.2/dataikuapi/dss/knowledgebank.py000066400000000000000000000042521471211575100246640ustar00rootroot00000000000000from .utils import DSSTaggableObjectListItem class DSSKnowledgeBankListItem(DSSTaggableObjectListItem): """ An item in a list of knowledege banks .. important:: Do not instantiate this class directly, instead use :meth:`dataikuapi.dss.project.DSSProject.list_knowledge_banks`. """ def __init__(self, client, data): super(DSSKnowledgeBankListItem, self).__init__(data) self.client = client def to_knowledge_bank(self): """ Convert the current item. :returns: A handle for the knowledge_bank. :rtype: :class:`dataikuapi.dss.knowledgebank.DSSKnowledgeBank` """ return DSSKnowledgeBank(self.client, self._data["projectKey"], self._data["id"]) def as_core_knowledge_bank(self): """ Get the :class:`dataiku.KnowledgeBank` object corresponding to this knowledge bank :rtype: :class:`dataiku.KnowledgeBank` """ import dataiku return dataiku.KnowledgeBank("%s.%s" % (self.project_key, self.id)) @property def project_key(self): """ :returns: The project :rtype: string """ return self._data["projectKey"] @property def id(self): """ :returns: The id of the knowledge bank. :rtype: string """ return self._data["id"] @property def name(self): """ :returns: The name of the knowledge bank. :rtype: string """ return self._data["name"] class DSSKnowledgeBank(object): """ A handle to interact with a DSS-managed knowledge bank. .. important:: Do not create this class directly, use :meth:`dataikuapi.dss.project.DSSProject.get_knowledge_bank` instead. """ def __init__(self, client, project_key, id): self.client = client self.project_key = project_key self.id = id def as_core_knowledge_bank(self): """ Get the :class:`dataiku.KnowledgeBank` object corresponding to this knowledge bank :rtype: :class:`dataiku.KnowledgeBank` """ import dataiku return dataiku.KnowledgeBank("%s.%s" % (self.project_key, self.id))dataiku-api-client-python-13.2.2/dataikuapi/dss/labeling_task.py000066400000000000000000000010561471211575100246470ustar00rootroot00000000000000class DSSLabelingTask(object): """ A handle to an existing labeling task on the DSS instance. .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.get_labeling_task()` """ def __init__(self, client, project_key, task_id): self.client = client self.project_key = project_key self.task_id = task_id @property def id(self): """ Returns the internal identifier of the labeling task :rtype: string """ return self.task_id dataiku-api-client-python-13.2.2/dataikuapi/dss/langchain/000077500000000000000000000000001471211575100234205ustar00rootroot00000000000000dataiku-api-client-python-13.2.2/dataikuapi/dss/langchain/__init__.py000066400000000000000000000001141471211575100255250ustar00rootroot00000000000000from .llm import DKULLM, DKUChatModel from .embeddings import DKUEmbeddings dataiku-api-client-python-13.2.2/dataikuapi/dss/langchain/embeddings.py000066400000000000000000000060131471211575100260730ustar00rootroot00000000000000"""Wrapper around Dataiku-mediated embedding LLMs""" import asyncio import concurrent import logging from typing import List, Any from pydantic import BaseModel, Extra from langchain.embeddings.base import Embeddings logger = logging.getLogger(__name__) CHUNK_SIZE = 1000 class DKUEmbeddings(BaseModel, Embeddings): """ Langchain-compatible wrapper around Dataiku-mediated embedding LLMs .. note:: Direct instantiation of this class is possible from within DSS, though it's recommended to instead use :meth:`dataikuapi.dss.llm.DSSLLM.as_langchain_embeddings`. """ llm_id: str """LLM identifier to use""" _llm_handle = None """:class:`dataikuapi.dss.llm.DSSLLM` object to wrap.""" class Config: extra = Extra.forbid underscore_attrs_are_private = True def __init__(self, llm_handle=None, **data: Any): if llm_handle is None: if data.get("llm_id") is None: raise Exception("One of llm_handle or llm_id is required") try: import dataiku except ImportError: raise Exception("llm_handle is required") llm_handle = dataiku.api_client().get_default_project().get_llm(data["llm_id"]) else: data['llm_id'] = llm_handle.llm_id super().__init__(**data) self._llm_handle = llm_handle def embed_documents(self, texts: List[str]) -> List[List[float]]: """Call out to Dataiku-mediated LLM Args: texts: The list of texts to embed. Returns: List of embeddings, one for each text. """ logging.info("Performing embedding of {num_texts} texts".format(num_texts=len(texts))) embeddings = [] for i in range(0, len(texts), CHUNK_SIZE): query = self._llm_handle.new_embeddings(text_overflow_mode="FAIL") for text in texts[i:i+CHUNK_SIZE]: query.add_text(text) resp = query.execute() # TODO #if not resp.success: # raise Exception("LLM call failed: %s" % resp._raw.get("errorMessage", "Unknown error")) embeddings.extend(resp.get_embeddings()) logging.info("Finished a chunk. Embedded {num_embedded} of {num_texts} texts".format( num_embedded=min(i + CHUNK_SIZE, len(texts)), num_texts=len(texts))) logging.info("Done performing embedding of {num_texts} texts".format(num_texts=len(texts))) return embeddings async def aembed_documents(self, texts: List[str]) -> List[List[float]]: loop = asyncio.get_event_loop() with concurrent.futures.ThreadPoolExecutor() as executor: result = await loop.run_in_executor(executor, self.embed_documents, texts) return result def embed_query(self, text: str) -> List[float]: return self.embed_documents([text])[0] async def aembed_query(self, text: str) -> List[float]: embeddings = await self.aembed_documents([text]) return embeddings[0] dataiku-api-client-python-13.2.2/dataikuapi/dss/langchain/llm.py000066400000000000000000000566331471211575100245730ustar00rootroot00000000000000"""Wrapper around Dataiku-mediated LLM""" import json import logging import re from typing import ( Any, Callable, Dict, Iterator, List, Literal, Optional, Sequence, Type, Union, ) from langchain.callbacks.manager import CallbackManagerForLLMRun, AsyncCallbackManagerForLLMRun from langchain.llms.base import BaseLLM from langchain_core.messages import ( AIMessage, AIMessageChunk, BaseMessage, ChatMessage, HumanMessage, InvalidToolCall, SystemMessage, ToolCall, ToolMessage, ) from langchain_core.outputs import Generation, GenerationChunk, ChatGenerationChunk, LLMResult, ChatResult from langchain_core.output_parsers.openai_tools import ( make_invalid_tool_call, parse_tool_call, ) from langchain_core.language_models import BaseChatModel from langchain_core.pydantic_v1 import BaseModel from langchain_core.tools import BaseTool from langchain_core.utils.function_calling import convert_to_openai_tool from langchain.schema import ChatGeneration from pydantic import Extra from dataikuapi.dss.tools.langchain import StopSequencesAwareStreamer logger = logging.getLogger(__name__) def _llm_settings(llm, stop_sequences, tools=None, tool_choice=None): """Returns a settings dict from a DKULLM or DKUChatLLM object""" settings = { "temperature": llm.temperature, "maxOutputTokens": llm.max_tokens, } if llm.top_p is not None: settings["topP"] = llm.top_p if llm.top_k is not None: settings["topK"] = llm.top_k if stop_sequences is not None: settings["stopSequences"] = stop_sequences if tool_choice is not None: settings["toolChoice"] = tool_choice if tools is not None: settings["tools"] = tools return settings def _completion_with_typed_messages(completion, messages): """Add all messages, with their proper type, to a completion object""" for message in messages: if isinstance(message, ChatMessage): completion.with_message(message.content, message.role) elif isinstance(message, HumanMessage): completion.with_message(message.content, "user") elif isinstance(message, AIMessage): raw_tool_calls = message.additional_kwargs.get("tool_calls") if message.tool_calls or message.invalid_tool_calls: # if there is an assistant message, add it first if message.content is not None and len(message.content) > 0: completion.with_message(message.content, "assistant") all_tool_calls = [ _lc_tool_call_to_dku_tool_call(tc) for tc in message.tool_calls ] + [ _lc_invalid_tool_call_to_dku_tool_call(tc) for tc in message.invalid_tool_calls ] completion.with_tool_calls(all_tool_calls, "assistant") # also considering the extra kwargs to replicate the behavior from # the official OpenAI wrapper. See # https://github.com/langchain-ai/langchain/blob/9ef15691d62c1f9f18fe7520cce7dafa82ea517e/libs/partners/openai/langchain_openai/chat_models/base.py#L196-L211 elif raw_tool_calls: # if there is an assistant message, add it first if message.content is not None and len(message.content) > 0: completion.with_message(message.content, "assistant") completion.with_tool_calls(raw_tool_calls, "assistant") else: # no tools, just add the assistant message, as is completion.with_message(message.content, "assistant") elif isinstance(message, SystemMessage): completion.with_message(message.content, "system") elif isinstance(message, ToolMessage): completion.with_tool_output(message.content, message.tool_call_id, "tool") else: raise ValueError(f"Got unknown type {message}") # TODO @langchain: named messages #if "name" in message.additional_kwargs: # message_dict["name"] = message.additional_kwargs["name"] return completion def _enforce_stop_sequences(text: str, stop: List[str]) -> str: """Cut off the text as soon as any stop words occur. Note: Same as the langchain version but with proper regex special character escaping. See: https://github.com/langchain-ai/langchain/blob/2f8dd1a1619f25daa4737df4d378b1acd6ff83c4/libs/community/langchain_community/llms/utils.py#L6 """ if stop is None: return text escaped_stop = [re.escape(s) for s in stop] return re.split("|".join(escaped_stop), text, maxsplit=1)[0] class DKULLM(BaseLLM): """ Langchain-compatible wrapper around Dataiku-mediated LLMs .. note:: Direct instantiation of this class is possible from within DSS, though it's recommended to instead use :meth:`dataikuapi.dss.llm.DSSLLM.as_langchain_llm`. Example: .. code-block:: python llm = dkullm.as_langchain_llm() # single prompt print(llm.invoke("tell me a joke")) # multiple prompts with batching for response in llm.batch(["tell me a joke in English", "tell me a joke in French"]): print(response) # streaming, with stop sequence for chunk in llm.stream("Explain photosynthesis in a few words in English then French", stop=["dioxyde de"]): print(chunk, end="", flush=True) """ llm_id: str """LLM identifier to use""" max_tokens: int = 1024 """Denotes the number of tokens to predict per generation.""" temperature: float = 0 """A non-negative float that tunes the degree of randomness in generation.""" top_k: int = None """Number of tokens to pick from when sampling.""" top_p: float = None """Sample from the top tokens whose probabilities add up to p.""" _llm_handle = None """:class:`dataikuapi.dss.llm.DSSLLM` object to wrap.""" class Config: extra = Extra.forbid underscore_attrs_are_private = True def __init__(self, llm_handle=None, **data: Any): if llm_handle is None: if data.get("llm_id") is None: raise Exception("One of llm_handle or llm_id is required") try: import dataiku except ImportError: raise Exception("llm_handle is required") llm_handle = dataiku.api_client().get_default_project().get_llm(data["llm_id"]) else: data['llm_id'] = llm_handle.llm_id super().__init__(**data) self._llm_handle = llm_handle @property def _llm_type(self): """Return type of llm.""" return "dku" def _generate( self, prompts: List[str], stop: Optional[List[str]] = None, run_manager: Optional[CallbackManagerForLLMRun] = None, **kwargs: Any, ) -> LLMResult: completions = self._llm_handle.new_completions() completions.settings.update(_llm_settings(self, stop)) for prompt in prompts: completion = completions.new_completion() completion.with_message(prompt) logger.info("Executing completion on DKULLM with settings: %s" % completions.settings) resp = completions.execute() total_prompt_tokens = 0 total_completion_tokens = 0 total_total_tokens = 0 token_counts_are_estimated = False total_estimated_cost = 0.0 generations = [] for prompt_resp in resp.responses: if not prompt_resp.success: raise Exception("LLM call failed: %s" % prompt_resp._raw.get("errorMessage", "Unknown error")) total_prompt_tokens += prompt_resp._raw.get("promptTokens", 0) total_completion_tokens += prompt_resp._raw.get("completionTokens", 0) total_total_tokens += prompt_resp._raw.get("totalTokens", 0) token_counts_are_estimated = token_counts_are_estimated or prompt_resp._raw.get("tokenCountsAreEstimated", False) total_estimated_cost += prompt_resp._raw.get("estimatedCost", 0) # Post enforcing them because stopSequences are not supported by all of our connections/models text = _enforce_stop_sequences(prompt_resp.text, stop) generations.append([Generation(text=text)]) llm_output = { 'promptTokens': total_prompt_tokens, 'completionTokens': total_completion_tokens, 'totalTokens': total_total_tokens, 'tokenCountsAreEstimated': token_counts_are_estimated, 'estimatedCost': total_estimated_cost } return LLMResult(generations=generations, llm_output=llm_output) def _stream( self, prompt: str, stop: Optional[List[str]] = None, run_manager: Optional[CallbackManagerForLLMRun] = None, **kwargs: Any, ) -> Iterator[GenerationChunk]: completion = self._llm_handle.new_completion() completion.with_message(prompt) completion.settings.update(_llm_settings(self, stop)) logger.info("Executing streamed completion on DKULLM with settings: %s" % completion.settings) def produce_chunk(chunk: GenerationChunk): if run_manager: run_manager.on_llm_new_token(chunk.text, chunk=chunk) return chunk # manually enforce stop sequences for models that don't support the stopSequences setting streamer = StopSequencesAwareStreamer(stop, _enforce_stop_sequences) for raw_chunk in completion.execute_streamed(): text = raw_chunk.data.get("text", "") new_chunk = GenerationChunk(text=text, generation_info=raw_chunk.data) streamer.append(new_chunk) if streamer.should_stop(): break if streamer.can_yield(): yield streamer.yield_(produce_chunk) # flush any remaining chunk if streamer and streamer.can_yield(): yield streamer.yield_(produce_chunk) class DKUChatModel(BaseChatModel): """ Langchain-compatible wrapper around Dataiku-mediated chat LLMs .. note:: Direct instantiation of this class is possible from within DSS, though it's recommended to instead use :meth:`dataikuapi.dss.llm.DSSLLM.as_langchain_chat_model`. Example: .. code-block:: python from langchain_core.prompts import ChatPromptTemplate llm = dkullm.as_langchain_chat_model() prompt = ChatPromptTemplate.from_template("tell me a joke about {topic}") chain = prompt | llm for chunk in chain.stream({"topic": "parrot"}): print(chunk.content, end="", flush=True) """ llm_id: str """LLM identifier to use""" max_tokens: int = 1024 """Denotes the number of tokens to predict per generation.""" temperature: float = 0 """A non-negative float that tunes the degree of randomness in generation.""" top_k: int = None """Number of tokens to pick from when sampling.""" top_p: float = None """Sample from the top tokens whose probabilities add up to p.""" _llm_handle = None """:class:`dataikuapi.dss.llm.DSSLLM` object to wrap.""" class Config: extra = Extra.forbid underscore_attrs_are_private = True def __init__(self, llm_handle=None, **data: Any): if llm_handle is None: if data.get("llm_id") is None: raise Exception("One of llm_handle or llm_id is required") try: import dataiku except ImportError: raise Exception("llm_handle is required") llm_handle = dataiku.api_client().get_default_project().get_llm(data["llm_id"]) else: data['llm_id'] = llm_handle.llm_id super().__init__(**data) self._llm_handle = llm_handle @property def _llm_type(self): """Return type of chat model.""" return "dku-chat" def _generate( self, messages: List[BaseMessage], stop: Optional[List[str]] = None, run_manager: Optional[CallbackManagerForLLMRun] = None, **kwargs: Any, ) -> ChatResult: tools = kwargs.get("tools", None) tool_choice = kwargs.get("tool_choice", None) logging.debug("DKUChatModel _generate called, messages=%s tools=%s stop=%s" % (messages, tools, stop)) completions = self._llm_handle.new_completions() completions.settings.update(_llm_settings(self, stop, tools, tool_choice)) _completion_with_typed_messages(completions.new_completion(), messages) resp = completions.execute() total_prompt_tokens = 0 total_completion_tokens = 0 total_total_tokens = 0 token_counts_are_estimated = False total_estimated_cost = 0.0 generations = [] for prompt_resp in resp.responses: if not prompt_resp.success: raise Exception("LLM call failed: %s" % prompt_resp._raw.get("errorMessage", "Unknown error")) total_prompt_tokens += prompt_resp._raw.get("promptTokens", 0) total_completion_tokens += prompt_resp._raw.get("completionTokens", 0) total_total_tokens += prompt_resp._raw.get("totalTokens", 0) token_counts_are_estimated = token_counts_are_estimated or prompt_resp._raw.get("tokenCountsAreEstimated", False) total_estimated_cost += prompt_resp._raw.get("estimatedCost", 0) # Post enforcing them because stopSequences are not supported by all of our connections/models # Default to empty string because AIMessage does not accept a None content text = _enforce_stop_sequences(prompt_resp.text, stop) if prompt_resp.text else "" additional_kwargs: Dict = {} tool_calls = [] invalid_tool_calls = [] raw_tool_calls = prompt_resp.tool_calls if raw_tool_calls: # adding the raw tool calls to the extra kwargs to replicate # the behavior from the official OpenAI wrapper. # https://github.com/langchain-ai/langchain/blob/9ef15691d62c1f9f18fe7520cce7dafa82ea517e/libs/partners/openai/langchain_openai/chat_models/base.py#L105-L130 additional_kwargs["tool_calls"] = raw_tool_calls for raw_tool_call in raw_tool_calls: try: tool_calls.append( parse_tool_call(raw_tool_call, return_id=True) ) except Exception as e: invalid_tool_calls.append( make_invalid_tool_call(raw_tool_call, str(e)) ) generations.append( ChatGeneration(message=AIMessage( content=text, additional_kwargs=additional_kwargs, tool_calls=tool_calls, invalid_tool_calls=invalid_tool_calls )) ) llm_output = { 'promptTokens': total_prompt_tokens, 'completionTokens': total_completion_tokens, 'totalTokens': total_total_tokens, 'tokenCountsAreEstimated': token_counts_are_estimated, 'estimatedCost': total_estimated_cost } return ChatResult(generations=generations, llm_output=llm_output) def _stream( self, messages: List[BaseMessage], stop: Optional[List[str]] = None, run_manager: Optional[CallbackManagerForLLMRun] = None, **kwargs: Any, ) -> Iterator[ChatGenerationChunk]: # Due to internals of how Langchain calls this, "tools" and "tool_choice" must not be declared # as params. Else, Langchain ends up putting them both in args and kwargs and leading to failure # when using agent_executor.astream_events tools = kwargs.get("tools", None) tool_choice = kwargs.get("tool_choice", None) logging.debug("DKUChatModel _stream called, messages=%s tools=%s stop=%s" % (messages, tools, stop)) completion = self._llm_handle.new_completion() completion = _completion_with_typed_messages(completion, messages) completion.settings.update(_llm_settings(self, stop, tools, tool_choice)) def produce_chunk(chunk: ChatGenerationChunk): if run_manager: run_manager.on_llm_new_token(chunk.text, chunk=chunk) return chunk # manually enforce stop sequences for models that don't support the stopSequences setting streamer = StopSequencesAwareStreamer(stop, _enforce_stop_sequences) for raw_chunk in completion.execute_streamed(): text = raw_chunk.data.get("text", "") additional_kwargs: Dict = {} tool_call_chunks = [] raw_tool_calls = raw_chunk.data.get("toolCalls") if raw_tool_calls: # adding the raw tool calls to the extra kwargs to replicate # the behavior from the official OpenAI wrapper. cf # https://github.com/langchain-ai/langchain/blob/86ca44d4514b409fed65e2ad8b2ae3c1ee7da48d/libs/partners/openai/langchain_openai/chat_models/base.py#L243 additional_kwargs["tool_calls"] = raw_tool_calls tool_call_chunks = _parse_tool_call_chunks(raw_tool_calls) new_chunk = ChatGenerationChunk( message=AIMessageChunk( content=text, tool_call_chunks=tool_call_chunks, additional_kwargs=additional_kwargs, ), generation_info=raw_chunk.data, ) streamer.append(new_chunk) if streamer.should_stop(): break if streamer.can_yield(): yield streamer.yield_(produce_chunk) # flush any remaining chunk if streamer and streamer.can_yield(): yield streamer.yield_(produce_chunk) def bind_tools( self, tools: Sequence[Union[Dict[str, Any], Type[BaseModel], Callable, BaseTool]], tool_choice: Optional[ Union[dict, str, Literal["auto", "none", "required", "any"], bool] ] = None, **kwargs: Any, ): """ Bind tool-like objects to this chat model. Args: tools: A list of tool definitions to bind to this chat model. Can be a dictionary, pydantic model, callable, or BaseTool. Pydantic models, callables, and BaseTools will be automatically converted to their schema dictionary representation. tool_choice: Which tool to request the model to call. Options are: - name of the tool (str): call the corresponding tool; - "auto": automatically select a tool (or no tool); - "none": do not call a tool; - "any" or "required": force at least one tool call; - True: call the one given tool (requires `tools` to be of length 1); - a dict of the form: {"type": "tool_name", "name": "<>"}, or {"type": "required"}, or {"type": "any"} or {"type": "none"}, or {"type": "auto"}; kwargs: Any additional parameters to bind. """ formatted_tools = [convert_to_openai_tool(tool) for tool in tools] if tool_choice: kwargs["tool_choice"] = _convert_to_llm_mesh_tool_choice(tool_choice, formatted_tools) return super().bind(tools=formatted_tools, **kwargs) def _lc_tool_call_to_dku_tool_call( tool_call: ToolCall ) -> dict: return { "type": "function", "id": tool_call["id"], "function": { "name": tool_call["name"], "arguments": json.dumps(tool_call["args"]), }, } def _lc_invalid_tool_call_to_dku_tool_call( invalid_tool_call: InvalidToolCall, ) -> dict: return { "type": "function", "id": invalid_tool_call["id"], "function": { "name": invalid_tool_call["name"], "arguments": invalid_tool_call["args"], }, } def _parse_tool_call_chunks(raw_tool_calls): try: return [ { "id": rtc.get("id"), "index": rtc.get("index"), "name": rtc["function"].get("name"), "args": rtc["function"].get("arguments"), } for rtc in raw_tool_calls ] except KeyError as e: logger.error("Error when constructing the tool call chunk: ", str(e)) return [] def _convert_to_llm_mesh_tool_choice( tool_choice: Union[dict, str, Literal["auto", "none", "required", "any"], bool], formatted_tools: Sequence[Dict[str, Any]] ) -> Dict[str, Any]: """Map the tool choice to its LLM mesh compatible representation""" tool_names = [ formatted_tool["function"]["name"] for formatted_tool in formatted_tools ] if isinstance(tool_choice, str): # supporting "any" because it seems common in the LangChain vocabulary if tool_choice == "any": tool_choice = "required" if tool_choice in ("auto", "none", "required"): return {"type": tool_choice} # tool_choice refers to a dedicated tool name if tool_choice not in tool_names: raise ValueError( f"Tool choice {tool_choice} was specified, but the only " f"provided tools were {tool_names}." ) return {"type": "tool_name", "name": tool_choice} if isinstance(tool_choice, bool) and tool_choice: if len(tool_names) != 1: raise ValueError( "tool_choice=True can only be used when a single tool is " f"passed in, received {len(tool_names)} tools." ) return { "type": "tool_name", "name": tool_names[0] } if isinstance(tool_choice, dict): expected_tool_choice_types = ["auto", "none", "any", "required", "tool_name"] if "type" not in tool_choice: raise ValueError( f'Tool choice {tool_choice} must contain a "type" property. ' f"Allowed values are {expected_tool_choice_types}." ) if tool_choice["type"] not in expected_tool_choice_types: raise ValueError( f'Invalid tool choice type: {tool_choice["type"]}. ' f"Allowed values are {expected_tool_choice_types}." ) # supporting "any" because it seems common in the LangChain vocabulary # note: we already support it for the 'string' case if tool_choice["type"] == "any": tool_choice["type"] = "required" if tool_choice["type"] == "tool_name": if "name" not in tool_choice: raise ValueError( f'Tool choice {tool_choice} must contain a "name" property.' ) if tool_choice["name"] not in tool_names: raise ValueError( f"Tool choice {tool_choice} was specified, but the only " f"provided tools were {tool_names}." ) return tool_choice # default, unknown case raise ValueError( f"Unrecognized tool_choice type. Expected str, bool or dict. " f"Received: {tool_choice}" ) dataiku-api-client-python-13.2.2/dataikuapi/dss/llm.py000066400000000000000000000542631471211575100226440ustar00rootroot00000000000000from .utils import DSSTaggableObjectListItem import json class DSSLLMListItem(DSSTaggableObjectListItem): """ An item in a list of llms .. important:: Do not instantiate this class directly, instead use :meth:`dataikuapi.dss.project.DSSProject.list_llms`. """ def __init__(self, client, project_key, data): super(DSSLLMListItem, self).__init__(data) self.project_key = project_key self.client = client def to_llm(self): """ Convert the current item. :returns: A handle for the llm. :rtype: :class:`dataikuapi.dss.llm.DSSLLM` """ return DSSLLM(self.client, self.project_key, self._data["id"]) @property def id(self): """ :returns: The id of the llm. :rtype: string """ return self._data["id"] @property def type(self): """ :returns: The type of the LLM :rtype: string """ return self._data["type"] @property def description(self): """ :returns: The description of the LLM :rtype: string """ return self._data["friendlyName"] class DSSLLM(object): """ A handle to interact with a DSS-managed LLM. .. important:: Do not create this class directly, use :meth:`dataikuapi.dss.project.DSSProject.get_llm` instead. """ def __init__(self, client, project_key, llm_id): self.client = client self.project_key = project_key self.llm_id = llm_id def new_completion(self): """ Create a new completion query. :returns: A handle on the generated completion query. :rtype: :class:`DSSLLMCompletionQuery` """ return DSSLLMCompletionQuery(self) def new_completions(self): """ Create a new multi-completion query. :returns: A handle on the generated multi-completion query. :rtype: :class:`DSSLLMCompletionsQuery` """ return DSSLLMCompletionsQuery(self) def new_embeddings(self, text_overflow_mode="FAIL"): """ Create a new embedding query. :param str text_overflow_mode: How to handle longer texts than what the model supports. Either 'TRUNCATE' or 'FAIL'. :returns: A handle on the generated embeddings query. :rtype: :class:`DSSLLMEmbeddingsQuery` """ return DSSLLMEmbeddingsQuery(self, text_overflow_mode) def new_images_generation(self): return DSSLLMImageGenerationQuery(self) def as_langchain_llm(self, **data): """ Create a langchain-compatible LLM object for this LLM. :returns: A langchain-compatible LLM object. :rtype: :class:`dataikuapi.dss.langchain.llm.DKULLM` """ from dataikuapi.dss.langchain.llm import DKULLM return DKULLM(llm_handle=self, **data) def as_langchain_chat_model(self, **data): """ Create a langchain-compatible chat LLM object for this LLM. :returns: A langchain-compatible LLM object. :rtype: :class:`dataikuapi.dss.langchain.llm.DKUChatModel` """ from dataikuapi.dss.langchain.llm import DKUChatModel return DKUChatModel(llm_handle=self, **data) def as_langchain_embeddings(self, **data): """ Create a langchain-compatible embeddings object for this LLM. :returns: A langchain-compatible embeddings object. :rtype: :class:`dataikuapi.dss.langchain.embeddings.DKUEmbeddings` """ from dataikuapi.dss.langchain.embeddings import DKUEmbeddings return DKUEmbeddings(llm_handle=self, **data) class DSSLLMEmbeddingsQuery(object): """ A handle to interact with an embedding query. Embedding queries allow you to transform text into embedding vectors using a DSS-managed model. .. important:: Do not create this class directly, use :meth:`dataikuapi.dss.llm.DSSLLM.new_embeddings` instead. """ def __init__(self, llm, text_overflow_mode): self.llm = llm self.eq = { "queries": [], "llmId": llm.llm_id, "settings": { "textOverflowMode": text_overflow_mode } } def add_text(self, text): """ Add text to the embedding query. :param str text: Text to add to the query. """ self.eq["queries"].append({"text": text}) return self def add_image(self, image): """ Add an image to the embedding query. :param image: Image content as bytes or str (base64) """ if isinstance(image, str): self.eq["queries"].append({"inlineImage": image}) elif isinstance(image, bytes): import base64 self.eq["queries"].append({"inlineImage": base64.b64encode(image).decode("utf8")}) return self def execute(self): """ Run the embedding query. :returns: The results of the embedding query. :rtype: :class:`DSSLLMEmbeddingsResponse` """ ret = self.llm.client._perform_json("POST", "/projects/%s/llms/embeddings" % (self.llm.project_key), body=self.eq) return DSSLLMEmbeddingsResponse(ret) class DSSLLMEmbeddingsResponse(object): """ A handle to interact with an embedding query result. .. important:: Do not create this class directly, use :meth:`dataikuapi.dss.llm.DSSLLMEmbeddingsQuery.execute` instead. """ def __init__(self, raw_resp): self._raw = raw_resp def get_embeddings(self): """ Retrieve vectors resulting from the embeddings query. :returns: A list of lists containing all embedding vectors. :rtype: list """ for r in self._raw["responses"]: if not "embedding" in r: raise Exception("At least one embedding request failed: %s" % r.get("errorMessage", "Unknown error")) return [r["embedding"] for r in self._raw["responses"]] class DSSLLMCompletionsQuerySingleQuery(object): def __init__(self): self.cq = {"messages": []} def new_multipart_message(self, role="user"): """ Start adding a multipart-message to the completion query. Use this to add image parts to the message. :param str role: The message role. Use ``system`` to set the LLM behavior, ``assistant`` to store predefined responses, ``user`` to provide requests or comments for the LLM to answer to. Defaults to ``user``. :rtype: :class:`DSSLLMCompletionQueryMultipartMessage` """ return DSSLLMCompletionQueryMultipartMessage(self, role) def with_message(self, message, role="user"): """ Add a message to the completion query. :param str message: The message text. :param str role: The message role. Use ``system`` to set the LLM behavior, ``assistant`` to store predefined responses, ``user`` to provide requests or comments for the LLM to answer to. Defaults to ``user``. """ role_message = { "role": role, "content": message, } self.cq["messages"].append(role_message) return self def with_tool_calls(self, tool_calls, role="assistant"): """ Add tool calls to the completion query. :param list[dict] tool_calls: Calls to tools that the LLM requested to use. :param str role: The message role. Defaults to ``assistant``. """ role_message = { "role": role, "toolCalls": tool_calls, } self.cq["messages"].append(role_message) return self def with_tool_output(self, tool_output, tool_call_id, role="tool"): """ Add a tool message to the completion query. :param str tool_output: The tool output, as a string. :param str tool_call_id: The tool call id, as provided by the LLM in the conversation messages. :param str role: The message role. Defaults to ``tool``. """ role_message = { "role": role, "toolOutputs": [{ "callId": tool_call_id, "output": tool_output, }], } self.cq["messages"].append(role_message) return self class SettingsMixin(object): def with_json_output(self): """ Request the model to generate a valid JSON response, for models that support it. Note that some models may require you to also explicitly request this in the user or system prompt to use this. """ self._settings["responseFormat"] = { "type": "json" } return self class DSSLLMCompletionQuery(DSSLLMCompletionsQuerySingleQuery, SettingsMixin): """ A handle to interact with a completion query. Completion queries allow you to send a prompt to a DSS-managed LLM and retrieve its response. .. important:: Do not create this class directly, use :meth:`dataikuapi.dss.llm.DSSLLM.new_completion` instead. """ def __init__(self, llm): super().__init__() self.llm = llm self._settings = {} @property def settings(self): """ :return: The completion query settings. :rtype: dict """ return self._settings def execute(self): """ Run the completion query and retrieve the LLM response. :returns: The LLM response. :rtype: :class:`DSSLLMCompletionResponse` """ queries = {"queries": [self.cq], "settings": self._settings, "llmId": self.llm.llm_id} ret = self.llm.client._perform_json("POST", "/projects/%s/llms/completions" % (self.llm.project_key), body=queries) return DSSLLMCompletionResponse(ret["responses"][0]) def execute_streamed(self): """ Run the completion query and retrieve the LLM response as streamed chunks. :returns: An iterator over the LLM response chunks :rtype: Iterator[Union[:class:`DSSLLMStreamedCompletionChunk`, :class:`DSSLLMStreamedCompletionFooter`]] """ request = {"query": self.cq, "settings": self.settings, "llmId": self.llm.llm_id} ret = self.llm.client._perform_raw("POST", "/projects/%s/llms/streamed-completion" % (self.llm.project_key), body=request) sseclient = _SSEClient(ret.raw) for evt in sseclient.iterevents(): if evt.event == "completion-chunk": yield DSSLLMStreamedCompletionChunk(json.loads(evt.data)) else: yield DSSLLMStreamedCompletionFooter(json.loads(evt.data)) class DSSLLMCompletionsQuery(SettingsMixin): """ A handle to interact with a multi-completion query. Completion queries allow you to send a prompt to a DSS-managed LLM and retrieve its response. .. important:: Do not create this class directly, use :meth:`dataikuapi.dss.llm.DSSLLM.new_completion` instead. """ def __init__(self, llm): self.llm = llm self.queries = [] self._settings = {} @property def settings(self): """ :return: The completion query settings. :rtype: dict """ return self._settings def new_completion(self): ret = DSSLLMCompletionsQuerySingleQuery() self.queries.append(ret) return ret def execute(self): """ Run the completions query and retrieve the LLM response. :returns: The LLM response. :rtype: :class:`DSSLLMCompletionsResponse` """ queries = {"queries": [q.cq for q in self.queries], "settings": self._settings, "llmId": self.llm.llm_id} ret = self.llm.client._perform_json("POST", "/projects/%s/llms/completions" % (self.llm.project_key), body=queries) return DSSLLMCompletionsResponse(ret["responses"]) class DSSLLMCompletionQueryMultipartMessage(object): """ .. important:: Do not create this class directly, use :meth:`dataikuapi.dss.llm.DSSLLMCompletionQuery.new_multipart_message` or :meth:`dataikuapi.dss.llm.DSSLLMCompletionsQuerySingleQuery.new_multipart_message`. """ def __init__(self, q, role): self.q = q self.msg = {"role": role, "parts" : []} def with_text(self, text): """ Add a text part to the multipart message """ self.msg["parts"].append({"type": "TEXT", "text": text}) return self def with_inline_image(self, image, mime_type=None): """ Add an image part to the multipart message :param image: bytes or str (base64) :param mime_type str: None for default """ img_b64 = None if isinstance(image, str): img_b64 = image elif isinstance(image, bytes): import base64 img_b64 = base64.b64encode(image).decode("utf8") part = { "type": "IMAGE_INLINE", "inlineImage": img_b64 } if mime_type is not None: part["imageMimeType"] = mime_type self.msg["parts"].append(part) return self def add(self): """Add this message to the completion query""" self.q.cq["messages"].append(self.msg) return self.q class DSSLLMStreamedCompletionChunk(object): def __init__(self, data): self.data = data def __repr__(self): return "" % self.data class DSSLLMStreamedCompletionFooter(object): def __init__(self, data): self.data = data def __repr__(self): return "" % self.data class _SSEEvent(object): def __init__(self, id=None, event=None, data=""): self.id = id self.event = event self.data = data class _SSEClient(object): def __init__(self, raw_source): self.raw_source = raw_source def _read(self): """Reads the raw source and yields events. Reassembles events that may span multiple HTTP chunks""" data = b'' for chunk in self.raw_source: for line in chunk.splitlines(True): data += line if data.endswith(b'\r\r') or data.endswith(b'\n\n') or data.endswith(b'\r\n\r\n'): yield data data = b'' if data: yield data def iterevents(self): for event_chunk in self._read(): evt = _SSEEvent() for line in event_chunk.splitlines(): line = line.decode("utf8") # Start with : --> comment if line.startswith(":"): continue data = line.split(":", 1) field = data[0] if len(data) > 1: value = data[1].strip() else: value = '' if field == 'data': evt.__dict__[field] += value + '\n' else: evt.__dict__[field] = value #print("Yielding event: %s" % evt.__dict__) yield evt class DSSLLMCompletionResponse(object): """ A handle to interact with a completion response. .. important:: Do not create this class directly, use :meth:`dataikuapi.dss.llm.DSSLLMCompletionQuery.execute` instead. """ def __init__(self, raw_resp): self._raw = raw_resp self._json = None @property def json(self): """ :return: LLM response parsed as a JSON object """ if not self.success: error_message = self._raw.get("errorMessage", "An unknown error occurred") raise Exception(error_message) if self._json is None: self._json = json.loads(self._raw["text"]) return self._json @property def success(self): """ :return: The outcome of the completion query. :rtype: bool """ return self._raw["ok"] @property def text(self): """ :return: The raw text of the LLM response. :rtype: Union[str, None] """ return self._raw.get("text") @property def tool_calls(self): """ :return: The tool calls of the LLM response. :rtype: Union[list, None] """ return self._raw.get("toolCalls") @property def log_probs(self): """ :return: The log probs of the LLM response. :rtype: Union[list, None] """ return self._raw.get("logProbs") class DSSLLMCompletionsResponse(object): """ A handle to interact with a multi-completion response. .. important:: Do not create this class directly, use :meth:`dataikuapi.dss.llm.DSSLLMCompletionsQuery.execute` instead. """ def __init__(self, raw_resp): self._raw = raw_resp @property def responses(self): """The array of responses""" return [DSSLLMCompletionResponse(x) for x in self._raw] class DSSLLMImageGenerationQuery(object): """ A handle to interact with an image generation query. .. important:: Do not create this class directly, use :meth:`dataikuapi.dss.llm.DSSLLM.new_images_generation` instead. """ def __init__(self, llm): self.llm = llm self.gq = { "prompts": [], "negativePrompts": [], "llmId": self.llm.llm_id } def with_prompt(self, prompt, weight=None): """ """ self.gq["prompts"].append({"prompt": prompt, "weight": weight}) return self def with_negative_prompt(self, prompt, weight=None): """ """ self.gq["negativePrompts"].append({"prompt": prompt, "weight": weight}) return self def with_original_image(self, image, mode=None, weight=None): if isinstance(image, str): self.gq["originalImage"] = image elif isinstance(image, bytes): import base64 self.gq["originalImage"] = base64.b64encode(image).decode("utf8") if mode is not None: self.gq["originalImageEditionMode"] = mode if weight is not None: self.gq["originalImageWeight"] = weight return self def with_mask(self, mode, image=None, text=None): self.gq["maskMode"] = mode if image is not None: if isinstance(image, str): self.gq["maskImage"] = image elif isinstance(image, bytes): import base64 self.gq["maskImage"] = base64.b64encode(image).decode("utf8") return self @property def inference_steps(self): return self.gq.get("nbInferenceSteps", None) @inference_steps.setter def inference_steps(self, new_value): self.gq["nbInferenceSteps"] = new_value @property def refiner_strength(self): return self.gq.get("refinerStrength", None) @refiner_strength.setter def refiner_strength(self, new_value): self.gq["refinerStrength"] = new_value @property def height(self): return self.gq.get("height", None) @height.setter def height(self, new_value): self.gq["height"] = new_value @property def width(self): return self.gq.get("width", None) @width.setter def width(self, new_value): self.gq["width"] = new_value @property def fidelity(self): return self.gq.get("fidelity", None) @fidelity.setter def fidelity(self, new_value): self.gq["fidelity"] = new_value @property def quality(self): return self.gq.get("quality", None) @quality.setter def quality(self, new_value): self.gq["quality"] = new_value @property def seed(self): return self.gq.get("seed", None) @seed.setter def seed(self, new_value): self.gq["seed"] = new_value @property def style(self): """Style of the image to generate. Valid values depend on the targeted model""" return self.gq.get("style", None) @style.setter def style(self, new_value): self.gq["style"] = new_value @property def images_to_generate(self): return self.gq.get("nbImagesToGenerate", None) @images_to_generate.setter def images_to_generate(self, new_value): self.gq["nbImagesToGenerate"] = new_value def with_aspect_ratio(self, ar): self.gq["height"] = 1024 self.gq["width"] = int(1024 * ar) return self def execute(self): """ Executes the image generation :rtype: :class:DSSLLMImageGenerationResponse """ ret = self.llm.client._perform_json("POST", "/projects/%s/llms/images" % (self.llm.project_key), body=self.gq) return DSSLLMImageGenerationResponse(ret) class DSSLLMImageGenerationResponse(object): """ A handle to interact with an image generation response. .. important:: Do not create this class directly, use :meth:`dataikuapi.dss.llm.DSSLLMImageGenerationQuery.execute` instead. """ def __init__(self, raw_resp): self._raw = raw_resp @property def success(self): """ :return: The outcome of the image generation query. :rtype: bool """ return self._raw["ok"] def first_image(self, as_type="bytes"): """ :return: The first generated image. :rtype: str """ if not self.success: raise Exception("Image generation did not succeed: %s" % self._raw["errorMessage"]) if len(self._raw["images"]) == 0: raise Exception("Image generation succeeded but did not return any image") if as_type == "bytes": import base64 return base64.b64decode(self._raw["images"][0]["data"]) else: return self._raw["images"][0]["data"] def get_images(self, as_type="bytes"): """ :return: The generated images. :rtype: List[str] """ if not self.success: raise Exception("Image generation did not succeed: %s" % self._raw["errorMessage"]) if len(self._raw["images"]) == 0: raise Exception("Image generation succeeded but did not return any image") if as_type == "bytes": import base64 return [base64.b64decode(image["data"]) for image in self._raw["images"]] else: return [image["data"] for image in self._raw["images"]] dataiku-api-client-python-13.2.2/dataikuapi/dss/macro.py000066400000000000000000000115011471211575100231450ustar00rootroot00000000000000import time import sys, json from ..utils import DataikuException class DSSMacro(object): """ A macro on the DSS instance. .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.get_macro()` """ def __init__(self, client, project_key, runnable_type, definition=None): self.client = client self.runnable_type = runnable_type self.project_key = project_key self.definition = definition def get_definition(self): """ Get the macro definition. .. note:: The **adminParams** field is empty unless the authentication of the API client covers admin rights. :return: the definition (read-only), as a dict. The fields mimic the contents of the `runnable.json` file of the macro. :rtype: dict """ if self.definition is None: self.definition = self.client._perform_json( "GET", "/projects/%s/runnables/%s" % (self.project_key, self.runnable_type)) return self.definition def run(self, params=None, admin_params=None, wait=True): """ Run the macro from the project .. note:: If the authentication of the api client does not have admin rights, admin params are ignored. Usage example: .. code-block:: python # list all datasets on a connection. connection_name = 'filesystem_managed' macro = project.get_macro('pyrunnable_builtin-macros_list-datasets-using-connection') run_id = macro.run(params={'connection': connection_name}, wait=True) # the result of this builtin macro is of type RESULT_TABLE result = macro.get_result(run_id, as_type="json") for record in result["records"]: print("Used by %s" % record[0]) :param dict params: parameters to the macro run (defaults to `{}`) :param dict admin_params: admin parameters to the macro run (defaults to `{}`) :param boolean wait: if True, the call blocks until the run is finished :return: a run identifier to use with :meth:`abort()`, :meth:`get_status()` and :meth:`get_result()` :rtype: string """ if params is None: params = {} if admin_params is None: admin_params = {} return self.client._perform_json( "POST", "/projects/%s/runnables/%s" % (self.project_key, self.runnable_type), params={'wait':wait}, body={'params':params, 'adminParams':admin_params})['runId'] def abort(self, run_id): """ Abort a run of the macro. :param string run_id: a run identifier, as returned by :meth:`run()` """ self.client._perform_empty( "POST", "/projects/%s/runnables/%s/abort/%s" % (self.project_key, self.runnable_type, run_id)) def get_status(self, run_id): """ Poll the status of a run of the macro. .. note:: Once the run is done, when :meth:`get_result()` is called, the run ceases to exist. Afterwards :meth:`get_status()` will answer that the run doesn't exist. :param string run_id: a run identifier, as returned by :meth:`run()` :return: the status, as a dict. Whether the run is still ongoing can be assessed with the **running** field. :rtype: dict """ return self.client._perform_json( "GET", "/projects/%s/runnables/%s/state/%s" % (self.project_key, self.runnable_type, run_id)) def get_result(self, run_id, as_type=None): """ Retrieve the result of a run of the macro. .. note:: If the macro is still running, an Exception is raised. The type of the contents of the result to expect can be checked using :meth:`get_definition()`, in particular the "resultType" field. :param string run_id: a run identifier, as returned by :meth:`run()` method :param string as_type: if not None, one of 'string' or 'json'. Use 'json' when the type of result advertised by the macro is RESULT_TABLE or JSON_OBJECT. :return: the contents of the result of the macro run, as a file-like is **as_type** is None; as a str if **as_type** is "string"; as an object if **as_type** is "json". :rtype: file-like, or string """ resp = self.client._perform_raw( "GET", "/projects/%s/runnables/%s/result/%s" % (self.project_key, self.runnable_type, run_id)) if as_type == 'string': with resp.raw as s: return s.read() elif as_type == 'json': with resp.raw as s: return json.load(s) else: return resp.raw dataiku-api-client-python-13.2.2/dataikuapi/dss/managedfolder.py000066400000000000000000000545001471211575100246420ustar00rootroot00000000000000from ..utils import DataikuException from ..utils import DataikuUTF8CSVReader from ..utils import DataikuStreamedHttpUTF8CSVReader from .utils import DSSTaggableObjectListItem, DSSTaggableObjectSettings import json import os from requests import utils from .metrics import ComputedMetrics from .future import DSSFuture from .discussion import DSSObjectDiscussions from .dataset import DSSDataset try: basestring except NameError: basestring = str class DSSManagedFolder(object): """ A handle to interact with a managed folder on the DSS instance. .. important:: Do not create this class directly, instead use :meth:`dataikuapi.dss.project.get_managed_folder` """ def __init__(self, client, project_key, odb_id): self.client = client self.project = client.get_project(project_key) self.project_key = project_key self.odb_id = odb_id @property def id(self): """ Returns the internal identifier of the managed folder, which is a 8-character random string, not to be confused with the managed folder's name. :rtype: string """ return self.odb_id ######################################################## # Managed folder deletion ######################################################## def delete(self): """ Delete the managed folder from the flow, and objects using it (recipes or labeling tasks) .. attention:: This call doesn't delete the managed folder's contents """ return self.client._perform_empty( "DELETE", "/projects/%s/managedfolders/%s" % (self.project_key, self.odb_id)) ######################################################## # Managed folder definition ######################################################## def get_definition(self): """ Get the definition of this managed folder. The definition contains name, description checklists, tags, connection and path parameters, metrics and checks setup. .. caution:: Deprecated. Please use :meth:`get_settings` :returns: the managed folder definition. :rtype: dict """ return self.client._perform_json( "GET", "/projects/%s/managedfolders/%s" % (self.project_key, self.odb_id)) def set_definition(self, definition): """ Set the definition of this managed folder. .. caution:: Deprecated. Please use :meth:`get_settings` then :meth:`~DSSManagedFolderSettings.save()` .. note:: the fields `id` and `projectKey` can't be modified Usage example: .. code-block:: python folder_definition = folder.get_definition() folder_definition['tags'] = ['tag1','tag2'] folder.set_definition(folder_definition) :param dict definition: the new state of the definition for the folder. You should only set a definition object that has been retrieved using the :meth:`get_definition` call :returns: a message upon successful completion of the definition update. Only contains one `msg` field :rtype: dict """ return self.client._perform_json( "PUT", "/projects/%s/managedfolders/%s" % (self.project_key, self.odb_id), body=definition) def get_settings(self): """ Returns the settings of this managed folder as a :class:`DSSManagedFolderSettings`. You must use :meth:`~DSSManagedFolderSettings.save()` on the returned object to make your changes effective on the managed folder. .. code-block:: python # Example: activating discrete partitioning folder = project.get_managed_folder("my_folder_id") settings = folder.get_settings() settings.add_discrete_partitioning_dimension("country") settings.save() :returns: the settings of the managed folder :rtype: :class:`DSSManagedFolderSettings` """ data = self.client._perform_json("GET", "/projects/%s/managedfolders/%s" % (self.project_key, self.odb_id)) return DSSManagedFolderSettings(self, data) ######################################################## # Managed folder contents ######################################################## def list_contents(self): """ Get the list of files in the managed folder Usage example: .. code-block:: python for content in folder.list_contents()['items']: last_modified_seconds = content["lastModified"] / 1000 last_modified_str = datetime.fromtimestamp(last_modified_seconds).strftime("%Y-%m-%d %H:%m:%S") print("size=%s mtime=%s %s" % (content["size"], last_modified_str, content["path"])) :returns: the list of files, in the `items` field. Each item has fields: * **path** : path of the file inside the folder * **size** : size of the file in bytes * **lastModified** : last modification time, in milliseconds since epoch :rtype: dict """ return self.client._perform_json( "GET", "/projects/%s/managedfolders/%s/contents" % (self.project_key, self.odb_id)) def get_file(self, path): """ Get a file from the managed folder Usage example: .. code-block:: python with folder.get_file("/kaggle_titanic_train.csv") as fd: df = pandas.read_csv(fd.raw) :param string path: the path of the file to read within the folder :returns: the HTTP request to stream the data from :rtype: :class:`requests.models.Response` """ return self.client._perform_raw( "GET", "/projects/%s/managedfolders/%s/contents/%s" % (self.project_key, self.odb_id, utils.quote(path))) def delete_file(self, path): """ Delete a file from the managed folder :param string path: the path of the file to read within the folder .. note:: No error is raised if the file doesn't exist """ return self.client._perform_empty( "DELETE", "/projects/%s/managedfolders/%s/contents/%s" % (self.project_key, self.odb_id, utils.quote(path))) def put_file(self, path, f): """ Upload the file to the managed folder. If the file already exists in the folder, it is overwritten. Usage example: .. code-block:: python with open("./some_local.csv") as fd: uploaded = folder.put_file("/target.csv", fd).json() print("Uploaded %s bytes" % uploaded["size"]) :param string path: the path of the file to write within the folder :param file f: a file-like .. note:: if using a string for the `f` parameter, the string itself is taken as the file content to upload :returns: information on the file uploaded to the folder, as a dict of: * **path** : path of the file inside the folder * **size** : size of the file in bytes * **lastModified** : last modification time, in milliseconds since epoch :rtype: dict """ return self.client._perform_json_upload( "POST", "/projects/%s/managedfolders/%s/contents/%s" % (self.project_key, self.odb_id, utils.quote(path)), "", f).json() def upload_folder(self, path, folder): """ Upload the content of a folder to a managed folder. .. note:: `upload_folder("/some/target", "./a/source/")` will result in "target" containing the contents of "source", but not the "source" folder being a child of "target" :param str path: the destination path of the folder in the managed folder :param str folder: local path (absolute or relative) of the source folder to upload """ for root, _, files in os.walk(folder): for file in files: filename = os.path.join(root, file) with open(filename, "rb") as f: rel_posix_path = "/".join(os.path.relpath(filename, folder).split(os.sep)) self.put_file("{}/{}".format(path, rel_posix_path), f) ######################################################## # Managed folder actions ######################################################## def compute_metrics(self, metric_ids=None, probes=None): """ Compute metrics on this managed folder. Usage example: .. code-block:: python future_resp = folder.compute_metrics() future = DSSFuture(client, future_resp.get("jobId", None), future_resp) metrics = future.wait_for_result() print("Computed in %s ms" % (metrics["endTime"] - metrics["startTime"])) for computed in metrics["computed"]: print("Metric %s = %s" % (computed["metricId"], computed["value"])) :param metric_ids: (optional) identifiers of metrics to compute, among the metrics defined on the folder :type metric_ids: list[string] :param probes: (optional) definition of metrics probes to use, in place of the ones defined on the folder. The current set of probes on the folder is the `probes` field in the dict returned by :meth:`get_definition` :type probes: dict :returns: a future as dict representing the task of computing the probes :rtype: dict """ url = "/projects/%s/managedfolders/%s/actions" % (self.project_key, self.odb_id) if metric_ids is not None: return self.client._perform_json( "POST" , "%s/computeMetricsFromIds" % url, body={"metricIds" : metric_ids}) elif probes is not None: return self.client._perform_json( "POST" , "%s/computeMetrics" % url, body=probes) else: return self.client._perform_json( "POST" , "%s/computeMetrics" % url) ######################################################## # Metrics ######################################################## def get_last_metric_values(self): """ Get the last values of the metrics on this managed folder. :returns: a handle on the values of the metrics :rtype: :class:`dataikuapi.dss.metrics.ComputedMetrics` """ return ComputedMetrics(self.client._perform_json( "GET", "/projects/%s/managedfolders/%s/metrics/last" % (self.project_key, self.odb_id))) def get_metric_history(self, metric): """ Get the history of the values of a metric on this managed folder. Usage example: .. code-block:: python history = folder.get_metric_history("basic:COUNT_FILES") for value in history["values"]: time_str = datetime.fromtimestamp(value["time"] / 1000).strftime("%Y-%m-%d %H:%m:%S") print("%s : %s" % (time_str, value["value"])) :param string metric: identifier of the metric to get values of :returns: an object containing the values of the metric, cast to the appropriate type (double, boolean,...). The identifier of the metric is in a **metricId** field. :rtype: dict """ return self.client._perform_json( "GET", "/projects/%s/managedfolders/%s/metrics/history" % (self.project_key, self.odb_id), params={'metricLookup' : metric if isinstance(metric, str) or isinstance(metric, unicode) else json.dumps(metric)}) ######################################################## # Misc ######################################################## def get_zone(self): """ Get the flow zone of this managed folder. :returns: a flow zone :rtype: :class:`dataikuapi.dss.flow.DSSFlowZone` """ return self.project.get_flow().get_zone_of_object(self) def move_to_zone(self, zone): """ Move this object to a flow zone. :param object zone: a :class:`dataikuapi.dss.flow.DSSFlowZone` where to move the object, or its identifier """ if isinstance(zone, basestring): zone = self.project.get_flow().get_zone(zone) zone.add_item(self) def share_to_zone(self, zone): """ Share this object to a flow zone. :param object zone: a :class:`dataikuapi.dss.flow.DSSFlowZone` where to share the object, or its identifier """ if isinstance(zone, basestring): zone = self.project.get_flow().get_zone(zone) zone.add_shared(self) def unshare_from_zone(self, zone): """ Unshare this object from a flow zone. :param object zone: a :class:`dataikuapi.dss.flow.DSSFlowZone` from where to unshare the object, or its identifier """ if isinstance(zone, basestring): zone = self.project.get_flow().get_zone(zone) zone.remove_shared(self) def get_usages(self): """ Get the recipes referencing this folder. Usage example: .. code-block:: python for usage in folder.get_usages(): if usage["type"] == 'RECIPE_INPUT': print("Used as input of %s" % usage["objectId"]) :returns: a list of usages, each one a dict of: * **type** : the type of usage, either "RECIPE_INPUT" or "RECIPE_OUTPUT" * **objectId** : name of the recipe * **objectProjectKey** : project of the recipe :rtype: list[dict] """ return self.client._perform_json("GET", "/projects/%s/managedfolders/%s/usages" % (self.project_key, self.odb_id)) def get_object_discussions(self): """ Get a handle to manage discussions on the managed folder. :returns: the handle to manage discussions :rtype: :class:`dataikuapi.dss.discussion.DSSObjectDiscussions` """ return DSSObjectDiscussions(self.client, self.project_key, "MANAGED_FOLDER", self.odb_id) ######################################################## # utilities ######################################################## def copy_to(self, target, write_mode="OVERWRITE"): """ Copy the data of this folder to another folder. :param object target: a :class:`dataikuapi.dss.managedfolder.DSSManagedFolder` representing the target location of this copy :returns: a DSSFuture representing the operation :rtype: :class:`dataikuapi.dss.future.DSSFuture` """ dqr = { "targetProjectKey" : target.project_key, "targetFolderId": target.odb_id, "writeMode" : write_mode } future_resp = self.client._perform_json("POST", "/projects/%s/managedfolders/%s/actions/copyTo" % (self.project_key, self.odb_id), body=dqr) return DSSFuture(self.client, future_resp.get("jobId", None), future_resp) def create_dataset_from_files(self, dataset_name): """ Create a new dataset of type 'FilesInFolder', taking its files from this managed folder, and return a handle to interact with it. The created dataset does not have its format and schema initialized, it is recommended to use :meth:`~dataikuapi.dss.dataset.DSSDataset.autodetect_settings` on the returned object :param str dataset_name: the name of the dataset to create. Must not already exist :returns: A dataset handle :rtype: :class:`dataikuapi.dss.dataset.DSSDataset` """ obj = { "name": dataset_name, "projectKey": self.project_key, "type": "FilesInFolder", "params": { "folderSmartId": self.odb_id } } self.client._perform_json("POST", "/projects/%s/datasets/" % self.project_key, body=obj) return DSSDataset(self.client, self.project_key, dataset_name) class DSSManagedFolderSettings(DSSTaggableObjectSettings): """ Base settings class for a DSS managed folder. Do not instantiate this class directly, use :meth:`DSSDSSManagedFolderSettings.get_settings` Use :meth:`save` to save your changes """ def __init__(self, folder, settings): super(DSSManagedFolderSettings, self).__init__(settings) self.folder = folder self.settings = settings def get_raw(self): """ Get the managed folder settings. :returns: the settings, as a dict. The definition of the actual location of the files in the managed folder is a **params** sub-dict. :rtype: dict """ return self.settings def get_raw_params(self): """ Get the type-specific (S3/ filesystem/ HDFS/ ...) params as a dict. :returns: the type-specific patams. Each type defines a set of fields; commonly found fields are : * **connection** : name of the connection used by the managed folder * **path** : root of the managed folder within the connection * **bucket** or **container** : the bucket/container name on cloud storages :rtype: dict """ return self.settings["params"] @property def type(self): """ Get the type of filesystem that the managed folder uses. :rtype: string """ return self.settings["type"] def save(self): """ Save the changes to the settings on the managed folder. Usage example: .. code-block:: python folder = project.get_managed_folder("my_folder_id") settings = folder.get_settings() settings.set_connection_and_path("some_S3_connection", None) settings.get_raw_params()["bucket"] = "some_S3_bucket" settings.save() """ self.folder.client._perform_empty( "PUT", "/projects/%s/managedfolders/%s" % (self.folder.project_key, self.folder.odb_id), body=self.settings) ######################################################## # Partitioning ######################################################## def remove_partitioning(self): """ Make the managed folder non-partitioned. """ self.settings["partitioning"] = {"dimensions" : []} def add_discrete_partitioning_dimension(self, dim_name): """ Add a discrete partitioning dimension. :param string dim_name: name of the partitioning dimension """ self.settings["partitioning"]["dimensions"].append({"name": dim_name, "type": "value"}) def add_time_partitioning_dimension(self, dim_name, period="DAY"): """ Add a time partitioning dimension. :param string dim_name: name of the partitioning dimension :param string period: granularity of the partitioning dimension (YEAR, MONTH, DAY (default), HOUR) """ self.settings["partitioning"]["dimensions"].append({"name": dim_name, "type": "time", "params":{"period": period}}) def set_partitioning_file_pattern(self, pattern): """ Set the partitioning pattern of the folder. The pattern indicates which paths inside the folder belong to which partition. Partition dimensions are written with: * `%{dim_name}` for discrete dimensions * `%Y` (=year), `%M` (=month), `%D` (=day) and `%H` (=hour) for time dimensions Besides the `%...` variables for injecting the partition dimensions, the pattern is a regular expression. Usage example: .. code-block:: python # partition a managed folder by month folder = project.get_managed_folder("my_folder_id") settings = folder.get_settings() settings.add_time_partitioning_dimension("my_date", "MONTH") settings.set_partitioning_file_pattern("/year=%Y/month=%M/.*") settings.save() :param string pattern: the partitioning pattern """ self.settings["partitioning"]["filePathPattern"] = pattern ######################################################## # Basic ######################################################## def set_connection_and_path(self, connection, path): """ Change the managed folder connection and/or path. .. note:: When changing the connection or path, the folder's files aren't moved or copied to the new location .. attention:: When changing the connection for a connection with a different type, for example going from a S3 connection to an Azure Blob Storage connection, only the managed folder type is changed. Type-specific fields are not converted. In the example of a S3 to Azure conversion, the S3 bucket isn't converted to a storage account container. :param string connection: the name of a file-based connection. If `None`, the connection of the managed folder is left unchanged :param string path: a path relative to the connection root. If `None`, the path of the managed folder is left unchanged """ if connection is not None: if connection != self.settings["params"]["connection"]: # get the actual connection type (and check that it exists) connection_info = self.folder.client.get_connection(connection).get_info(self.folder.project_key) connection_type = connection_info["type"] if connection_type == 'EC2': self.settings["type"] = 'S3' # the fsprovider type is different elif connection_type == 'SSH': # can be SCP or SFTP, default to SCP if connection type changed if self.settings["type"] not in ['SCP', 'SFTP']: self.settings["type"] = 'SCP' else: self.settings["type"] = connection_type self.settings["params"]["connection"] = connection if path is not None: self.settings["params"]["path"] = path dataiku-api-client-python-13.2.2/dataikuapi/dss/meaning.py000066400000000000000000000022471471211575100234710ustar00rootroot00000000000000 class DSSMeaning(object): """ A user-defined meaning on the DSS instance """ def __init__(self, client, id): self.client = client self.id = id ######################################################## # Meaning definition ######################################################## def get_definition(self): """ Get the meaning's definition. .. attention:: This call requires an API key with admin rights :returns: the meaning definition. The precise structure of the dict depends on the meaning type and is not documented. :rtype: dict """ return self.client._perform_json("GET", "/meanings/%s" % self.id) def set_definition(self, definition): """ Set the meaning's definition. .. attention:: This call requires an API key with admin rights :param dict definition: the definition for the meaning, as a dict. You should only ``set_definition`` on a modified version of a dict you retrieved via :py:meth:`get_definition` """ return self.client._perform_json("PUT", "/meanings/%s" % self.id, body=definition) dataiku-api-client-python-13.2.2/dataikuapi/dss/messaging_channel.py000066400000000000000000000132521471211575100255160ustar00rootroot00000000000000import json class DSSMessagingChannelListItem(object): """ A generic messaging channel in DSS. .. important:: Do not instantiate this class, use :meth:`dataikuapi.DSSClient.list_messaging_channels` """ def __init__(self, client, data): self.client = client self._data = data if data is None: self._data = dict() else: self._data = data self._id = data.get("id", None) self._type = data.get("type", None) self._family = data.get("family", None) @property def id(self): """ ID of the messaging channel :type: str """ return self._id @property def type(self): """ Type of the messaging channel :type: str """ return self._type @property def family(self): """ Family of the messaging channel where relevant - e.g. "mail" :type: str """ return self._family def get_raw(self): """ :return: Gets the raw representation of this :class:`DSSMessagingChannelListItem`, any edit is reflected in the object. :rtype: dict """ return self._data def __repr__(self): return self.__class__.__name__ + "(" + self._data.__repr__() + ")" def get_as_messaging_channel(self): """ :return: The same messaging channel but as the appropriate object type :rtype: DSSMessagingChannel """ if self._family == 'mail': return DSSMailMessagingChannel(self, self._data) else: return DSSMessagingChannel(self, self._data) class DSSMessagingChannel(object): """ A handle to interact with a messaging channel on the DSS instance. A generic DSS messaging channel .. important:: Do not instantiate this class directly, use :meth:`dataikuapi.DSSClient.get_messaging_channel` """ def __init__(self, client, data=None): self.client = client if data is None: self._data = dict() else: self._data = data self._id = data.get("id", None) self._type = data.get("type", None) self._family = data.get("family", None) @property def id(self): """ ID of the messaging channel :rtype: str """ return self._id @property def type(self): """ Type of the messaging channel :rtype: str """ return self._type @property def family(self): """ Family of the messaging channel where relevant - e.g. "mail" :type: str """ return self._family class DSSMailMessagingChannel(DSSMessagingChannel): """ A handle to interact with an email messaging channel on the DSS instance - a subclass of :class:`DSSMessagingChannel` .. important:: Do not instantiate this class directly, use :meth:`dataikuapi.DSSClient.get_messaging_channel` """ def __init__(self, client, data): super().__init__(client, data) self._sender = data.get("sender", None) @property def sender(self): """ Sender for the messaging channel, if present :rtype: str """ return self._sender def send(self, project_key, to, subject, body, attachments=None, plain_text=False, sender=None, cc=None, bcc=None): """ Send an email with or without attachments to a list of recipients .. code-block:: python channel = client.get_messaging_channel("mail-channel-id") channel.send("PROJECT_KEY", ["john.doe@dataiku.com", "jane.doe@dataiku.com"], "Hello there!", "Some HTML body") channel = client.get_messaging_channel("other-mail-channel-id") for file in paths: with open(file) as f: # Optionally include file type ("text/csv") attachments.append(file, f.read(), "text/csv") channel.send("PROJECT_KEY", ["joe@dataiku.com"], "Subject", "Body in plain text", attachments=attachments, False) :param project_key: project issuing the email. The user must have "Write content" permission on the specified project. :type project_key: str :param to: email addresses of recipients :type to: list[str] :param subject: email subject :type subject: str :param body: email body (in plain text or HTML format) :type body: str :param attachments: files to be attached to the mail, defaults to None :type attachments: list[BufferedReader] :param plain_text: True to send email as plain text, False to send it as HTML. Defaults to False. :type plain_text: bool :param sender: sender email address. Use None to use the sender defined at the channel level. :type sender: str :param cc: email addresses of recipients in carbon copy :type cc: list[str] :param bcc: email addresses of recipients in blind carbon copy :type bcc: list[str] """ payload = { "from": sender, "to": to, "cc": cc, "bcc": bcc, "subject": subject, "body": body, "plainText": plain_text } files = [ ('message', ('send-payload.json', json.dumps(payload), 'application/json')), ] if attachments: for attachment in attachments: files.append(('attachments', attachment)) self.client._perform_http("POST", "/messaging-channels/%s/actions/send?projectKey=%s" % (self.id, project_key), stream=False, files=files) dataiku-api-client-python-13.2.2/dataikuapi/dss/metrics.py000066400000000000000000000134131471211575100235160ustar00rootroot00000000000000 class ComputedMetrics(object): """ Handle to the metrics of a DSS object and their last computed value .. important:: Do not create this class directly, instead use :meth:`.DSSDataset.get_last_metric_values`, :meth:`.DSSSavedModel.get_metric_values`, :meth:`.DSSManagedFolder.get_last_metric_values`. """ def __init__(self, raw): self.raw = raw def get_raw(self): """ Get the raw metrics data :rtype: dict """ return self.raw def get_metric_by_id(self, id): """ Retrieve the info for a given metric Usage example .. code-block:: python dataset = project.get_ataset("my_dataset") metrics = dataset.get_last_metric_values() count_files_metric = metrics.get_metric_by_id("basic:COUNT_FILES") for value in count_files_metric['lastValues']: print("partition=%s -> count of files=%s" % (value['partition'], value['value'])) :param string metric_id: identifier of the metric :return: information about the metric and its values. Since the last value of the metric depends on the partition considered, the last values of the metric are given in a sub-list of the dict. :rtype: dict """ all_ids = [] for metric in self.raw["metrics"]: all_ids.append(metric["metric"]["id"]) if metric["metric"]["id"] == id: return metric raise Exception("Metric %s not found among: %s" % (id, all_ids)) def get_global_data(self, metric_id): """ Get the global value point of a given metric, or throws. For a partitioned dataset, the global value is the value of the metric computed on the whole dataset (coded as partition 'ALL'). :param string metric_id: identifier of the metric :returns: the metric data, as a dict. The value itself is a **value** string field. :rtype: dict """ for partition_data in self.get_metric_by_id(metric_id)["lastValues"]: if partition_data["partition"]== "NP" or partition_data["partition"] == "ALL": return partition_data raise Exception("No data found for global partition for metric %s" % metric_id) def get_global_value(self, metric_id): """ Get the global value of a given metric, or throws. For a partitioned dataset, the global value is the value of the metric computed on the whole dataset (coded as partition 'ALL'). Usage example .. code-block:: python dataset = project.get_ataset("my_dataset") metrics = dataset.get_last_metric_values() print("record count = %s" % metrics.get_global_value('records:COUNT_RECORDS')) :param string metric_id: identifier of the metric :returns: the value of the metric for the partition :rtype: str, int or float """ return ComputedMetrics.get_value_from_data(self.get_global_data(metric_id)) def get_partition_data(self, metric_id, partition): """ Get the value point of a given metric for a given partition, or throws. :param string metric_id: identifier of the metric :param string partition: partition identifier :returns: the metric data, as a dict. The value itself is a **value** string field. :rtype: dict """ for partition_data in self.get_metric_by_id(metric_id)["lastValues"]: if partition_data["partition"] == partition: return partition_data def get_partition_value(self, metric_id, partition): """ Get the value of a given metric for a given partition, or throws. :param string metric_id: identifier of the metric :param string partition: partition identifier :returns: the value of the metric for the partition :rtype: str, int or float """ return ComputedMetrics.get_value_from_data(self.get_partition_data(metric_id, partition)) def get_first_partition_data(self, metric_id): """ Get a value point of a given metric, or throws. The first value encountered is returned. :param string metric_id: identifier of the metric :returns: the metric data, as a dict. The value itself is a **value** string field. :rtype: dict """ for partition_data in self.get_metric_by_id(metric_id)["lastValues"]: return partition_data raise Exception("No partition data for metric %s" % metric_id) def get_all_ids(self): """ Get the identifiers of all metrics defined in this object :returns: list of metric identifiers :rtype: list[string] """ all_ids = [] for metric in self.raw["metrics"]: all_ids.append(metric["metric"]["id"]) return all_ids @staticmethod def get_value_from_data(data): """ Retrieves the value from a metric point, cast in the appropriate type (str, int or float). For other types, the value is not cast and left as a string. :param dict data: a value point for a metric, retrieved with :meth:`~get_global_data`, :meth:`~get_partition_data` or :meth:`~get_first_partition_data` :returns: the value, cast to the appropriate Python type :rtype: str, int or float """ dtype = data.get("dataType", "STRING") if dtype in ["BIGINT", "INT"]: return int(data["value"]) elif dtype in ["FLOAT", "DOUBLE"]: return float(data["value"]) else: return data["value"] dataiku-api-client-python-13.2.2/dataikuapi/dss/ml.py000066400000000000000000007050751471211575100224740ustar00rootroot00000000000000import json import logging import re import time import warnings from six import string_types from .future import DSSFuture from .utils import DSSDatasetSelectionBuilder from .utils import DSSFilterBuilder from ..utils import DataikuException from ..utils import _write_response_content_to_file logger = logging.getLogger("dataikuapi.dss.ml") class PredictionSplitParamsHandler(object): """ Object to modify the train/test dataset splitting params. .. important:: Do not create this class directly, use :meth:`DSSMLTaskSettings.get_split_params()` """ SPLIT_PARAMS_KEY = 'splitParams' def __init__(self, mltask_settings): self.mltask_settings = mltask_settings def get_raw(self): """ Gets the raw settings of the prediction split configuration. This returns a reference to the raw settings, rather than a copy, so any changes made to the returned object will be reflected when saving. :return: The raw prediction split parameter settings :rtype: dict """ return self.mltask_settings[PredictionSplitParamsHandler.SPLIT_PARAMS_KEY] def set_split_random(self, train_ratio=0.8, selection=None, dataset_name=None): """ Sets the train/test split mode to random splitting over an extract from a single dataset :param float train_ratio: Ratio of rows to use for the train set. Must be between 0 and 1 (defaults to **0.8**) :param selection: Optional builder or dict defining the settings of the extract from the dataset (defaults to **None**). A dict with the appropriate schema can be generated via :meth:`dataikuapi.dss.utils.DSSDatasetSelectionBuilder.build()` :type selection: Union[:class:`dataikuapi.dss.utils.DSSDatasetSelectionBuilder`, dict], optional :param dataset_name: Name of the dataset to split on. If None (default), uses the main dataset used to create the visual analysis :type dataset_name: str, optional """ sp = self.mltask_settings[PredictionSplitParamsHandler.SPLIT_PARAMS_KEY] sp["ttPolicy"] = "SPLIT_SINGLE_DATASET" if selection is not None: if isinstance(selection, DSSDatasetSelectionBuilder): sp["ssdSelection"] = selection.build() else: sp["ssdSelection"] = selection sp["ssdTrainingRatio"] = train_ratio sp["kfold"] = False if dataset_name is not None: sp["ssdDatasetSmartName"] = dataset_name return self def set_split_kfold(self, n_folds=5, selection=None, dataset_name=None): """ Sets the train/test split mode to k-fold splitting over an extract from a single dataset :param int n_folds: number of folds. Must be greater than 0 (defaults to **5**) :param selection: Optional builder or dict defining the settings of the extract from the dataset (defaults to **None**) A dict with the appropriate schema can be generated via :meth:`dataikuapi.dss.utils.DSSDatasetSelectionBuilder.build()` :type selection: Union[:class:`~dataikuapi.dss.utils.DSSDatasetSelectionBuilder`, dict], optional :param dataset_name: Name of the dataset to split on. If None (default), uses the main dataset used to create the visual analysis :type dataset_name: str, optional """ sp = self.mltask_settings[PredictionSplitParamsHandler.SPLIT_PARAMS_KEY] sp["ttPolicy"] = "SPLIT_SINGLE_DATASET" if selection is not None: if isinstance(selection, DSSDatasetSelectionBuilder): sp["ssdSelection"] = selection.build() else: sp["ssdSelection"] = selection sp["kfold"] = True sp["nFolds"] = n_folds if dataset_name is not None: sp["ssdDatasetSmartName"] = dataset_name return self def set_split_explicit(self, train_selection, test_selection, dataset_name=None, test_dataset_name=None, train_filter=None, test_filter=None): """ Sets the train/test split to an explicit extract from one or two dataset(s) :param train_selection: Builder or dict defining the settings of the extract for the train dataset. May be None (won't be changed). A dict with the appropriate schema can be generated via :meth:`dataikuapi.dss.utils.DSSDatasetSelectionBuilder.build()` :type train_selection: Union[:class:`~dataikuapi.dss.utils.DSSDatasetSelectionBuilder`, dict] :param test_selection: Builder or dict defining the settings of the extract for the test dataset. May be None (won't be changed). A dict with the appropriate schema can be generated via :meth:`dataikuapi.dss.utils.DSSDatasetSelectionBuilder.build()` :type test_selection: Union[:class:`~dataikuapi.dss.utils.DSSDatasetSelectionBuilder`, dict] :param dataset_name: Name of the dataset to split on. If None (default), uses the main dataset used to create the visual analysis :type dataset_name: str, optional :param test_dataset_name: Optional name of a second dataset to use for the test data extract. If None (default), both extracts are done from dataset_name :type test_dataset_name: str, optional :param train_filter: Builder or dict defining the settings of the filter for the train dataset. Defaults to None (won't be changed). A dict with the appropriate schema can be generated via :meth:`dataikuapi.dss.utils.DSSFilterBuilder.build()` :type train_filter: Union[:class:`~dataikuapi.dss.utils.DSSFilterBuilder`, dict], optional :param test_filter: Builder or dict defining the settings of the filter for the test dataset. Defaults to None (won't be changed). A dict with the appropriate schema can be generated via :meth:`dataikuapi.dss.utils.DSSFilterBuilder.build()` :type test_filter: Union[:class:`~dataikuapi.dss.utils.DSSFilterBuilder`, dict], optional """ sp = self.mltask_settings[PredictionSplitParamsHandler.SPLIT_PARAMS_KEY] if dataset_name is None: raise Exception("For explicit splitting a dataset_name is mandatory") if test_dataset_name is None or test_dataset_name == dataset_name: sp["ttPolicy"] = "EXPLICIT_FILTERING_SINGLE_DATASET" train_split ={} test_split = {} sp['efsdDatasetSmartName'] = dataset_name sp['efsdTrain'] = train_split sp['efsdTest'] = test_split else: sp["ttPolicy"] = "EXPLICIT_FILTERING_TWO_DATASETS" train_split ={'datasetSmartName' : dataset_name} test_split = {'datasetSmartName' : test_dataset_name} sp['eftdTrain'] = train_split sp['eftdTest'] = test_split if train_selection is not None: if isinstance(train_selection, DSSDatasetSelectionBuilder): train_split["selection"] = train_selection.build() else: train_split["selection"] = train_selection if test_selection is not None: if isinstance(test_selection, DSSDatasetSelectionBuilder): test_split["selection"] = test_selection.build() else: test_split["selection"] = test_selection if train_filter is not None: if isinstance(train_filter, DSSFilterBuilder): train_split["filter"] = train_filter.build() else: train_split["filter"] = train_filter if test_filter is not None: if isinstance(test_filter, DSSFilterBuilder): test_split["filter"] = test_filter.build() else: test_split["filter"] = test_filter return self def set_time_ordering(self, feature_name, ascending=True): """ Enables time based ordering and sets the feature upon which to sort the train/test split and hyperparameter optimization data by time. :param str feature_name: The name of the feature column to use. This feature must be present in the output of the preparation steps of the analysis. When there are no preparation steps, it means this feature must be present in the analyzed dataset. :param bool ascending: True (default) means the test set is expected to have larger time values than the train set """ self.unset_time_ordering() if not feature_name in self.mltask_settings["preprocessing"]["per_feature"]: raise ValueError("Feature %s doesn't exist in this ML task, can't use as time" % feature_name) self.mltask_settings['time']['enabled'] = True self.mltask_settings['time']['timeVariable'] = feature_name self.mltask_settings['time']['ascending'] = ascending self.mltask_settings['preprocessing']['per_feature'][feature_name]['missing_handling'] = "DROP_ROW" if self.mltask_settings['splitParams']['ttPolicy'] == "SPLIT_SINGLE_DATASET": self.mltask_settings['splitParams']['ssdSplitMode'] = "SORTED" self.mltask_settings['splitParams']['ssdColumn'] = feature_name if self.mltask_settings['modeling']['gridSearchParams']['mode'] == "KFOLD": self.mltask_settings['modeling']['gridSearchParams']['mode'] = "TIME_SERIES_KFOLD" elif self.mltask_settings['modeling']['gridSearchParams']['mode'] == "SHUFFLE": self.mltask_settings['modeling']['gridSearchParams']['mode'] = "TIME_SERIES_SINGLE_SPLIT" return self def unset_time_ordering(self): """ Disables time-based ordering for train/test split and hyperparameter optimization """ self.mltask_settings['time']['enabled'] = False self.mltask_settings['time']['timeVariable'] = None if self.mltask_settings['splitParams']['ttPolicy'] == "SPLIT_SINGLE_DATASET": self.mltask_settings['splitParams']['ssdSplitMode'] = "RANDOM" self.mltask_settings['splitParams']['ssdColumn'] = None if self.mltask_settings['modeling']['gridSearchParams']['mode'] == "TIME_SERIES_KFOLD": self.mltask_settings['modeling']['gridSearchParams']['mode'] = "KFOLD" elif self.mltask_settings['modeling']['gridSearchParams']['mode'] == "TIME_SERIES_SINGLE_SPLIT": self.mltask_settings['modeling']['gridSearchParams']['mode'] = "SHUFFLE" return self def has_time_ordering(self): """ :return: True if the split uses time-based ordering :rtype: bool """ return 'time' in self.mltask_settings and self.mltask_settings['time']['enabled'] def get_time_ordering_variable(self): """ :return: If enabled, the name of the ordering variable for time based ordering (the feature name). Returns None if time based ordering is not enabled. :rtype: Union[str, None] """ if self.has_time_ordering(): return self.mltask_settings['time']['timeVariable'] else: warnings.warn("Time-based ordering is disabled in the current MLTask") def is_time_ordering_ascending(self): """ :return: True if the time-ordering is set to sort in ascending order. Returns None if time based ordering is not enabled. :rtype: Union[bool, None] """ if self.has_time_ordering(): return self.mltask_settings['time']['ascending'] else: warnings.warn("Time-based ordering is disabled in the current MLTask") class DSSMLTaskSettings(object): """ Object to read and modify the settings of an existing ML task. .. important:: Do not create this class directly, use :meth:`DSSMLTask.get_settings()` instead Usage example: .. code-block:: python project_key = 'DKU_CHURN' fmi = 'A-DKU_CHURN-RADgquHe-5nJtl88L-s1-pp1-m1' client = dataiku.api_client() task = dataikuapi.dss.ml.DSSMLTask.from_full_model_id(client, fmi, project_key) task_settings = task.get_settings() task_settings.set_diagnostics_enabled(False) task_settings.save() """ def __init__(self, client, project_key, analysis_id, mltask_id, mltask_settings): self.client = client self.project_key = project_key self.analysis_id = analysis_id self.mltask_id = mltask_id self.mltask_settings = mltask_settings def get_raw(self): """ Gets the raw settings of this ML Task. This returns a reference to the raw settings, rather than a copy, so any changes made to the returned object will be reflected when saving. :return: The raw settings of this ML Task :rtype: dict """ return self.mltask_settings def get_feature_preprocessing(self, feature_name): """ Gets the feature preprocessing parameters for a particular feature. This returns a reference to the selected features' settings, rather than a copy, so any changes made to the returned object will be reflected when saving. :param str feature_name: Name of the feature whose parameters will be returned :return: A dict of the preprocessing settings for a feature :rtype: dict """ return self.mltask_settings["preprocessing"]["per_feature"][feature_name] def foreach_feature(self, fn, only_of_type=None): """ Applies a function to all features, including REJECTED features, except for the target feature :param fn: Function handle of the form ``fn(feature_name, feature_params) -> dict``, where `feature_name` is the feature name as a `str`, and feature_params is a `dict` containing the specific feature params. The function should return a dict of edited parameters for the feature. :type fn: function :param only_of_type: If set, only applies the function to features matching the given type. Must be one of ``CATEGORY``, ``NUMERIC``, ``TEXT`` or ``VECTOR``. :type only_of_type: Union[str, None], optional """ import copy new_per_feature = {} for (k, v) in self.mltask_settings["preprocessing"]["per_feature"].items(): if v["role"] == "TARGET" or (only_of_type is not None and v["type"] != only_of_type): new_per_feature[k] = v else: new_per_feature[k] = fn(k, copy.deepcopy(v)) self.mltask_settings["preprocessing"]["per_feature"] = new_per_feature def reject_feature(self, feature_name): """ Marks a feature as 'rejected', disabling it from being used as an input when training. This reverses the effect of the :meth:`use_feature` method. :param str feature_name: Name of the feature to reject """ self.get_feature_preprocessing(feature_name)["role"] = "REJECT" def use_feature(self, feature_name): """ Marks a feature to be used (enabled) as an input when training. This reverses the effect of the :meth:`reject_feature` method. :param str feature_name: Name of the feature to use/enable """ self.get_feature_preprocessing(feature_name)["role"] = "INPUT" def get_algorithm_settings(self, algorithm_name): """ .. caution:: Not Implemented, throws `NotImplementedError` """ raise NotImplementedError() def _get_custom_algorithm_settings(self, algorithm_name): # returns the first algorithm with this name for algo in self.mltask_settings["modeling"]["custom_mllib"]: if algorithm_name == algo["name"]: return algo for algo in self.mltask_settings["modeling"]["custom_python"]: if algorithm_name == algo["name"]: return algo raise ValueError("Unknown algorithm: {}".format(algorithm_name)) def get_diagnostics_settings(self): """ Gets the ML Tasks diagnostics' settings. This returns a reference to the diagnostics' settings, rather than a copy, so changes made to the returned object will be reflected when saving. This method returns a dictionary of the settings with: * **enabled** (boolean): Indicates if the diagnostics are enabled globally, if False, all diagnostics will be disabled * **settings** (List[dict]): A list of dicts. Each dict will contain the following: * **type** (str): The diagnostic type name, in uppercase * **enabled** (boolean): Indicates if the diagnostic type is enabled. If False, all diagnostics of that type will be disabled Please refer to the documentation for details on available diagnostics. :return: A dict of diagnostics settings :rtype: dict """ return self.mltask_settings["diagnosticsSettings"] def set_diagnostics_enabled(self, enabled): """ Globally enables or disables the calculation of all diagnostics :param bool enabled: True if the diagnostics should be enabled, False otherwise """ settings = self.get_diagnostics_settings() settings["enabled"] = enabled def set_diagnostic_type_enabled(self, diagnostic_type, enabled): """ Enables or disables the calculation of a set of diagnostics given their type. .. attention:: This is overridden by whether diagnostics are enabled globally; If diagnostics are disabled globally, nothing will be calculated. Diagnostics can be enabled/disabled globally via the :meth:`.set_diagnostics_enabled()` method. Usage example: .. code-block:: python mltask_settings = task.get_settings() mltask_settings.set_diagnostics_enabled(True) mltask_settings.set_diagnostic_type_enabled("ML_DIAGNOSTICS_DATASET_SANITY_CHECKS", False) mltask_settings.set_diagnostic_type_enabled("ML_DIAGNOSTICS_LEAKAGE_DETECTION", False) mltask_settings.save() Please refer to the documentation for details on available diagnostics. :param str diagnostic_type: Name of the diagnostic type, in uppercase. :param bool enabled: True if the diagnostic should be enabled, False otherwise """ settings = self.get_diagnostics_settings()["settings"] diagnostic = [h for h in settings if h["type"] == diagnostic_type] if len(diagnostic) == 0: raise ValueError("Diagnostic type '{}' not found in settings".format(diagnostic_type)) if len(diagnostic) > 1: raise ValueError("Should not happen: multiple diagnostic types '{}' found in settings".format(diagnostic_type)) diagnostic[0]["enabled"] = enabled def set_algorithm_enabled(self, algorithm_name, enabled): """ Enables or disables an algorithm given its name. Exact algorithm names can be found using the :meth:`.get_all_possible_algorithm_names()` method. Please refer to the documentation for further information on available algorithms. :param str algorithm_name: Name of the algorithm, in uppercase. :param bool enabled: True if the algorithm should be enabled, False otherwise """ self.get_algorithm_settings(algorithm_name)["enabled"] = enabled def disable_all_algorithms(self): """ Disables all algorithms """ for algorithm_key in self.__class__._algorithm_remap.keys(): algorithm_meta = self.__class__._algorithm_remap[algorithm_key] if isinstance(algorithm_meta, PredictionAlgorithmMeta): key = algorithm_meta.algorithm_name else: key = algorithm_meta if key in self.mltask_settings["modeling"]: self.mltask_settings["modeling"][key]["enabled"] = False for custom_mllib in self.mltask_settings["modeling"]["custom_mllib"]: custom_mllib["enabled"] = False for custom_python in self.mltask_settings["modeling"]["custom_python"]: custom_python["enabled"] = False for plugin in self.mltask_settings["modeling"].get("plugin_python", {}).values(): plugin["enabled"] = False def get_all_possible_algorithm_names(self): """ Gets the list of possible algorithm names This can be used to find the list of valid identifiers for the :meth:`set_algorithm_enabled` and :meth:`get_algorithm_settings` methods. This includes all possible algorithms, regardless of the prediction kind (regression/classification etc) or engine, so some algorithms may be irrelevant to the current task. :returns: The list of algorithm names as a list of strings :rtype: list[str] """ # this returns all possible alg names, but only the custom algorithm names for defined custom algorithms on the MLTask engine (python/mlflow) return list(self.__class__._algorithm_remap.keys()) + self._get_custom_algorithm_names() def _get_custom_algorithm_names(self): """ Gets the list of names of any custom models. Depending on the engine, this will return the names of any defined custom Python or MLlib models. :returns: The list of custom model names :rtype: list[str] """ return ([algo["name"] for algo in self.mltask_settings["modeling"]["custom_mllib"]] + [algo["name"] for algo in self.mltask_settings["modeling"]["custom_python"]]) def get_enabled_algorithm_names(self): """ Gets the list of enabled algorithm names :returns: The list of enabled algorithm names :rtype: list[str] """ return [algo_name for algo_name in self.get_all_possible_algorithm_names() if self.get_algorithm_settings(algo_name).get("enabled", False)] def get_enabled_algorithm_settings(self): """ Gets the settings for each enabled algorithm Returns a dictionary where: * Each key is the name of an enabled algorithm * Each value is the result of calling :meth:`.get_algorithm_settings()` with the key as the parameter :returns: The dict of enabled algorithm names with their settings :rtype: dict """ return {key: self.get_algorithm_settings(key) for key in self.get_enabled_algorithm_names()} def set_metric(self, metric=None, custom_metric=None, custom_metric_greater_is_better=True, custom_metric_use_probas=False, custom_metric_name=None): """ Sets the score metric to optimize for a prediction ML Task When using a custom optimisation metric, the `metric` parameter must be kept as `None`, and a string containing the metric code should be passed to the `custom_metric` parameter. :param metric: Name of the metric to use. Must be left empty to use a custom metric (defaults to **None**). :type metric: str, optional :param custom_metric: Code for the custom optimisation metric (defaults to **None**) :type custom_metric: str, optional :param custom_metric_greater_is_better: Whether the custom metric function returns a score (True, default) or a loss (False). Score functions return higher values as the model improves, whereas loss functions return lower values. :type custom_metric_greater_is_better: bool, optional :param bool custom_metric_use_probas: If True, will use the classes' probas or the predicted value (for classification) (defaults to **False**) :type custom_metric_use_probas: bool, optional :param custom_metric_name: Name of your custom metric. If not set, it will generate one. :type custom_metric_name: str, optional """ def get_evaluation_metric_default_name(): custom_evaluation_name_base_str = "Custom score #" custom_metrics = set(map(lambda cm: cm["name"], self.mltask_settings["modeling"]["metrics"]["customMetrics"])) i = 1 while custom_evaluation_name_base_str + str(i) in custom_metrics: i += 1 return custom_evaluation_name_base_str + str(i) if custom_metric is None and metric is None: raise ValueError("Either metric or custom_metric must be defined") self.mltask_settings["modeling"]["metrics"]["evaluationMetric"] = metric if custom_metric is None else 'CUSTOM' if custom_metric: new_custom_metric = { "name": custom_metric_name or get_evaluation_metric_default_name(), "metricCode": custom_metric, "greaterIsBetter": custom_metric_greater_is_better, "needsProbability": custom_metric_use_probas, "description": "Custom scoring function" } self.mltask_settings["modeling"]["metrics"]["customEvaluationMetricName"] = new_custom_metric["name"] self.mltask_settings["modeling"]["metrics"]["customMetrics"].append(new_custom_metric) def add_custom_python_model(self, name="Custom Python Model", code=""): """ Adds a new custom python model and enables it. Your code must create a 'clf' variable. This clf must be a scikit-learn compatible estimator, ie, it should: 1. have at least `fit(X,y)` and `predict(X)` methods 2. inherit `sklearn.base.BaseEstimator` 3. handle the attributes in the `__init__` function 4. have a `classes_` attribute (for classification tasks) 5. have a `predict_proba` method (optional) Example: .. code-block:: python mltask_settings = task.get_settings() code = \""" from sklearn.ensemble import AdaBoostClassifier clf = AdaBoostClassifier(n_estimators=20) \""" mltask_settings.add_custom_python_model(name="sklearn adaboost custom", code=code) mltask_settings.save() See: https://doc.dataiku.com/dss/latest/machine-learning/custom-models.html :param str name: The name of the custom model (defaults to **"Custom Python Model"**) :param str code: The code for the custom model (defaults to **""**) """ self.mltask_settings["modeling"]["custom_python"].append({ "name": name, "code": code, "enabled": True }) def add_custom_mllib_model(self, name="Custom MLlib Model", code=""): """ Adds a new custom MLlib model and enables it This example has sample code that uses a standard MLlib algorithm, the RandomForestClassifier: .. code-block:: scala mltask_settings = task.get_settings() code = \""" // import the Estimator from spark.ml import org.apache.spark.ml.classification.RandomForestClassifier // instantiate the Estimator new RandomForestClassifier() .setLabelCol("Survived") // Must be the target column .setFeaturesCol("__dku_features") // Must always be __dku_features .setPredictionCol("prediction") // Must always be prediction .setNumTrees(50) .setMaxDepth(8) \""" mltask_settings.add_custom_mllib_model(name="spark random forest custom", code=code) mltask_settings.save() :param str name: The name of the custom model (defaults to **"Custom MLlib Model"**) :param str code: The code for the custom model (defaults to **""**) """ self.mltask_settings["modeling"]["custom_mllib"].append({ "name": name, "initializationCode": code, "enabled": True }) def save(self): """ Saves the settings back to the ML Task """ self.client._perform_empty( "POST", "/projects/%s/models/lab/%s/%s/settings" % (self.project_key, self.analysis_id, self.mltask_id), body = self.mltask_settings) class HyperparameterSearchSettings(object): """ Object to read and modify hyperparameter search settings. This is available for all non-clustering ML Tasks. .. important:: Do not create this class directly, use :meth:`AbstractTabularPredictionMLTaskSettings.get_hyperparameter_search_settings()` """ def __init__(self, raw_settings): self._raw_settings = raw_settings def _key_repr(self, key): if isinstance(self._raw_settings[key], string_types): return " \"{}\"=\"{}\"\n".format(key, self._raw_settings[key]) else: return " \"{}\"={}\n".format(key, self._raw_settings[key]) def _repr_html_(self): res = "
"
        res += self.__class__.__name__ + "(\n"
        res += "Search Strategy:\n"
        res += self._key_repr("strategy")
        if self._raw_settings["strategy"] == "BAYESIAN":
            res += self._key_repr("bayesianOptimizer")

        res += "Search Validation:\n"
        res += self._key_repr("mode")
        if self._raw_settings["mode"] in {"SHUFFLE", "TIME_SERIES_SINGLE_SPLIT"}:
            res += self._key_repr("splitRatio")
        elif self._raw_settings["mode"] in {"KFOLD", "TIME_SERIES_KFOLD"}:
            res += self._key_repr("nFolds")
        res += self._key_repr("cvSeed")
        res += self._key_repr("stratified")

        res += "Execution Settings:\n"
        if self._raw_settings.get("timeout", 0) > 0:
            res += self._key_repr("timeout")
        if self._raw_settings["strategy"] == "GRID":
            res += self._key_repr("nIter")
            res += self._key_repr("randomized")
            if self._raw_settings.get("randomized", False):
                res += self._key_repr("seed")
        else:
            # RANDOM and BAYESIAN search strategies
            res += self._key_repr("nIterRandom")
            res += self._key_repr("seed")

        res += "Parallelism Settings:\n"
        res += self._key_repr("nJobs")
        res += self._key_repr("distributed")
        if self._raw_settings.get("distributed", False):
            res += self._key_repr("nContainers")
        res += ")
" res += "
{}
".format(self.__repr__()) return res def __repr__(self): return self.__class__.__name__ + "(settings={})".format(self._raw_settings) __str__ = __repr__ def _set_seed(self, seed): if seed is not None: if not isinstance(seed, int): warnings.warn("HyperparameterSearchSettings ignoring invalid input: seed") warnings.warn("seed must be an integer") else: self._raw_settings["seed"] = seed def _set_cv_seed(self, seed): if seed is not None: assert isinstance(seed, int), "HyperparameterSearchSettings invalid input: cvSeed must be an integer" self._raw_settings["cvSeed"] = seed @property def strategy(self): """ :return: The hyperparameter search strategy. Will be one of "GRID" | "RANDOM" | "BAYESIAN". :rtype: str """ return self._raw_settings["strategy"] @strategy.setter def strategy(self, strategy): """ :param strategy: The hyperparameter search strategy. Must be one of "GRID" | "RANDOM" | "BAYESIAN". :type strategy: str """ assert strategy in {"GRID", "RANDOM", "BAYESIAN"} self._raw_settings["strategy"] = strategy def set_grid_search(self, shuffle=True, seed=1337): """ Sets the search strategy to "GRID", to perform a grid-search over the hyperparameters. :param shuffle: if True (default), iterate over a shuffled grid as opposed to lexicographical iteration over the cartesian product of the hyperparameters :type shuffle: bool :param seed: Seed value used to ensure reproducible results (defaults to **1337**) :type seed: int """ self._raw_settings["strategy"] = "GRID" if shuffle is not None: if not isinstance(shuffle, bool): warnings.warn("HyperparameterSearchSettings.set_grid_search ignoring invalid input: shuffle") warnings.warn("shuffle must be a boolean") else: self._raw_settings["randomized"] = shuffle self._set_seed(seed) def set_random_search(self, seed=1337): """ Sets the search strategy to "RANDOM", to perform a random search over the hyperparameters. :param seed: Seed value used to ensure reproducible results (defaults to **1337**) :type seed: int """ self._raw_settings["strategy"] = "RANDOM" self._set_seed(seed) def set_bayesian_search(self, seed=1337): """ Sets the search strategy to "BAYESIAN", to perform a Bayesian search over the hyperparameters. :param seed: Seed value used to ensure reproducible results (defaults to **1337**) :type seed: int """ self._raw_settings["strategy"] = "BAYESIAN" self._set_seed(seed) @property def validation_mode(self): """ :return: The cross-validation strategy. Will be one of "KFOLD" | "SHUFFLE" | "TIME_SERIES_KFOLD" | "TIME_SERIES_SINGLE_SPLIT" | "CUSTOM". :rtype: str """ return self._raw_settings["mode"] @validation_mode.setter def validation_mode(self, mode): """ :param mode: The cross-validation strategy. Must be one of "KFOLD" | "SHUFFLE" | "TIME_SERIES_KFOLD" | "TIME_SERIES_SINGLE_SPLIT" | "CUSTOM". :type mode: str """ assert mode in {"KFOLD", "SHUFFLE", "TIME_SERIES_KFOLD", "TIME_SERIES_SINGLE_SPLIT", "CUSTOM"} self._raw_settings["mode"] = mode @property def fold_offset(self): """ :return: Whether there is an offset between validation sets, to avoid overlap between cross-test sets (model evaluation) and cross-validation sets (hyperparameter search), if both are using k-fold. Only relevant for time series forecasting :rtype: bool """ return self._raw_settings["foldOffset"] @fold_offset.setter def fold_offset(self, value): """ :param value: Whether there is an offset between validation sets, to avoid overlap between cross-test sets (model evaluation) and cross-validation sets (hyperparameter search), if both are using k-fold. Only relevant for time series forecasting :type value: bool """ assert isinstance(value, bool) self._raw_settings["foldOffset"] = value @property def equal_duration_folds(self): """ :return: Whether every fold in cross-test and cross-validation should be of equal duration when using k-fold. Only relevant for time series forecasting. :rtype: bool """ return self._raw_settings["equalDurationFolds"] @equal_duration_folds.setter def equal_duration_folds(self, value): """ :param value: Whether every fold in cross-test and cross-validation should be of equal duration when using k-fold. Only relevant for time series forecasting. :type value: bool """ assert isinstance(value, bool) self._raw_settings["equalDurationFolds"] = value @property def cv_seed(self): """ :return: cross-validation seed for splitting the data during hyperparameter search :rtype: int """ return self._raw_settings["cvSeed"] @cv_seed.setter def cv_seed(self, seed): """ :param seed: cross-validation seed for splitting the data during hyperparameter search :type seed: int """ self._set_cv_seed(seed) def set_kfold_validation(self, n_folds=5, stratified=True, cv_seed=1337): """ Sets the validation mode to k-fold cross-validation. The mode will be set to either "KFOLD" or "TIME_SERIES_KFOLD", depending on whether time-based ordering is enabled. :param n_folds: The number of folds used for the hyperparameter search (defaults to **5**) :type n_folds: int :param stratified: If True, keep the same proportion of each target classes in all folds (defaults to **True**) :type stratified: bool :param cv_seed: Seed for cross-validation (defaults to **1337**) :type cv_seed: int """ if self._raw_settings["mode"] == "TIME_SERIES_SINGLE_SPLIT": self._raw_settings["mode"] = "TIME_SERIES_KFOLD" else: self._raw_settings["mode"] = "KFOLD" if n_folds is not None: if not (isinstance(n_folds, int) and n_folds > 0): warnings.warn("HyperparameterSearchSettings.set_kfold_validation ignoring invalid input: n_folds") warnings.warn("n_folds must be a positive integer") else: self._raw_settings["nFolds"] = n_folds if stratified is not None: if not isinstance(stratified, bool): warnings.warn("HyperparameterSearchSettings.set_validation_mode_to_kfold ignoring invalid input: stratified") warnings.warn("stratified must be a boolean") else: self._raw_settings["stratified"] = stratified self._set_cv_seed(cv_seed) def set_single_split_validation(self, split_ratio=0.8, stratified=True, cv_seed=1337): """ Sets the validation mode to single split. The mode will be set to either "SHUFFLE" or "TIME_SERIES_SINGLE_SPLIT", depending on whether time-based ordering is enabled. :param split_ratio: The ratio of the data used for training during hyperparameter search (defaults to **0.8**) :type split_ratio: float :param stratified: If True, keep the same proportion of each target classes in both splits (defaults to **True**) :type stratified: bool :param cv_seed: Seed for cross-validation (defaults to **1337**) :type cv_seed: int """ if self._raw_settings["mode"] == "TIME_SERIES_KFOLD": self._raw_settings["mode"] = "TIME_SERIES_SINGLE_SPLIT" else: self._raw_settings["mode"] = "SHUFFLE" if split_ratio is not None: if not (isinstance(split_ratio, float) and split_ratio > 0 and split_ratio < 1): warnings.warn("HyperparameterSearchSettings.set_single_split_validation ignoring invalid input: split_ratio") warnings.warn(" split_ratio must be float between 0 and 1") else: self._raw_settings["splitRatio"] = split_ratio if stratified is not None: if not isinstance(stratified, bool): warnings.warn("HyperparameterSearchSettings.set_single_split_validation ignoring invalid input: stratified") warnings.warn("stratified must be a boolean") else: self._raw_settings["stratified"] = stratified self._set_cv_seed(cv_seed) def set_custom_validation(self, code=None): """ Sets the validation mode to "CUSTOM", and sets the custom validation code. Your code must create a 'cv' variable. This 'cv' must be compatible with the scikit-learn 'CV splitter class family. Example splitter classes can be found here: https://scikit-learn.org/stable/modules/classes.html#splitter-classes See also: https://scikit-learn.org/stable/glossary.html#term-CV-splitter This example code uses the 'repeated K-fold' splitter of scikit-learn: .. code-block:: python from sklearn.model_selection import RepeatedKFold cv = RepeatedKFold(n_splits=3, n_repeats=5) :param code: definition of the validation :type code: str """ self._raw_settings["mode"] = "CUSTOM" if code is not None: if not isinstance(code, string_types): warnings.warn("HyperparameterSearchSettings.set_custom_validation ignoring invalid input: code") warnings.warn("code must be a Python interpretable string") else: self._raw_settings["code"] = code def set_search_distribution(self, distributed=False, n_containers=4): """ Sets the distribution parameters for the hyperparameter search execution. :param distributed: if True, distribute search in the Kubernetes cluster selected in the runtime environment's containerized execution configuration (defaults to **False**) :type distributed: bool :param n_containers: number of containers to use for the distributed search (defaults to **4**) :type n_containers: int """ assert isinstance(distributed, bool) if n_containers is not None: assert isinstance(n_containers, int) self._raw_settings["nContainers"] = n_containers self._raw_settings["distributed"] = distributed @property def distributed(self): """ :return: Whether the search is set to distributed :rtype: bool """ return self._raw_settings["distributed"] @distributed.setter def distributed(self, distributed): """ :param distributed: Whether the search will be distributed :type distributed: bool """ assert isinstance(distributed, bool) self._raw_settings["distributed"] = distributed @property def timeout(self): """ :return: The search timeout :rtype: int """ return self._raw_settings["timeout"] @timeout.setter def timeout(self, timeout): """ :param timeout: The search timeout :type timeout: int """ assert isinstance(timeout, int) self._raw_settings["timeout"] = timeout @property def n_iter(self): """ :return: The number of search iterations :rtype: int """ if self._raw_settings["strategy"] == "GRID": return self._raw_settings["nIter"] else: # RANDOM and BAYESIAN search strategies return self._raw_settings["nIterRandom"] @n_iter.setter def n_iter(self, n_iter): """ :param n_iter: The number of search iterations :type n_iter: int """ assert isinstance(n_iter, int) if self._raw_settings["strategy"] == "GRID": self._raw_settings["nIter"] = n_iter else: self._raw_settings["nIterRandom"] = n_iter @property def parallelism(self): """ :return: The number of threads used for the search :rtype: int """ return self._raw_settings["nJobs"] @parallelism.setter def parallelism(self, n_jobs): """ :param n_jobs: The number of threads used for the search :type n_jobs: int """ assert isinstance(n_jobs, int) self._raw_settings["nJobs"] = n_jobs class HyperparameterSettings(object): def __init__(self, name, algo_settings): self.name = name self._algo_settings = algo_settings def _repr_html_(self): return "
" + self._pretty_repr() + "
{}
".format(self.__repr__()) def _pretty_repr(self): raise NotImplementedError() class NumericalHyperparameterSettings(HyperparameterSettings): def _pretty_repr(self): raw_hyperparam = self._algo_settings[self.name] pretty_hyperparam = dict() if self._algo_settings.strategy == "GRID": pretty_hyperparam["definition_mode"] = raw_hyperparam["gridMode"] else: # RANDOM and BAYESIAN strategies pretty_hyperparam["definition_mode"] = raw_hyperparam["randomMode"] if self.definition_mode == "EXPLICIT": pretty_hyperparam["values"] = raw_hyperparam["values"] else: pretty_hyperparam["range"] = raw_hyperparam["range"] return self.__class__.__name__ + "(hyperparameter=\"{}\", settings={})".format(self.name, json.dumps(pretty_hyperparam, indent=4)) def __repr__(self): raw_dict = self._algo_settings[self.name] return self.__class__.__name__ + "(hyperparameter=\"{}\", settings={})".format(self.name, json.dumps(raw_dict)) __str__ = __repr__ @property def definition_mode(self): """ "EXPLICIT" means that the hyperparameter search is performed over a given set of values (default for grid search) "RANGE" means that the hyperparameter search is performed over a range of values (default for random and Bayesian searches) :return: str mode: "EXPLICIT" | "RANGE" """ if self._algo_settings.strategy == "GRID": return self._algo_settings[self.name]["gridMode"] else: # RANDOM and BAYESIAN search strategies return self._algo_settings[self.name]["randomMode"] @definition_mode.setter def definition_mode(self, mode): """ :param mode: "EXPLICIT" | "RANGE" :type mode: str """ assert mode in ["EXPLICIT", "RANGE"], "Hyperparameter definition mode must be either \"EXPLICIT\" or \"RANGE\"" if self._algo_settings.strategy == "GRID": self._algo_settings[self.name]["gridMode"] = mode else: # RANDOM and BAYESIAN search strategies self._algo_settings[self.name]["randomMode"] = mode def set_explicit_values(self, values): """ Sets both: - the explicit values to search over for the current numerical hyperparameter - the definition mode of the current numerical hyperparameter to "EXPLICIT" :param values: the explicit list of numerical values considered for this hyperparameter in the search :type values: list of float | list of int """ self.values = values self.definition_mode = "EXPLICIT" @property def values(self): """ :return: the explicit list of numerical values considered for this hyperparameter in the search :rtype: list """ return self._algo_settings[self.name]["values"] @values.setter def values(self, values): """ :param values: the explicit list of numerical values considered for this hyperparameter in the search :type values: list of float | list of int """ error_message = "Invalid values input type for hyperparameter " \ "\"{}\": ".format(self.name) + \ " expecting a non-empty list of numbers" assert values is not None and isinstance(values, list) and len(values) > 0, error_message for val in values: assert isinstance(val, int) or isinstance(val, float), error_message limit_min = self._algo_settings[self.name]["limit"].get("min") if limit_min is not None: assert all(limit_min <= val for val in values), "Value(s) below hyperparameter \"{}\" limit {}".format(self.name, limit_min) limit_max = self._algo_settings[self.name]["limit"].get("max") if limit_max is not None: assert all(val <= limit_max for val in values), "Value(s) above hyperparameter \"{}\" limit {}".format(self.name, limit_max) if len(set(values)) < len(values): warnings.warn("Detected duplicates in provided values: " + str(sorted(values))) self._algo_settings[self.name]["values"] = values def _check_number_input(self, input): assert isinstance(input, int) or isinstance(input, float), \ "Invalid input type for hyperparameter \"{}\": ".format(self.name) + \ "range bounds must be numbers" def _set_range(self, min=None, max=None, nb_values=None): if min is None and max is None and nb_values is None: warnings.warn("Numerical range for hyperparameter \"{}\" not modified".format(self.name)) else: # Check all the Range parameters input before setting any of them if min is not None: self._check_number_input(min) limit_min = self._algo_settings[self.name]["limit"].get("min") if limit_min is not None: assert limit_min <= min, "Range min {} is below hyperparameter \"{}\" limit {}".format(min, self.name, limit_min) if max is not None: self._check_number_input(max) limit_max = self._algo_settings[self.name]["limit"].get("max") if limit_max is not None: assert max <= limit_max, "Range max {} is above hyperparameter \"{}\" limit {}".format(max, self.name, limit_max) if min is not None and max is not None: assert min <= max, "Invalid Range: min {} is greater max {}".format(min, max) if nb_values is not None: assert isinstance(nb_values, int) and nb_values >= 2, "Range number of values for hyperparameter \"{}\" must be an integer and >= 2".format(self.name) # Set the Range parameters after they have been checked if min is not None: self._algo_settings[self.name]["range"]["min"] = min if max is not None: self._algo_settings[self.name]["range"]["max"] = max if nb_values is not None: self._algo_settings[self.name]["range"]["nbValues"] = nb_values class RangeSettings(object): """ [Internal] Range of a numerical hyperparameter (points to the algorithm settings) Should not be used directly by end users of the API """ def __init__(self, numerical_hyperparameter_settings): self._numerical_hyperparameter_settings = numerical_hyperparameter_settings self._range_dict = self._numerical_hyperparameter_settings._algo_settings[numerical_hyperparameter_settings.name]["range"] def __repr__(self): return "RangeSettings(min={}, max={}, nb_values={})".format(self.min, self.max, self.nb_values) @property def min(self): """ :return: the lower bound of the range for this hyperparameter :rtype: float | int """ return self._range_dict["min"] @min.setter def min(self, value): """ :param value: the lower bound of the range for this hyperparameter :type value: float | int """ self._numerical_hyperparameter_settings._set_range(min=value) @property def max(self): """ :return: the upper bound of the range for this hyperparameter :rtype: float | int """ return self._range_dict["max"] @max.setter def max(self, value): """ :param value: the upper bound of the range for this hyperparameter :type value: float | int """ self._numerical_hyperparameter_settings._set_range(max=value) @property def nb_values(self): """ :return: for grid-search ("GRID" strategy) only, the number of values between min and max to consider :rtype: int """ return self._range_dict["nbValues"] @nb_values.setter def nb_values(self, value): """ :param value: for grid-search ("GRID" strategy) only, the number of values between min and max to consider :type value: int """ self._numerical_hyperparameter_settings._set_range(nb_values=value) def set_range(self, min=None, max=None, nb_values=None): """ Sets both: - the range parameters to search over for the current numerical hyperparameter - the definition mode of the current numerical hyperparameter to "RANGE" :param min: the lower bound of the range for this hyperparameter :type min: float | int :param max: the upper bound of the range for this hyperparameter :type max: float | int :param nb_values: for grid-search ("GRID" strategy) only, the number of values between min and max to consider :type nb_values: int """ self._set_range(min=min, max=max, nb_values=nb_values) self.definition_mode = "RANGE" @property def range(self): return NumericalHyperparameterSettings.RangeSettings(self) class Range(object): """ Range of a numerical hyperparameter (min, max, nb_values) Use this class to define explicitly the parameters of the range of a numerical hyperparameter """ def _check_input(self, value): assert isinstance(value, (int, float)), "Invalid input type for Range: {}".format(type(value)) def __init__(self, min, max, nb_values=None): self._check_input(min) self._check_input(max) assert min <= max, "Invalid Range: min must be lower than max" self.min = min self.max = max self.nb_values = nb_values def __repr__(self): return "Range(min={}, max={}, nb_values={})".format(self.min, self.max, self.nb_values) class CategoricalHyperparameterSettings(HyperparameterSettings): def __repr__(self): return self.__class__.__name__ + "(hyperparameter=\"{}\", settings={})".format(self.name, json.dumps(self._algo_settings[self.name])) __str__ = __repr__ def _pretty_repr(self): return self.__class__.__name__ + "(hyperparameter=\"{}\", settings={})".format(self.name, json.dumps(self._algo_settings[self.name], indent=4)) def set_values(self, values): """ Enables the search over listed values (categories). :param values: list of values to enable, all other values will be disabled :type values: list of str """ assert isinstance(values, list), \ "Invalid input type {} for categorical hyperparameter {}: must be a list of strings".format(type(values), self.name) all_possible_values = self.get_all_possible_values() for category in values: assert isinstance(category, string_types), \ "Invalid input type {} for categorical hyperparameter {}: must be a string".format(type(category), self.name) assert category in all_possible_values, \ "Invalid input value \"{}\" for categorical hyperparameter {}: must be a member of {}".format(category, self.name, all_possible_values) for category in all_possible_values: if category in values: self._algo_settings[self.name]["values"][category] = {"enabled": True} else: self._algo_settings[self.name]["values"][category] = {"enabled": False} def get_values(self): """ :return: list of enabled categories for this hyperparameter :rtype: list of str """ values_dict = self._algo_settings[self.name]["values"] return [value for value in values_dict.keys() if values_dict[value]["enabled"]] def get_all_possible_values(self): """ :return: list of possible values for this hyperparameter :rtype: list of str """ return list(self._algo_settings[self.name]["values"].keys()) class SingleValueHyperparameterSettings(HyperparameterSettings): def __init__(self, name, algo_settings, accepted_types=None): super(SingleValueHyperparameterSettings, self).__init__(name, algo_settings) self.accepted_types = accepted_types def __repr__(self): return self.__class__.__name__ + "(hyperparameter=\"{}\", value={})".format(self.name, self._algo_settings[self.name]) __str__ = __repr__ _pretty_repr = __repr__ def set_value(self, value): """ :param value: :type value: bool | int | float | NoneType """ if self.accepted_types is not None: assert isinstance(value, tuple(self.accepted_types)), "Invalid type for hyperparameter {}. Must be one of: {}".format(self.name, self.accepted_types) self._algo_settings[self.name] = value def get_value(self): """ :return: current value :rtype: bool | int | float | NoneType """ return self._algo_settings[self.name] def get_accepted_types(self): """ :return: valid types for this hyperparameter """ return self.accepted_types class SingleCategoryHyperparameterSettings(HyperparameterSettings): def __init__(self, name, algo_settings, accepted_values=None): super(SingleCategoryHyperparameterSettings, self).__init__(name, algo_settings) self.accepted_values = accepted_values def __repr__(self): if self.accepted_values is not None: return self.__class__.__name__ + "(hyperparameter=\"{}\", value=\"{}\", accepted_values={})".format(self.name, self._algo_settings[self.name], self.accepted_values) else: return self.__class__.__name__ + "(hyperparameter=\"{}\", value=\"{}\")".format(self.name, self._algo_settings[self.name]) __str__ = __repr__ _pretty_repr = __repr__ def set_value(self, value): """ :param value: :type value: str """ if self.accepted_values is not None: assert value in self.accepted_values, "Invalid value for hyperparameter {}. Must be in {}".format(self.name, json.dumps(self.accepted_values)) self._algo_settings[self.name] = value def get_value(self): """ :return: current value :rtype: str """ return self._algo_settings[self.name] def get_all_possible_values(self): """ :return: list of possible values for this hyperparameter :rtype: list of str """ return self.accepted_values class PredictionAlgorithmSettings(dict): """ Object to read and modify the settings of a prediction ML algorithm. .. important:: Do not create this object directly, use :meth:`DSSMLTask.get_algorithm_settings` instead """ def __init__(self, raw_settings, hyperparameter_search_params): super(PredictionAlgorithmSettings, self).__init__(raw_settings) self._hyperparameter_search_params = hyperparameter_search_params self._hyperparameters_registry = dict() self._attr_to_json_remapping = dict() def __setattr__(self, attr_name, value): if not hasattr(self, attr_name): # call from __init__ super(PredictionAlgorithmSettings, self).__setattr__(attr_name, value) else: if attr_name in self._attr_to_json_remapping: # attribute name and json key mismatch (e.g. "lambda", "alphaMode") json_key = self._attr_to_json_remapping[attr_name] else: json_key = attr_name if json_key in self._hyperparameters_registry: # syntactic sugars target = self._hyperparameters_registry[json_key] if isinstance(target, (SingleValueHyperparameterSettings, SingleCategoryHyperparameterSettings)): target.set_value(value) elif isinstance(target, CategoricalHyperparameterSettings): target.set_values(value) elif isinstance(target, NumericalHyperparameterSettings): if isinstance(value, list): # algo.hyperparam = [x, y, z] target.set_explicit_values(values=value) elif isinstance(value, Range): # algo.hyperparam = Range(min=x, max=y, nb_values=z) target.set_range(min=value.min, max=value.max, nb_values=value.nb_values) elif isinstance(value, NumericalHyperparameterSettings): # algo.hyperparam = other_algo.other_hyperparam target.set_range(min=value.range.min, max=value.range.max, nb_values=value.range.nb_values) target.set_explicit_values(values=list(value.values)) target.definition_mode = value.definition_mode else: raise TypeError(("Invalid type for NumericalHyperparameterSettings {}\n" + "Expecting either list, Range or NumericalHyperparameterSettings").format(attr_name)) else: # simple parameter assert isinstance(value, type(target)), "Invalid type {} for parameter {}: expected {}".format(type(value), attr_name, type(target)) super(PredictionAlgorithmSettings, self).__setattr__(attr_name, value) # update attribute value self[json_key] = value # update underlying dict value for key json_key self._hyperparameters_registry[json_key] = value else: # other cases (properties setter, ...) super(PredictionAlgorithmSettings, self).__setattr__(attr_name, value) def _register_numerical_hyperparameter(self, json_key, attr_name=None): if attr_name is not None: self._attr_to_json_remapping[attr_name] = json_key self._hyperparameters_registry[json_key] = NumericalHyperparameterSettings(json_key, self) return self._hyperparameters_registry[json_key] def _register_categorical_hyperparameter(self, json_key, attr_name=None): if attr_name is not None: self._attr_to_json_remapping[attr_name] = json_key self._hyperparameters_registry[json_key] = CategoricalHyperparameterSettings(json_key, self) return self._hyperparameters_registry[json_key] def _register_single_category_hyperparameter(self, json_key, accepted_values=None, attr_name=None): if attr_name is not None: self._attr_to_json_remapping[attr_name] = json_key self._hyperparameters_registry[json_key] = SingleCategoryHyperparameterSettings(json_key, self, accepted_values=accepted_values) return self._hyperparameters_registry[json_key] def _register_single_value_hyperparameter(self, json_key, accepted_types=None, attr_name=None): if attr_name is not None: self._attr_to_json_remapping[attr_name] = json_key self._hyperparameters_registry[json_key] = SingleValueHyperparameterSettings(json_key, self, accepted_types=accepted_types) return self._hyperparameters_registry[json_key] def _register_simple_parameter(self, json_key, attr_name=None): if attr_name is not None: self._attr_to_json_remapping[attr_name] = json_key self._hyperparameters_registry[json_key] = self[json_key] return self._hyperparameters_registry[json_key] def _repr_html_(self): res = "
" + self.__class__.__name__ + "(\n"
        res += "    \"enabled\": {}".format(self.enabled) + "\n"
        for name, hyperparam_settings in self._hyperparameters_registry.items():
            if isinstance(hyperparam_settings, HyperparameterSettings):
                res += "    \"{}\": {}".format(name, hyperparam_settings._pretty_repr()) + "\n"
            else:
                res += "    \"{}\": {}".format(name, hyperparam_settings) + "\n"
        res += ")
" return res + "
{}
".format(self.__repr__()) def __repr__(self): return self.__class__.__name__ + "(values={})".format(dict.__repr__(self)) __str__ = __repr__ def _get_all_hyperparameter_names(self): return list(self._hyperparameters_registry.keys()) @property def enabled(self): """ :rtype: bool """ return self["enabled"] @enabled.setter def enabled(self, enabled): """ :param enabled: :type enabled: bool """ assert isinstance(enabled, bool), "enabled property must be a boolean" self["enabled"] = enabled @property def strategy(self): return self._hyperparameter_search_params["strategy"] @strategy.setter def strategy(self, _): raise AttributeError("The strategy must be set at the MLTask settings level.\n" "To update the search strategy, use .strategy = ..., " "obtained with .get_hyperparameter_search_settings()") class RandomForestSettings(PredictionAlgorithmSettings): def __init__(self, raw_settings, hyperparameter_search_params): super(RandomForestSettings, self).__init__(raw_settings, hyperparameter_search_params) self.n_estimators = self._register_numerical_hyperparameter("n_estimators") self.min_samples_leaf = self._register_numerical_hyperparameter("min_samples_leaf") self.max_tree_depth = self._register_numerical_hyperparameter("max_tree_depth") self.max_feature_prop = self._register_numerical_hyperparameter("max_feature_prop") self.max_features = self._register_numerical_hyperparameter("max_features") self.n_jobs = self._register_simple_parameter("n_jobs") self.selection_mode = self._register_single_category_hyperparameter("selection_mode", accepted_values=["sqrt", "log2", "number", "prop"]) class LightGBMSettings(PredictionAlgorithmSettings): def __init__(self, raw_settings, hyperparameter_search_params): super(LightGBMSettings, self).__init__(raw_settings, hyperparameter_search_params) self.boosting_type = self._register_categorical_hyperparameter("boosting_type") self.num_leaves = self._register_numerical_hyperparameter("num_leaves") self.learning_rate = self._register_numerical_hyperparameter("learning_rate") self.n_estimators = self._register_numerical_hyperparameter("n_estimators") self.min_split_gain = self._register_numerical_hyperparameter("min_split_gain") self.min_child_weight = self._register_numerical_hyperparameter("min_child_weight") self.min_child_samples = self._register_numerical_hyperparameter("min_child_samples") self.colsample_bytree = self._register_numerical_hyperparameter("colsample_bytree") self.reg_alpha = self._register_numerical_hyperparameter("reg_alpha") self.reg_lambda = self._register_numerical_hyperparameter("reg_lambda") self.early_stopping = self._register_single_value_hyperparameter("early_stopping", accepted_types=[bool]) self.early_stopping_rounds = self._register_single_value_hyperparameter("early_stopping_rounds", accepted_types=[int]) self.random_state = self._register_single_value_hyperparameter("random_state", accepted_types=[int]) self.n_jobs = self._register_single_value_hyperparameter("n_jobs", accepted_types=[int]) self.max_depth = self._register_single_value_hyperparameter("max_depth", accepted_types=[int]) self.subsample = self._register_single_value_hyperparameter("subsample", accepted_types=[float]) self.subsample_freq = self._register_single_value_hyperparameter("subsample_freq", accepted_types=[int]) self.use_bagging = self._register_single_value_hyperparameter("use_bagging", accepted_types=[bool]) class XGBoostSettings(PredictionAlgorithmSettings): def __init__(self, raw_settings, hyperparameter_search_params): super(XGBoostSettings, self).__init__(raw_settings, hyperparameter_search_params) self.max_depth = self._register_numerical_hyperparameter("max_depth") self.learning_rate = self._register_numerical_hyperparameter("learning_rate") self.gamma = self._register_numerical_hyperparameter("gamma") self.min_child_weight = self._register_numerical_hyperparameter("min_child_weight") self.max_delta_step = self._register_numerical_hyperparameter("max_delta_step") self.subsample = self._register_numerical_hyperparameter("subsample") self.colsample_bytree = self._register_numerical_hyperparameter("colsample_bytree") self.colsample_bylevel = self._register_numerical_hyperparameter("colsample_bylevel") self.alpha = self._register_numerical_hyperparameter("alpha") self.lambda_ = self._register_numerical_hyperparameter("lambda", attr_name="lambda_") self.booster = self._register_categorical_hyperparameter("booster") self.objective = self._register_categorical_hyperparameter("objective") self.n_estimators = self._register_single_value_hyperparameter("n_estimators", accepted_types=[int]) self.nthread = self._register_simple_parameter("nthread") self.scale_pos_weight = self._register_single_value_hyperparameter("scale_pos_weight", accepted_types=[int, float]) self.base_score = self._register_single_value_hyperparameter("base_score", accepted_types=[int, float]) self.impute_missing = self._register_single_value_hyperparameter("impute_missing", accepted_types=[bool]) self.missing = self._register_single_value_hyperparameter("missing", accepted_types=[int, float]) self.tree_method = self._register_single_category_hyperparameter("tree_method", accepted_values=["auto", "exact", "approx", "hist"]) self.seed = self._register_single_value_hyperparameter("seed", accepted_types=[int]) self.enable_early_stopping = self._register_single_value_hyperparameter("enable_early_stopping", accepted_types=[bool]) self.early_stopping_rounds = self._register_single_value_hyperparameter("early_stopping_rounds", accepted_types=[int]) self.tweedie_variance_power = self._register_single_value_hyperparameter("tweedie_variance_power", accepted_types=[int, float]) @property def enable_cuda(self): raise AttributeError("The XGBoost enable_cuda parameter has been removed. To enable GPU execution, please do so in the MLTask settings.") @enable_cuda.setter def enable_cuda(self, value): raise AttributeError("The XGBoost enable_cuda parameter has been removed. To enable GPU execution, please do so in the MLTask settings.") @property def cpu_tree_method(self): warnings.warn("cpu_tree_method parameter is deprecated, please use tree_method instead") return self.tree_method @cpu_tree_method.setter def cpu_tree_method(self, value): warnings.warn("cpu_tree_method parameter is deprecated, please use tree_method instead") self.tree_method.set_value(value) @property def gpu_tree_method(self): warnings.warn("gpu_tree_method parameter is deprecated, please use tree_method instead") return self.tree_method @gpu_tree_method.setter def gpu_tree_method(self, value): warnings.warn("gpu_tree_method parameter is deprecated, please use tree_method instead") if value == "gpu_exact": self.tree_method.set_value("exact") elif value == "gpu_hist": self.tree_method.set_value("hist") else: logger.warning("Unexpected value for tree_method") self.tree_method.set_value(value) class GradientBoostedTreesSettings(PredictionAlgorithmSettings): def __init__(self, raw_settings, hyperparameter_search_params): super(GradientBoostedTreesSettings, self).__init__(raw_settings, hyperparameter_search_params) self.n_estimators = self._register_numerical_hyperparameter("n_estimators") self.max_depth = self._register_numerical_hyperparameter("max_depth") self.min_samples_leaf = self._register_numerical_hyperparameter("min_samples_leaf") self.max_features = self._register_numerical_hyperparameter("max_features") self.max_feature_prop = self._register_numerical_hyperparameter("max_feature_prop") self.learning_rate = self._register_numerical_hyperparameter("learning_rate") self.loss = self._register_categorical_hyperparameter("loss") self.selection_mode = self._register_single_category_hyperparameter("selection_mode", accepted_values=["sqrt", "log2", "number", "prop"]) class DecisionTreeSettings(PredictionAlgorithmSettings): def __init__(self, raw_settings, hyperparameter_search_params): super(DecisionTreeSettings, self).__init__(raw_settings, hyperparameter_search_params) self.max_depth = self._register_numerical_hyperparameter("max_depth") self.min_samples_leaf = self._register_numerical_hyperparameter("min_samples_leaf") self.criterion = self._register_categorical_hyperparameter("criterion") self.splitter = self._register_categorical_hyperparameter("splitter") class LogitSettings(PredictionAlgorithmSettings): def __init__(self, raw_settings, hyperparameter_search_params): super(LogitSettings, self).__init__(raw_settings, hyperparameter_search_params) self.C = self._register_numerical_hyperparameter("C") self.penalty = self._register_categorical_hyperparameter("penalty") self.multi_class = self._register_single_category_hyperparameter("multi_class", accepted_values=["multinomial", "ovr"]) self.n_jobs = self._register_simple_parameter("n_jobs") class RidgeRegressionSettings(PredictionAlgorithmSettings): def __init__(self, raw_settings, hyperparameter_search_params): super(RidgeRegressionSettings, self).__init__(raw_settings, hyperparameter_search_params) self.alpha = self._register_numerical_hyperparameter("alpha") self.alpha_mode = self._register_single_category_hyperparameter("alphaMode", accepted_values=["MANUAL", "AUTO"], attr_name="alpha_mode") class LassoRegressionSettings(PredictionAlgorithmSettings): def __init__(self, raw_settings, hyperparameter_search_params): super(LassoRegressionSettings, self).__init__(raw_settings, hyperparameter_search_params) self.alpha = self._register_numerical_hyperparameter("alpha") self.alpha_mode = self._register_single_category_hyperparameter("alphaMode", accepted_values=["MANUAL", "AUTO_CV", "AUTO_IC"], attr_name="alpha_mode") class OLSSettings(PredictionAlgorithmSettings): def __init__(self, raw_settings, hyperparameter_search_params): super(OLSSettings, self).__init__(raw_settings, hyperparameter_search_params) self.n_jobs = self._register_simple_parameter("n_jobs") class LARSSettings(PredictionAlgorithmSettings): def __init__(self, raw_settings, hyperparameter_search_params): super(LARSSettings, self).__init__(raw_settings, hyperparameter_search_params) self.max_features = self._register_single_value_hyperparameter("max_features", accepted_types=[int]) self.K = self._register_single_value_hyperparameter("K", accepted_types=[int]) class SGDSettings(PredictionAlgorithmSettings): def __init__(self, raw_settings, hyperparameter_search_params): super(SGDSettings, self).__init__(raw_settings, hyperparameter_search_params) self.alpha = self._register_numerical_hyperparameter("alpha") self.loss = self._register_categorical_hyperparameter("loss") self.penalty = self._register_categorical_hyperparameter("penalty") self.l1_ratio = self._register_single_value_hyperparameter("l1_ratio", accepted_types=[int, float]) self.max_iter = self._register_single_value_hyperparameter("max_iter", accepted_types=[int]) self.tol = self._register_single_value_hyperparameter("tol", accepted_types=[int, float]) self.n_jobs = self._register_simple_parameter("n_jobs") class KNNSettings(PredictionAlgorithmSettings): def __init__(self, raw_settings, hyperparameter_search_params): super(KNNSettings, self).__init__(raw_settings, hyperparameter_search_params) self.k = self._register_numerical_hyperparameter("k") self.algorithm = self._register_single_category_hyperparameter("algorithm", accepted_values=["auto", "kd_tree", "ball_tree", "brute"]) self.distance_weighting = self._register_single_value_hyperparameter("distance_weighting", accepted_types=[bool]) self.p = self._register_single_value_hyperparameter("p", accepted_types=[int]) self.leaf_size = self._register_single_value_hyperparameter("leaf_size", accepted_types=[int]) class SVMSettings(PredictionAlgorithmSettings): def __init__(self, raw_settings, hyperparameter_search_params): super(SVMSettings, self).__init__(raw_settings, hyperparameter_search_params) self.custom_gamma = self._register_numerical_hyperparameter("custom_gamma") self.C = self._register_numerical_hyperparameter("C") self.gamma = self._register_categorical_hyperparameter("gamma") self.kernel = self._register_categorical_hyperparameter("kernel") self.coef0 = self._register_single_value_hyperparameter("coef0", accepted_types=[int, float]) self.tol = self._register_single_value_hyperparameter("tol", accepted_types=[int, float]) self.max_iter = self._register_single_value_hyperparameter("max_iter", accepted_types=[int]) class SingleLayerPerceptronSettings(PredictionAlgorithmSettings): def __init__(self, raw_settings, hyperparameter_search_params): super(SingleLayerPerceptronSettings, self).__init__(raw_settings, hyperparameter_search_params) self.layer_sizes = self._register_numerical_hyperparameter("layer_sizes") self.activation = self._register_single_category_hyperparameter("activation", accepted_values=["relu", "identity", "logistic", "tanh"]) self.solver = self._register_single_category_hyperparameter("solver", accepted_values=["adam", "sgd"]) self.alpha = self._register_single_value_hyperparameter("alpha", accepted_types=[int, float]) self.batch_size = self._register_single_value_hyperparameter("batch_size", accepted_types=[int]) self.auto_batch = self._register_single_value_hyperparameter("auto_batch", accepted_types=[bool]) self.max_iter = self._register_single_value_hyperparameter("max_iter", accepted_types=[int]) self.seed = self._register_single_value_hyperparameter("seed", accepted_types=[int]) self.tol = self._register_single_value_hyperparameter("tol", accepted_types=[int, float]) self.early_stopping = self._register_single_value_hyperparameter("early_stopping", accepted_types=[bool]) self.validation_fraction = self._register_single_value_hyperparameter("validation_fraction", accepted_types=[int, float]) self.beta_1 = self._register_single_value_hyperparameter("beta_1", accepted_types=[int, float]) self.beta_2 = self._register_single_value_hyperparameter("beta_2", accepted_types=[int, float]) self.epsilon = self._register_single_value_hyperparameter("epsilon", accepted_types=[int, float]) self.learning_rate = self._register_single_category_hyperparameter("learning_rate", accepted_values=["constant", "invscaling", "adaptive"]) self.power_t = self._register_single_value_hyperparameter("power_t", accepted_types=[int, float]) self.momentum = self._register_single_value_hyperparameter("momentum", accepted_types=[int, float]) self.nesterovs_momentum = self._register_single_value_hyperparameter("nesterovs_momentum", accepted_types=[bool]) self.shuffle = self._register_single_value_hyperparameter("shuffle", accepted_types=[bool]) self.learning_rate_init = self._register_single_value_hyperparameter("learning_rate_init", accepted_types=[int, float]) class DeepNeuralNetworkSettings(PredictionAlgorithmSettings): def __init__(self, raw_settings, hyperparameter_search_params): super(DeepNeuralNetworkSettings, self).__init__(raw_settings, hyperparameter_search_params) self.learning_rate = self._register_numerical_hyperparameter("learning_rate") self.hidden_layers = self._register_numerical_hyperparameter("hidden_layers") self.units = self._register_numerical_hyperparameter("units") self.batch_size = self._register_single_value_hyperparameter("batch_size", accepted_types=[int]) self.max_epochs = self._register_single_value_hyperparameter("max_epochs", accepted_types=[int]) self.early_stopping_enabled = self._register_single_value_hyperparameter("early_stopping_enabled", accepted_types=[bool]) self.early_stopping_patience = self._register_single_value_hyperparameter("early_stopping_patience", accepted_types=[int]) self.early_stopping_threshold = self._register_single_value_hyperparameter("early_stopping_threshold", accepted_types=[float]) @property def device(self): raise AttributeError("The Deep Neural Network device parameter has been removed. To enable GPU execution, please do so in the MLTask settings.") @device.setter def device(self, value): raise AttributeError("The Deep Neural Network device parameter has been removed. To enable GPU execution, please do so in the MLTask settings.") class MLLibLogitSettings(PredictionAlgorithmSettings): def __init__(self, raw_settings, hyperparameter_search_params): super(MLLibLogitSettings, self).__init__(raw_settings, hyperparameter_search_params) self.reg_param = self._register_numerical_hyperparameter("reg_param") self.enet_param = self._register_numerical_hyperparameter("enet_param") self.max_iter = self._register_single_value_hyperparameter("max_iter", accepted_types=[int]) class MLLibNaiveBayesSettings(PredictionAlgorithmSettings): def __init__(self, raw_settings, hyperparameter_search_params): super(MLLibNaiveBayesSettings, self).__init__(raw_settings, hyperparameter_search_params) self.lambda_ = self._register_numerical_hyperparameter("lambda", attr_name="lambda_") class MLLibLinearRegressionSettings(PredictionAlgorithmSettings): def __init__(self, raw_settings, hyperparameter_search_params): super(MLLibLinearRegressionSettings, self).__init__(raw_settings, hyperparameter_search_params) self.reg_param = self._register_numerical_hyperparameter("reg_param") self.enet_param = self._register_numerical_hyperparameter("enet_param") self.max_iter = self._register_single_value_hyperparameter("max_iter", accepted_types=[int]) class MLLibDecisionTreeSettings(PredictionAlgorithmSettings): def __init__(self, raw_settings, hyperparameter_search_params): super(MLLibDecisionTreeSettings, self).__init__(raw_settings, hyperparameter_search_params) self.max_depth = self._register_numerical_hyperparameter("max_depth") self.cache_node_ids = self._register_simple_parameter("cache_node_ids") self.checkpoint_interval = self._register_single_value_hyperparameter("checkpoint_interval", accepted_types=[int]) self.max_bins = self._register_single_value_hyperparameter("max_bins", accepted_types=[int]) self.max_memory_mb = self._register_simple_parameter("max_memory_mb") self.min_info_gain = self._register_single_value_hyperparameter("min_info_gain", accepted_types=[int, float]) self.min_instance_per_node = self._register_single_value_hyperparameter("min_instance_per_node", accepted_types=[int]) class _MLLibTreeEnsembleSettings(PredictionAlgorithmSettings): def __init__(self, raw_settings, hyperparameter_search_params): super(_MLLibTreeEnsembleSettings, self).__init__(raw_settings, hyperparameter_search_params) self.max_depth = self._register_numerical_hyperparameter("max_depth") self.num_trees = self._register_numerical_hyperparameter("num_trees") self.cache_node_ids = self._register_simple_parameter("cache_node_ids") self.checkpoint_interval = self._register_single_value_hyperparameter("checkpoint_interval", accepted_types=[int]) self.max_bins = self._register_single_value_hyperparameter("max_bins", accepted_types=[int]) self.max_memory_mb = self._register_simple_parameter("max_memory_mb") self.min_info_gain = self._register_single_value_hyperparameter("min_info_gain", accepted_types=[int, float]) self.min_instance_per_node = self._register_single_value_hyperparameter("min_instance_per_node", accepted_types=[int]) self.seed = self._register_single_value_hyperparameter("seed", accepted_types=[int]) self.subsampling_rate = self._register_single_value_hyperparameter("subsampling_rate", accepted_types=[int, float]) class MLLibRandomForestSettings(_MLLibTreeEnsembleSettings): def __init__(self, raw_settings, hyperparameter_search_params): super(MLLibRandomForestSettings, self).__init__(raw_settings, hyperparameter_search_params) self.impurity = self._register_single_category_hyperparameter("impurity", accepted_values=["gini", "entropy", "variance"]) self.subset_strategy = self._register_single_category_hyperparameter("subset_strategy", accepted_values=["auto", "all", "onethird", "sqrt", "log2"]) class MLLibGBTSettings(_MLLibTreeEnsembleSettings): def __init__(self, raw_settings, hyperparameter_search_params): super(MLLibGBTSettings, self).__init__(raw_settings, hyperparameter_search_params) self.step_size = self._register_numerical_hyperparameter("step_size") class SeasonalNaiveSettings(PredictionAlgorithmSettings): def __init__(self, raw_settings, hyperparameter_search_params): super(SeasonalNaiveSettings, self).__init__(raw_settings, hyperparameter_search_params) self.season_length = self._register_numerical_hyperparameter("season_length") class AutoArimaSettings(PredictionAlgorithmSettings): def __init__(self, raw_settings, hyperparameter_search_params): super(AutoArimaSettings, self).__init__(raw_settings, hyperparameter_search_params) self.m = self._register_numerical_hyperparameter("m") self.information_criterion = self._register_categorical_hyperparameter("information_criterion") self.test = self._register_categorical_hyperparameter("test") self.seasonal_test = self._register_categorical_hyperparameter("seasonal_test") self.method = self._register_categorical_hyperparameter("method") self.start_p = self._register_single_value_hyperparameter("start_p", accepted_types=[int]) self.max_p = self._register_single_value_hyperparameter("max_p", accepted_types=[int]) self.d = self._register_single_value_hyperparameter("d", accepted_types=[int, type(None)]) self.max_d = self._register_single_value_hyperparameter("max_d", accepted_types=[int]) self.start_q = self._register_single_value_hyperparameter("start_q", accepted_types=[int]) self.max_q = self._register_single_value_hyperparameter("max_q", accepted_types=[int]) self.start_P = self._register_single_value_hyperparameter("start_P", accepted_types=[int]) self.max_P = self._register_single_value_hyperparameter("max_P", accepted_types=[int]) self.D = self._register_single_value_hyperparameter("D", accepted_types=[int, type(None)]) self.max_D = self._register_single_value_hyperparameter("max_D", accepted_types=[int]) self.start_Q = self._register_single_value_hyperparameter("start_Q", accepted_types=[int]) self.max_Q = self._register_single_value_hyperparameter("max_Q", accepted_types=[int]) self.max_order = self._register_single_value_hyperparameter("max_order", accepted_types=[int]) self.stationary = self._register_single_value_hyperparameter("stationary", accepted_types=[bool]) self.maxiter = self._register_single_value_hyperparameter("maxiter", accepted_types=[int]) class SeasonalLoessSettings(PredictionAlgorithmSettings): def __init__(self, raw_settings, hyperparameter_search_params): super(SeasonalLoessSettings, self).__init__(raw_settings, hyperparameter_search_params) self.period = self._register_numerical_hyperparameter("period") self.seasonal = self._register_numerical_hyperparameter("seasonal") self.trend = self._register_numerical_hyperparameter("trend") self.low_pass = self._register_numerical_hyperparameter("low_pass") self.seasonal_deg = self._register_categorical_hyperparameter("seasonal_deg") self.trend_deg = self._register_categorical_hyperparameter("trend_deg") self.low_pass_deg = self._register_categorical_hyperparameter("low_pass_deg") self.seasonal_jump = self._register_numerical_hyperparameter("seasonal_jump") self.trend_jump = self._register_numerical_hyperparameter("trend_jump") self.low_pass_jump = self._register_numerical_hyperparameter("low_pass_jump") self.auto_trend = self._register_single_value_hyperparameter("auto_trend", accepted_types=[bool]) self.auto_low_pass = self._register_single_value_hyperparameter("auto_low_pass", accepted_types=[bool]) class ProphetSettings(PredictionAlgorithmSettings): def __init__(self, raw_settings, hyperparameter_search_params): super(ProphetSettings, self).__init__(raw_settings, hyperparameter_search_params) self.growth = self._register_single_category_hyperparameter("growth", accepted_values=["linear", "logistic", "flat"]) self.floor = self._register_single_value_hyperparameter("floor", accepted_types=[float]) self.cap = self._register_single_value_hyperparameter("cap", accepted_types=[float]) self.n_changepoints = self._register_single_value_hyperparameter("n_changepoints", accepted_types=[int]) self.changepoint_range = self._register_single_value_hyperparameter("changepoint_range", accepted_types=[float]) self.yearly_seasonality = self._register_single_category_hyperparameter("yearly_seasonality", accepted_values=["auto", "always", "never"]) self.weekly_seasonality = self._register_single_category_hyperparameter("weekly_seasonality", accepted_values=["auto", "always", "never"]) self.daily_seasonality = self._register_single_category_hyperparameter("daily_seasonality", accepted_values=["auto", "always", "never"]) self.seed = self._register_single_value_hyperparameter("seed", accepted_types=[int]) self.seasonality_mode = self._register_categorical_hyperparameter("seasonality_mode") self.seasonality_prior_scale = self._register_numerical_hyperparameter("seasonality_prior_scale") self.changepoint_prior_scale = self._register_numerical_hyperparameter("changepoint_prior_scale") self.holidays_prior_scale = self._register_numerical_hyperparameter("holidays_prior_scale") class GluonTSNPTSForecasterSettings(PredictionAlgorithmSettings): def __init__(self, raw_settings, hyperparameter_search_params): super(GluonTSNPTSForecasterSettings, self).__init__(raw_settings, hyperparameter_search_params) self.context_length = self._register_numerical_hyperparameter("context_length") self.kernel_type = self._register_categorical_hyperparameter("kernel_type") self.exp_kernel_weights = self._register_numerical_hyperparameter("exp_kernel_weights") self.feature_scale = self._register_numerical_hyperparameter("feature_scale") self.full_context = self._register_single_value_hyperparameter("full_context", accepted_types=[bool]) self.use_seasonal_model = self._register_single_value_hyperparameter("use_seasonal_model", accepted_types=[bool]) self.use_default_time_features = self._register_single_value_hyperparameter("use_default_time_features", accepted_types=[bool]) self.seed = self._register_single_value_hyperparameter("seed", accepted_types=[int]) class GluonTSSimpleFeedForwardSettings(PredictionAlgorithmSettings): def __init__(self, raw_settings, hyperparameter_search_params): super(GluonTSSimpleFeedForwardSettings, self).__init__(raw_settings, hyperparameter_search_params) self.context_length = self._register_numerical_hyperparameter("context_length") self.distr_output = self._register_categorical_hyperparameter("distr_output") self.batch_normalization = self._register_categorical_hyperparameter("batch_normalization") self.mean_scaling = self._register_categorical_hyperparameter("mean_scaling") self.num_hidden_dimensions = self._register_single_value_hyperparameter("num_hidden_dimensions", accepted_types=[list]) self.full_context = self._register_single_value_hyperparameter("full_context", accepted_types=[bool]) self.num_parallel_samples = self._register_single_value_hyperparameter("num_parallel_samples", accepted_types=[int]) self.batch_size = self._register_single_value_hyperparameter("batch_size", accepted_types=[int]) self.epochs = self._register_single_value_hyperparameter("epochs", accepted_types=[int]) self.auto_num_batches_per_epoch = self._register_single_value_hyperparameter("auto_num_batches_per_epoch", accepted_types=[bool]) self.num_batches_per_epoch = self._register_single_value_hyperparameter("num_batches_per_epoch", accepted_types=[int]) self.seed = self._register_single_value_hyperparameter("seed", accepted_types=[int]) class GluonTSDeepARSettings(PredictionAlgorithmSettings): def __init__(self, raw_settings, hyperparameter_search_params): super(GluonTSDeepARSettings, self).__init__(raw_settings, hyperparameter_search_params) self.context_length = self._register_numerical_hyperparameter("context_length") self.num_layers = self._register_numerical_hyperparameter("num_layers") self.num_cells = self._register_numerical_hyperparameter("num_cells") self.cell_type = self._register_categorical_hyperparameter("cell_type") self.dropoutcell_type = self._register_categorical_hyperparameter("dropoutcell_type") self.dropout_rate = self._register_numerical_hyperparameter("dropout_rate") self.alpha = self._register_numerical_hyperparameter("alpha") self.beta = self._register_numerical_hyperparameter("beta") self.distr_output = self._register_categorical_hyperparameter("distr_output") self.full_context = self._register_single_value_hyperparameter("full_context", accepted_types=[bool]) self.scaling = self._register_single_value_hyperparameter("scaling", accepted_types=[bool]) self.num_parallel_samples = self._register_single_value_hyperparameter("num_parallel_samples", accepted_types=[int]) self.minimum_scale = self._register_single_value_hyperparameter("minimum_scale", accepted_types=[float]) self.batch_size = self._register_single_value_hyperparameter("batch_size", accepted_types=[int]) self.epochs = self._register_single_value_hyperparameter("epochs", accepted_types=[int]) self.auto_num_batches_per_epoch = self._register_single_value_hyperparameter("auto_num_batches_per_epoch", accepted_types=[bool]) self.num_batches_per_epoch = self._register_single_value_hyperparameter("num_batches_per_epoch", accepted_types=[int]) self.seed = self._register_single_value_hyperparameter("seed", accepted_types=[int]) class GluonTSTransformerSettings(PredictionAlgorithmSettings): def __init__(self, raw_settings, hyperparameter_search_params): super(GluonTSTransformerSettings, self).__init__(raw_settings, hyperparameter_search_params) self.context_length = self._register_numerical_hyperparameter("context_length") self.model_dim = self._register_numerical_hyperparameter("model_dim") self.inner_ff_dim_scale = self._register_numerical_hyperparameter("inner_ff_dim_scale") self.num_heads = self._register_numerical_hyperparameter("num_heads") self.dropout_rate = self._register_numerical_hyperparameter("dropout_rate") self.distr_output = self._register_categorical_hyperparameter("distr_output") self.full_context = self._register_single_value_hyperparameter("full_context", accepted_types=[bool]) self.num_parallel_samples = self._register_single_value_hyperparameter("num_parallel_samples", accepted_types=[int]) self.batch_size = self._register_single_value_hyperparameter("batch_size", accepted_types=[int]) self.epochs = self._register_single_value_hyperparameter("epochs", accepted_types=[int]) self.auto_num_batches_per_epoch = self._register_single_value_hyperparameter("auto_num_batches_per_epoch", accepted_types=[bool]) self.num_batches_per_epoch = self._register_single_value_hyperparameter("num_batches_per_epoch", accepted_types=[int]) self.seed = self._register_single_value_hyperparameter("seed", accepted_types=[int]) class GluonTSMQCNNSettings(PredictionAlgorithmSettings): def __init__(self, raw_settings, hyperparameter_search_params): super(GluonTSMQCNNSettings, self).__init__(raw_settings, hyperparameter_search_params) self.context_length = self._register_numerical_hyperparameter("context_length") self.full_context = self._register_single_value_hyperparameter("full_context", accepted_types=[bool]) self.seed = self._register_single_value_hyperparameter("seed", accepted_types=[int]) self.decoder_mlp_dim_seq = self._register_single_value_hyperparameter("decoder_mlp_dim_seq", accepted_types=[list]) self.channels_seq = self._register_single_value_hyperparameter("channels_seq", accepted_types=[list]) self.dilation_seq = self._register_single_value_hyperparameter("dilation_seq", accepted_types=[list]) self.kernel_size_seq = self._register_single_value_hyperparameter("kernel_size_seq", accepted_types=[list]) self.scaling = self._register_single_value_hyperparameter("scaling", accepted_types=[bool]) self.batch_size = self._register_single_value_hyperparameter("batch_size", accepted_types=[int]) self.epochs = self._register_single_value_hyperparameter("epochs", accepted_types=[int]) self.auto_num_batches_per_epoch = self._register_single_value_hyperparameter("auto_num_batches_per_epoch", accepted_types=[bool]) self.num_batches_per_epoch = self._register_single_value_hyperparameter("num_batches_per_epoch", accepted_types=[int]) class CausalForestSettings(PredictionAlgorithmSettings): def __init__(self, raw_settings, hyperparameter_search_params): super(CausalForestSettings, self).__init__(raw_settings, hyperparameter_search_params) self.n_estimators = self._register_numerical_hyperparameter("n_estimators") self.max_depth = self._register_numerical_hyperparameter("max_depth") self.min_samples_leaf = self._register_numerical_hyperparameter("min_samples_leaf") self.max_feature_prop = self._register_numerical_hyperparameter("max_feature_prop") self.max_features = self._register_numerical_hyperparameter("max_features") self.criterion = self._register_categorical_hyperparameter("criterion") self.selection_mode = self._register_single_category_hyperparameter("selection_mode", accepted_values=["auto", "sqrt", "log2", "number", "prop"]) self.honest = self._register_single_value_hyperparameter("honest", accepted_types=[bool]) self.n_jobs = self._register_simple_parameter("n_jobs") class PredictionAlgorithmMeta: def __init__(self, algorithm_name, algorithm_settings_class=PredictionAlgorithmSettings): self.algorithm_name = algorithm_name self.algorithm_settings_class = algorithm_settings_class class AbstractTabularPredictionMLTaskSettings(DSSMLTaskSettings): def get_prediction_type(self): return self.mltask_settings['predictionType'] def get_algorithm_settings(self, algorithm_name): """ Gets the training settings for a particular algorithm. This returns a reference to the algorithm's settings, not a copy, so changes made to the returned object will be reflected when saving. This method returns the settings for this algorithm as an PredictionAlgorithmSettings (extended dict). All algorithm dicts have at least an "enabled" property/key in the settings. The "enabled" property/key indicates whether this algorithm will be trained. Other settings are algorithm-dependent and are the various hyperparameters of the algorithm. The precise properties/keys for each algorithm are not all documented. You can print the returned AlgorithmSettings to learn more about the settings of each particular algorithm. Please refer to the documentation for details on available algorithms. :param algorithm_name: Name of the algorithm, in uppercase. :type algorithm_name: str :return: A PredictionAlgorithmSettings (extended dict) for one of the built-in prediction algorithms :rtype: PredictionAlgorithmSettings """ if algorithm_name in self.__class__._algorithm_remap: algorithm_meta = self.__class__._algorithm_remap[algorithm_name] algorithm_name = algorithm_meta.algorithm_name algorithm_settings_class = algorithm_meta.algorithm_settings_class algorithm_settings = self.mltask_settings["modeling"][algorithm_name.lower()] if not isinstance(algorithm_settings, PredictionAlgorithmSettings): raw_hyperparameter_search_params = self.mltask_settings["modeling"]["gridSearchParams"] algorithm_settings = algorithm_settings_class(algorithm_settings, raw_hyperparameter_search_params) # Subsequent calls get the same object self.mltask_settings["modeling"][algorithm_name.lower()] = algorithm_settings return self.mltask_settings["modeling"][algorithm_name.lower()] elif algorithm_name in self._get_custom_algorithm_names(): return self._get_custom_algorithm_settings(algorithm_name) else: raise ValueError("Unknown algorithm: {}".format(algorithm_name)) def get_hyperparameter_search_settings(self): """ Gets the hyperparameter search parameters of the current DSSPredictionMLTaskSettings instance as a HyperparameterSearchSettings object. This object can be used to both get and set properties relevant to hyperparameter search, such as search strategy, cross-validation method, execution limits and parallelism. :return: A HyperparameterSearchSettings :rtype: :class:`HyperparameterSearchSettings` """ return HyperparameterSearchSettings(self.mltask_settings["modeling"]["gridSearchParams"]) @property def split_params(self): """ Deprecated. Use get_split_params() """ return self.get_split_params() def get_split_params(self): """ Gets a handle to modify train/test splitting params. :rtype: :class:`PredictionSplitParamsHandler` """ return PredictionSplitParamsHandler(self.mltask_settings) def get_assertions_params(self): """ Retrieves the assertions parameters for this ml task :rtype: :class:`DSSMLAssertionsParams` """ raise NotImplementedError("ML assertions are not available for {}".format(self.__class__.__name__)) class DSSPredictionMLTaskSettings(AbstractTabularPredictionMLTaskSettings): """ For classical prediction ML tasks """ __doc__ = [] _algorithm_remap = { "RANDOM_FOREST_CLASSIFICATION": PredictionAlgorithmMeta("random_forest_classification", RandomForestSettings), "RANDOM_FOREST_REGRESSION": PredictionAlgorithmMeta("random_forest_regression", RandomForestSettings), "EXTRA_TREES": PredictionAlgorithmMeta("extra_trees", RandomForestSettings), "GBT_CLASSIFICATION": PredictionAlgorithmMeta("gbt_classification", GradientBoostedTreesSettings), "GBT_REGRESSION": PredictionAlgorithmMeta("gbt_regression", GradientBoostedTreesSettings), "DECISION_TREE_CLASSIFICATION": PredictionAlgorithmMeta("decision_tree_classification", DecisionTreeSettings), "DECISION_TREE_REGRESSION": PredictionAlgorithmMeta("decision_tree_regression", DecisionTreeSettings), "RIDGE_REGRESSION": PredictionAlgorithmMeta("ridge_regression", RidgeRegressionSettings), "LASSO_REGRESSION": PredictionAlgorithmMeta("lasso_regression", LassoRegressionSettings), "LEASTSQUARE_REGRESSION": PredictionAlgorithmMeta("leastsquare_regression", OLSSettings), "SGD_REGRESSION": PredictionAlgorithmMeta("sgd_regression", SGDSettings), "KNN": PredictionAlgorithmMeta("knn", KNNSettings), "LOGISTIC_REGRESSION": PredictionAlgorithmMeta("logistic_regression", LogitSettings), "NEURAL_NETWORK": PredictionAlgorithmMeta("neural_network", SingleLayerPerceptronSettings), "DEEP_NEURAL_NETWORK_REGRESSION": PredictionAlgorithmMeta("deep_neural_network_regression", DeepNeuralNetworkSettings), "DEEP_NEURAL_NETWORK_CLASSIFICATION": PredictionAlgorithmMeta("deep_neural_network_classification", DeepNeuralNetworkSettings), "SVC_CLASSIFICATION": PredictionAlgorithmMeta("svc_classifier", SVMSettings), "SVM_REGRESSION": PredictionAlgorithmMeta("svm_regression", SVMSettings), "SGD_CLASSIFICATION": PredictionAlgorithmMeta("sgd_classifier", SGDSettings), "LARS": PredictionAlgorithmMeta("lars_params", LARSSettings), "LIGHTGBM_CLASSIFICATION": PredictionAlgorithmMeta("lightgbm_classification", LightGBMSettings), "LIGHTGBM_REGRESSION": PredictionAlgorithmMeta("lightgbm_regression", LightGBMSettings), "XGBOOST_CLASSIFICATION": PredictionAlgorithmMeta("xgboost", XGBoostSettings), "XGBOOST_REGRESSION": PredictionAlgorithmMeta("xgboost", XGBoostSettings), "SPARKLING_DEEP_LEARNING": PredictionAlgorithmMeta("deep_learning_sparkling"), "SPARKLING_GBM": PredictionAlgorithmMeta("gbm_sparkling"), "SPARKLING_RF": PredictionAlgorithmMeta("rf_sparkling"), "SPARKLING_GLM": PredictionAlgorithmMeta("glm_sparkling"), "SPARKLING_NB": PredictionAlgorithmMeta("nb_sparkling"), "MLLIB_LOGISTIC_REGRESSION": PredictionAlgorithmMeta("mllib_logit", MLLibLogitSettings), "MLLIB_NAIVE_BAYES": PredictionAlgorithmMeta("mllib_naive_bayes", MLLibNaiveBayesSettings), "MLLIB_LINEAR_REGRESSION": PredictionAlgorithmMeta("mllib_linreg", MLLibLinearRegressionSettings), "MLLIB_RANDOM_FOREST": PredictionAlgorithmMeta("mllib_rf", MLLibRandomForestSettings), "MLLIB_GBT": PredictionAlgorithmMeta("mllib_gbt", MLLibGBTSettings), "MLLIB_DECISION_TREE": PredictionAlgorithmMeta("mllib_dt", MLLibDecisionTreeSettings), "VERTICA_LINEAR_REGRESSION": PredictionAlgorithmMeta("vertica_linear_regression"), "VERTICA_LOGISTIC_REGRESSION": PredictionAlgorithmMeta("vertica_logistic_regression"), "KERAS_CODE": PredictionAlgorithmMeta("keras") } class PredictionTypes: """ Possible prediction types """ BINARY = "BINARY_CLASSIFICATION" REGRESSION = "REGRESSION" MULTICLASS = "MULTICLASS" OTHER = "OTHER" def __init__(self, client, project_key, analysis_id, mltask_id, mltask_settings): super(DSSPredictionMLTaskSettings, self).__init__(client, project_key, analysis_id, mltask_id, mltask_settings) prediction_type = self.get_prediction_type() if prediction_type not in [self.PredictionTypes.BINARY, self.PredictionTypes.REGRESSION, self.PredictionTypes.MULTICLASS]: raise ValueError("Unknown prediction type: {}".format(prediction_type)) self.classification_prediction_types = [self.PredictionTypes.BINARY, self.PredictionTypes.MULTICLASS] def get_all_possible_algorithm_names(self): """ Returns the list of possible algorithm names. This includes the names of algorithms from installed plugins. This can be used as the list of valid identifiers for :meth:`set_algorithm_enabled` and :meth:`get_algorithm_settings`. This includes all possible algorithms, regardless of the prediction kind (regression/classification) or the engine, so some algorithms may be irrelevant to the current task. :returns: The list of algorithm names :rtype: list[str] """ return super(DSSPredictionMLTaskSettings, self).get_all_possible_algorithm_names() + self._get_plugin_algorithm_names() def _get_plugin_algorithm_names(self): return list(self.mltask_settings["modeling"]["plugin_python"].keys()) def _get_plugin_algorithm_settings(self, algorithm_name): if algorithm_name in self.mltask_settings["modeling"]["plugin_python"]: return self.mltask_settings["modeling"]["plugin_python"][algorithm_name] raise ValueError("Unknown algorithm: {}".format(algorithm_name)) def get_enabled_algorithm_names(self): algo_names = super(DSSPredictionMLTaskSettings, self).get_enabled_algorithm_names() # Hide either "XGBOOST_CLASSIFICATION" or "XGBOOST_REGRESSION" which point to the same key "xgboost" if self.mltask_settings["predictionType"] == "REGRESSION": excluded_names = {"XGBOOST_CLASSIFICATION"} else: excluded_names = {"XGBOOST_REGRESSION"} return [algo_name for algo_name in algo_names if algo_name not in excluded_names] def get_algorithm_settings(self, algorithm_name): if algorithm_name in self._get_plugin_algorithm_names(): return self._get_plugin_algorithm_settings(algorithm_name) else: return super(DSSPredictionMLTaskSettings, self).get_algorithm_settings(algorithm_name) def split_ordered_by(self, feature_name, ascending=True): """ Deprecated. Use split_params.set_time_ordering() """ warnings.warn("split_ordered_by() is deprecated, please use split_params.set_time_ordering() instead", DeprecationWarning) self.split_params.set_time_ordering(feature_name, ascending=ascending) return self def remove_ordered_split(self): """ Deprecated. Use split_params.unset_time_ordering() """ warnings.warn("remove_ordered_split() is deprecated, please use split_params.unset_time_ordering() instead", DeprecationWarning) self.split_params.unset_time_ordering() return self def use_sample_weighting(self, feature_name): """ Deprecated. use set_weighting() """ warnings.warn("use_sample_weighting() is deprecated, please use set_weighting() instead", DeprecationWarning) return self.set_weighting(method='SAMPLE_WEIGHT', feature_name=feature_name, ) def set_weighting(self, method, feature_name=None): """ Sets the method for weighting samples. If there was a WEIGHT feature declared previously, it will be set back as an INPUT feature first. :param str method: Weighting nethod to use. One of NO_WEIGHTING, SAMPLE_WEIGHT (requires a feature name), CLASS_WEIGHT or CLASS_AND_SAMPLE_WEIGHT (requires a feature name) :param feature_name: Name of the feature to use as sample weight :type feature_name: str, optional """ # First, if there was a WEIGHT feature, restore it as INPUT for other_feature_name in self.mltask_settings['preprocessing']['per_feature']: if self.mltask_settings['preprocessing']['per_feature'][other_feature_name]['role'] == 'WEIGHT': self.mltask_settings['preprocessing']['per_feature'][other_feature_name]['role'] = 'INPUT' if method == "NO_WEIGHTING": self.mltask_settings['weight']['weightMethod'] = method elif method == "SAMPLE_WEIGHT": if not feature_name in self.mltask_settings["preprocessing"]["per_feature"]: raise ValueError("Feature %s doesn't exist in this ML task, can't use as weight" % feature_name) self.mltask_settings['weight']['weightMethod'] = method self.mltask_settings['weight']['sampleWeightVariable'] = feature_name self.mltask_settings['preprocessing']['per_feature'][feature_name]['role'] = 'WEIGHT' elif method == "CLASS_WEIGHT": if self.get_prediction_type() not in self.classification_prediction_types: raise ValueError("Weighting method: {} not compatible with prediction type: {}, should be in {}".format(method, self.get_prediction_type(), self.classification_prediction_types)) self.mltask_settings['weight']['weightMethod'] = method elif method == "CLASS_AND_SAMPLE_WEIGHT": if self.get_prediction_type() not in self.classification_prediction_types: raise ValueError("Weighting method: {} not compatible with prediction type: {}, should be in {}".format(method, self.get_prediction_type(), self.classification_prediction_types)) if not feature_name in self.mltask_settings["preprocessing"]["per_feature"]: raise ValueError("Feature %s doesn't exist in this ML task, can't use as weight" % feature_name) self.mltask_settings['weight']['weightMethod'] = method self.mltask_settings['weight']['sampleWeightVariable'] = feature_name self.mltask_settings['preprocessing']['per_feature'][feature_name]['role'] = 'WEIGHT' else: raise ValueError("Unknown weighting method: {}".format(method)) return self def remove_sample_weighting(self): """ Deprecated. Use set_weighting(method=\"NO_WEIGHTING\") instead """ warnings.warn("remove_sample_weighting() is deprecated, please use set_weighting(method=\"NO_WEIGHTING\") instead", DeprecationWarning) return self.set_weighting(method="NO_WEIGHTING") def get_assertions_params(self): """ Retrieves the ML Task assertion parameters :return: The assertions parameters for this ML task :rtype: :class:`DSSMLAssertionsParams` """ return DSSMLAssertionsParams(self.mltask_settings["assertionsParams"]) class DSSClusteringMLTaskSettings(DSSMLTaskSettings): __doc__ = [] _algorithm_remap = { "DBSCAN": "db_scan_clustering", "HDBSCAN": "hdb_scan_clustering", "SPECTRAL": "spectral_clustering", "WARD": "ward_clustering", "KMEANS": "kmeans_clustering", "MINIBATCH_KMEANS": "mini_batch_kmeans_clustering", "GAUSSIAN_MIXTURE": "gmm_clustering", "TWO_STEP": "two_step", "ISOLATION_FOREST": "isolation_forest", "MLLIB_KMEANS": "mllib_kmeans_clustering", "MLLIB_GAUSSIAN_MIXTURE": "mllib_gaussian_mixture_clustering", "H20_KMEANS": "h2o_kmeans" } def get_algorithm_settings(self, algorithm_name): """ Gets the training settings of a particular algorithm. This returns a reference to the algorithm's settings, rather than a copy, so any changes made to the returned object will be reflected when saving. This method returns a dictionary of the settings for this algorithm. All algorithm dicts contain an "enabled" key, which indicates whether this algorithm will be trained Other settings are algorithm-dependent and include the various hyperparameters of the algorithm. The precise keys for each algorithm are not all documented. You can print the returned dictionary to learn more about the settings of each particular algorithm. Please refer to the documentation for details on available algorithms. :param algorithm_name: Name of the algorithm, in uppercase. :type algorithm_name: str :return: A dict containing the settings for the algorithm :rtype: dict """ if algorithm_name in self.__class__._algorithm_remap: algorithm_name = self.__class__._algorithm_remap[algorithm_name] return self.mltask_settings["modeling"][algorithm_name.lower()] elif algorithm_name in self._get_custom_algorithm_names(): return self._get_custom_algorithm_settings(algorithm_name) else: raise ValueError("Unknown algorithm: {}".format(algorithm_name)) class DSSTimeseriesForecastingMLTaskSettings(AbstractTabularPredictionMLTaskSettings): __doc__ = [] _algorithm_remap = { "TRIVIAL_IDENTITY_TIMESERIES": PredictionAlgorithmMeta("trivial_identity_timeseries"), "SEASONAL_NAIVE": PredictionAlgorithmMeta("seasonal_naive_timeseries", SeasonalNaiveSettings), "AUTO_ARIMA": PredictionAlgorithmMeta("autoarima_timeseries", AutoArimaSettings), "SEASONAL_LOESS": PredictionAlgorithmMeta("seasonal_loess_timeseries", SeasonalLoessSettings), "PROPHET": PredictionAlgorithmMeta("prophet_timeseries", ProphetSettings), "GLUONTS_NPTS_FORECASTER": PredictionAlgorithmMeta("gluonts_npts_timeseries", GluonTSNPTSForecasterSettings), "GLUONTS_SIMPLE_FEEDFORWARD": PredictionAlgorithmMeta("gluonts_simple_feed_forward_timeseries", GluonTSSimpleFeedForwardSettings), "GLUONTS_DEEPAR": PredictionAlgorithmMeta("gluonts_deepar_timeseries", GluonTSDeepARSettings), "GLUONTS_TRANSFORMER": PredictionAlgorithmMeta("gluonts_transformer_timeseries", GluonTSTransformerSettings), "GLUONTS_MQCNN": PredictionAlgorithmMeta("gluonts_mqcnn_timeseries", GluonTSMQCNNSettings), } _TIME_UNITS = {"MILLISECOND", "SECOND", "MINUTE", "HOUR", "DAY", "BUSINESS_DAY", "WEEK", "MONTH", "QUARTER", "HALF_YEAR", "YEAR"} _INTERPOLATION_METHODS = {"NEAREST", "PREVIOUS", "NEXT", "LINEAR", "QUADRATIC", "CUBIC", "CONSTANT"} _EXTRAPOLATION_METHODS = {"PREVIOUS_NEXT", "NO_EXTRAPOLATION", "CONSTANT", "LINEAR", "QUADRATIC", "CUBIC"} _CATEGORICAL_IMPUTATION_METHODS = {"MOST_COMMON", "NULL", "CONSTANT", "PREVIOUS_NEXT", "PREVIOUS", "NEXT"} _DUPLICATE_TIMESTAMPS_HANDLING_METHODS = {"FAIL_IF_CONFLICTING", "DROP_IF_CONFLICTING", "MEAN_MODE"} class PredictionTypes: """ Possible prediction types """ TIMESERIES_FORECAST = "TIMESERIES_FORECAST" def __init__(self, client, project_key, analysis_id, mltask_id, mltask_settings): super(DSSTimeseriesForecastingMLTaskSettings, self).__init__(client, project_key, analysis_id, mltask_id, mltask_settings) prediction_type = self.get_prediction_type() if prediction_type != self.PredictionTypes.TIMESERIES_FORECAST: raise ValueError("Unknown prediction type: {}".format(prediction_type)) def get_time_step_params(self): """ Gets the time step parameters for the time series forecasting task. This returns a reference to the time step parameters, rather than a copy, so any changes made to the returned object will be reflected when saving. :return: A dict of the time step parameters :rtype: dict """ return self.mltask_settings["timestepParams"] def set_time_step(self, time_unit=None, n_time_units=None, end_of_week_day=None, reguess=True, update_algorithm_settings=True, unit_alignment=None): """ Sets the time step parameters for the time series forecasting task. :param time_unit: time unit for forecasting step. Valid values are: MILLISECOND, SECOND, MINUTE, HOUR, DAY, BUSINESS_DAY, WEEK, MONTH, QUARTER, HALF_YEAR, YEAR (defaults to **None**, i.e. don't change) :type time_unit: str, optional :param n_time_units: number of time units within a time step (defaults to **None**, i.e. don't change) :type n_time_units: int, optional :param end_of_week_day: only useful for the WEEK time unit. Valid values are: 1 (Sunday), 2 (Monday), ..., 7 (Saturday) (defaults to **None**, i.e. don't change) :type end_of_week_day: int, optional :param reguess: Whether to reguess the ML task settings after changing the time step params (defaults to **True**) :type reguess: bool :param update_algorithm_settings: Whether the algorithm settings should also be reguessed if reguessing the ML Task (defaults to **True**) :type update_algorithm_settings: bool :param unit_alignment: month for each step when time_unit is QUARTER or YEAR, between 1 and 3 for QUARTER and 1 and 12 for YEAR (defaults to **None**, i.e. don't change) :type unit_alignment: int, optional """ time_step_params = self.get_time_step_params() if time_unit is not None: assert time_unit in self._TIME_UNITS, "Invalid time unit, should be in {}".format(self._TIME_UNITS) time_step_params["timeunit"] = time_unit if n_time_units is not None: assert isinstance(n_time_units, int), "n_time_units should be an int" assert n_time_units > 0, "n_time_units should be positive" time_step_params["numberOfTimeunits"] = n_time_units if end_of_week_day is not None: assert end_of_week_day in {1, 2, 3, 4, 5, 6, 7}, "Invalid end_of_week_day, should be in [1, 2, 3, 4, 5, 6, 7]" if time_step_params["timeunit"] != "WEEK": logger.warning("Changing end of week day, but time unit is not WEEK") time_step_params["endOfWeekDay"] = end_of_week_day if unit_alignment is not None: assert isinstance(unit_alignment, int), "unit_alignment should be an int" max_selected_month = 12 if time_step_params["timeunit"] == "QUARTER": max_selected_month = 3 assert unit_alignment >= 1 and unit_alignment <= max_selected_month, "unit_alignment should be in [1, {}]".format(max_selected_month) if time_step_params["timeunit"] not in ["QUARTER", "YEAR"]: logger.warning('Changing unit alignment, but time unit is not in ["QUARTER", "YEAR"]') time_step_params["unitAlignment"] = unit_alignment if reguess: logger.info("Reguessing ML task settings after changing time step params") self.client._perform_empty( "POST", "/projects/{project_key}/models/lab/{analysis_id}/{mltask_id}/reguess-with-forecasting-params".format( project_key=self.project_key, analysis_id=self.analysis_id, mltask_id=self.mltask_id ), body={"timestepParams": time_step_params, "updateAlgorithmSettings": update_algorithm_settings}, ) def get_resampling_params(self): """ Gets the time series resampling parameters for the time series forecasting task. This returns a reference to the time series resampling parameters, rather than a copy, so any changes made to the returned object will be reflected when saving. :return: A dict of the resampling parameters :rtype: dict """ return self.mltask_settings["preprocessing"]["timeseriesSampling"] def set_numerical_interpolation(self, method=None, constant=None): """ Sets the time series resampling numerical interpolation parameters :param method: Interpolation method. Valid values are: NEAREST, PREVIOUS, NEXT, LINEAR, QUADRATIC, CUBIC, CONSTANT (defaults to **None**, i.e. don't change) :type method: str, optional :param constant: Value for the CONSTANT interpolation method (defaults to **None**, i.e. don't change) :type constant: float, optional """ resampling_params = self.get_resampling_params() if method is not None: assert method in self._INTERPOLATION_METHODS, "Invalid interpolation method, should be in {}".format(self._INTERPOLATION_METHODS) resampling_params["numericalInterpolateMethod"] = method if constant is not None: assert isinstance(constant, float) or isinstance(constant, int), "Constant value should be a number" if resampling_params["numericalInterpolateMethod"] != "CONSTANT": logger.warning("Changing numerical interpolation constant value, but interpolation method is not CONSTANT") resampling_params["numericalInterpolateConstantValue"] = constant def set_numerical_extrapolation(self, method=None, constant=None): """ Sets the time series resampling numerical extrapolation parameters :param method: Extrapolation method. Valid values are: PREVIOUS_NEXT, NO_EXTRAPOLATION, CONSTANT, LINEAR, QUADRATIC, CUBIC (defaults to **None**, i.e. don't change) :type method: str, optional :param constant: Value for the CONSTANT extrapolation method (defaults to **None**) :type constant: float, optional """ resampling_params = self.get_resampling_params() if method is not None: assert method in self._EXTRAPOLATION_METHODS, "Invalid extrapolation method, should be in {}".format(self._EXTRAPOLATION_METHODS) resampling_params["numericalExtrapolateMethod"] = method if constant is not None: assert isinstance(constant, float) or isinstance(constant, int), "Constant value should be a number" if resampling_params["numericalExtrapolateMethod"] != "CONSTANT": logger.warning("Changing numerical extrapolation constant value, but extrapolation method is not CONSTANT") resampling_params["numericalExtrapolateConstantValue"] = constant def set_categorical_imputation(self, method=None, constant=None): """ Sets the time series resampling categorical imputation parameters :param method: Imputation method. Valid values are: MOST_COMMON, NULL, CONSTANT, PREVIOUS_NEXT, PREVIOUS, NEXT (defaults to **None**, i.e. don't change) :type method: str, optional :param constant: Value for the CONSTANT imputation method (defaults to **None**, i.e. don't change) :type constant: str, optional """ resampling_params = self.get_resampling_params() if method is not None: assert method in self._CATEGORICAL_IMPUTATION_METHODS, "Invalid imputation method, should be in {}".format(self._CATEGORICAL_IMPUTATION_METHODS) resampling_params["categoricalImputeMethod"] = method if constant is not None: assert isinstance(constant, str), "Constant value should be a string" if resampling_params["categoricalImputeMethod"] != "CONSTANT": logger.warning("Changing categorical imputation constant value, but imputation method is not CONSTANT") resampling_params["categoricalConstantValue"] = constant def set_duplicate_timestamp_handling(self, method): """ Sets the time series duplicate timestamp handling method :param method: Duplicate timestamp handling method. Valid values are: FAIL_IF_CONFLICTING, DROP_IF_CONFLICTING, MEAN_MODE. :type method: str """ resampling_params = self.get_resampling_params() assert method in self._DUPLICATE_TIMESTAMPS_HANDLING_METHODS, "Invalid duplicate timestamp handling method, should be in {}".format( self._DUPLICATE_TIMESTAMPS_HANDLING_METHODS ) resampling_params["duplicateTimestampsHandlingMethod"] = method @property def forecast_horizon(self): """ :return: Number of time steps to be forecast :rtype: int """ return self.mltask_settings["predictionLength"] def set_forecast_horizon(self, forecast_horizon, reguess=True, update_algorithm_settings=True, validation_horizons=None): """ Sets the time series forecast horizon :param forecast_horizon: Number of time steps to be forecast :type forecast_horizon: int :param reguess: Whether to reguess the ML task settings after changing the forecast horizon (defaults to **True**) :type reguess: bool :param update_algorithm_settings: Whether the algorithm settings should be reguessed after the forecast horizon (defaults to **True**) :type update_algorithm_settings: bool :param validation_horizons: The number of validation horizons to be set. If omitted, retains the previous ratio. :type validation_horizons: int|None """ assert isinstance(forecast_horizon, int) if validation_horizons is None: validation_horizons = self.mltask_settings["evaluationParams"]["testSize"] / self.mltask_settings["predictionLength"] assert validation_horizons % 1 == 0, "Validation horizons is not a whole integer" validation_horizons = int(validation_horizons) self.mltask_settings["predictionLength"] = forecast_horizon self.mltask_settings["evaluationParams"]["testSize"] = forecast_horizon * validation_horizons if reguess: logger.info("Reguessing ML task settings after changing forecast horizon") self.client._perform_empty( "POST", "/projects/{project_key}/models/lab/{analysis_id}/{mltask_id}/reguess-with-forecasting-params".format( project_key=self.project_key, analysis_id=self.analysis_id, mltask_id=self.mltask_id ), body={ "forecastHorizon": forecast_horizon, "updateAlgorithmSettings": update_algorithm_settings, "validationHorizons": validation_horizons }, ) @property def evaluation_gap(self): """ :return: Number of skipped time steps for evaluation :rtype: int """ return self.mltask_settings["evaluationParams"]["gapSize"] @evaluation_gap.setter def evaluation_gap(self, value): """ :param value: Number of skipped time steps for evaluation :type value: int """ assert isinstance(value, int) assert value < self.forecast_horizon, "Gap must be smaller than the forecast horizon" self.mltask_settings["evaluationParams"]["gapSize"] = value self.mltask_settings["evaluationParams"]["testSize"] = self.forecast_horizon - value @property def time_variable(self): """ :return: Feature used as time variable (read-only) :rtype: str """ return self.mltask_settings["timeVariable"] @property def timeseries_identifiers(self): """ :return: Features used as time series identifiers (read-only copy) :rtype: list """ return list(self.mltask_settings["timeseriesIdentifiers"]) @property def quantiles_to_forecast(self): """ :return: List of quantiles to forecast :rtype: list """ return self.mltask_settings["quantilesToForecast"] @quantiles_to_forecast.setter def quantiles_to_forecast(self, values): """ :param values: List of quantiles to forecast: - 0.5 should always be included - Values should be between 0.0001 and 0.9999 - Values should be defined up to 4 digits after decimal point :type values: list """ assert isinstance(values, list) self.mltask_settings["quantilesToForecast"] = sorted(set(values)) @property def skip_too_short_timeseries_for_training(self): """ :return: Whether we skip too short time series during training, or fail the whole training when only one time series is too short. :rtype: bool """ return self.mltask_settings.get("skipTooShortTimeseriesForTraining") @skip_too_short_timeseries_for_training.setter def skip_too_short_timeseries_for_training(self, value): self.mltask_settings["skipTooShortTimeseriesForTraining"] = value class DSSCausalPredictionMLTaskSettings(AbstractTabularPredictionMLTaskSettings): """ Object to read and modify the settings of a causal prediction ML task. """ _algorithm_remap = { "CAUSAL_FOREST": PredictionAlgorithmMeta("causal_forest", CausalForestSettings), # Base learners "RANDOM_FOREST_CLASSIFICATION": PredictionAlgorithmMeta("random_forest_classification", RandomForestSettings), "RANDOM_FOREST_REGRESSION": PredictionAlgorithmMeta("random_forest_regression", RandomForestSettings), "EXTRA_TREES": PredictionAlgorithmMeta("extra_trees", RandomForestSettings), "GBT_CLASSIFICATION": PredictionAlgorithmMeta("gbt_classification", GradientBoostedTreesSettings), "GBT_REGRESSION": PredictionAlgorithmMeta("gbt_regression", GradientBoostedTreesSettings), "DECISION_TREE_CLASSIFICATION": PredictionAlgorithmMeta("decision_tree_classification", DecisionTreeSettings), "DECISION_TREE_REGRESSION": PredictionAlgorithmMeta("decision_tree_regression", DecisionTreeSettings), "RIDGE_REGRESSION": PredictionAlgorithmMeta("ridge_regression", RidgeRegressionSettings), "LASSO_REGRESSION": PredictionAlgorithmMeta("lasso_regression", LassoRegressionSettings), "LEASTSQUARE_REGRESSION": PredictionAlgorithmMeta("leastsquare_regression", OLSSettings), "SGD_REGRESSION": PredictionAlgorithmMeta("sgd_regression", SGDSettings), "KNN": PredictionAlgorithmMeta("knn", KNNSettings), "LOGISTIC_REGRESSION": PredictionAlgorithmMeta("logistic_regression", LogitSettings), "NEURAL_NETWORK": PredictionAlgorithmMeta("neural_network", SingleLayerPerceptronSettings), "DEEP_NEURAL_NETWORK_REGRESSION": PredictionAlgorithmMeta("deep_neural_network_regression", DeepNeuralNetworkSettings), "DEEP_NEURAL_NETWORK_CLASSIFICATION": PredictionAlgorithmMeta("deep_neural_network_classification", DeepNeuralNetworkSettings), "SVC_CLASSIFICATION": PredictionAlgorithmMeta("svc_classifier", SVMSettings), "SVM_REGRESSION": PredictionAlgorithmMeta("svm_regression", SVMSettings), "SGD_CLASSIFICATION": PredictionAlgorithmMeta("sgd_classifier", SGDSettings), "LARS": PredictionAlgorithmMeta("lars_params", LARSSettings), "LIGHTGBM_CLASSIFICATION": PredictionAlgorithmMeta("lightgbm_classification", LightGBMSettings), "LIGHTGBM_REGRESSION": PredictionAlgorithmMeta("lightgbm_regression", LightGBMSettings), "XGBOOST_CLASSIFICATION": PredictionAlgorithmMeta("xgboost", XGBoostSettings), "XGBOOST_REGRESSION": PredictionAlgorithmMeta("xgboost", XGBoostSettings), } class PredictionTypes: CAUSAL_REGRESSION = "CAUSAL_REGRESSION" CAUSAL_BINARY_CLASSIFICATION = "CAUSAL_BINARY_CLASSIFICATION" class MetaLearners: S_LEARNER = "S_LEARNER" T_LEARNER = "T_LEARNER" X_LEARNER = "X_LEARNER" def __init__(self, client, project_key, analysis_id, mltask_id, mltask_settings): super(DSSCausalPredictionMLTaskSettings, self).__init__(client, project_key, analysis_id, mltask_id, mltask_settings) prediction_type = self.get_prediction_type() if prediction_type not in [self.PredictionTypes.CAUSAL_REGRESSION, self.PredictionTypes.CAUSAL_BINARY_CLASSIFICATION]: raise ValueError("Unknown prediction type: {}".format(prediction_type)) @property def positive_outcome_class(self): if self.get_prediction_type() != self.PredictionTypes.CAUSAL_BINARY_CLASSIFICATION: raise ValueError("Positive outcome class is only defined for causal binary classifications") return self.mltask_settings["positiveClass"] @positive_outcome_class.setter def positive_outcome_class(self, outcome_class): if self.get_prediction_type() != self.PredictionTypes.CAUSAL_BINARY_CLASSIFICATION: raise ValueError("Positive outcome class is only defined for causal binary classifications") outcome_classes = [x['sourceValue'] for x in self.mltask_settings["preprocessing"]["target_remapping"]] if outcome_class not in outcome_classes: raise ValueError("Unknown value for outcome variable: '{}'\nMust be in {}".format(outcome_class, str(outcome_classes))) self.mltask_settings["positiveClass"] = outcome_class @property def control_value(self): return self.mltask_settings["controlValue"] @control_value.setter def control_value(self, control_value): if control_value not in self.mltask_settings["treatmentValues"]: raise ValueError("Unknown value for treatment variable: {}\nMust be in {}".format(control_value, str(self.mltask_settings["treatmentValues"]))) self.mltask_settings["controlValue"] = control_value @property def enable_multi_valued_treatment(self): return self.mltask_settings["enableMultiTreatment"] @enable_multi_valued_treatment.setter def enable_multi_valued_treatment(self, enabled): self.mltask_settings["enableMultiTreatment"] = enabled if enabled and len(self.mltask_settings["treatmentValues"]) <= 2: logger.warning("Multi-valued treatment requires at least 3 values for the treatment variable") def _check_algorithm(self, algorithm_name): if algorithm_name != "CAUSAL_FOREST" and len(self.meta_learners) == 0: logger.warning("No meta-learner enabled for selected base learner: {}".format(algorithm_name)) def _check_meta_learner(self, meta_leaner_name): m = DSSCausalPredictionMLTaskSettings.MetaLearners all_meta_learners = [m.S_LEARNER, m.T_LEARNER, m.X_LEARNER] if meta_leaner_name not in all_meta_learners: raise ValueError('Invalid meta-learner name: {}\nMust be in {}'.format(meta_leaner_name, all_meta_learners)) enabled_base_learners = [a for a in self.get_enabled_algorithm_names() if a != "CAUSAL_FOREST"] if len(enabled_base_learners) == 0: logger.warning("No base-learner enabled, meta-learner {} will be ignored under current settings".format(meta_leaner_name)) def get_algorithm_settings(self, algorithm_name): self._check_algorithm(algorithm_name) return super(DSSCausalPredictionMLTaskSettings, self).get_algorithm_settings(algorithm_name) def set_algorithm_enabled(self, algorithm_name, enabled): if enabled: self._check_algorithm(algorithm_name) super(DSSCausalPredictionMLTaskSettings, self).set_algorithm_enabled(algorithm_name, enabled) def get_enabled_algorithm_names(self): enabled_algo_names = super(DSSCausalPredictionMLTaskSettings, self).get_enabled_algorithm_names() for algo_name in enabled_algo_names: self._check_algorithm(algo_name) return enabled_algo_names @property def meta_learners(self): return self.mltask_settings["modeling"]["meta_learners"] @meta_learners.setter def meta_learners(self, meta_learners): if type(meta_learners) != list or any([type(x) != str for x in meta_learners]): raise ValueError("Meta-learners must be specified as a list of str") for meta_leaner in meta_learners: self._check_meta_learner(meta_leaner) self.mltask_settings["modeling"]["meta_learners"] = meta_learners def get_treatment_analysis_settings(self): return self.mltask_settings["modeling"]["propensityModeling"] def set_treatment_analysis_enabled(self, enabled, calibrate_propensity=False, calibration_data_ratio=0.1): self.mltask_settings["modeling"]["propensityModeling"]["enabled"] = enabled self.mltask_settings["modeling"]["propensityModeling"]["calibrateProbabilities"] = calibrate_propensity self.mltask_settings["modeling"]["propensityModeling"]["calibrationDataRatio"] = calibration_data_ratio def add_custom_mllib_model(self, name="Custom MLlib Model", code=""): raise NotImplementedError("MLLib backend is not supported for causal predictions.") class DSSTrainedModelDetails(object): def __init__(self, details, snippet, saved_model=None, saved_model_version=None, mltask=None, mltask_model_id=None): self.details = details self.snippet = snippet self.saved_model = saved_model self.saved_model_version = saved_model_version self.mltask = mltask self.mltask_model_id = mltask_model_id def get_raw(self): """ Gets the raw dictionary of trained model details """ return self.details @property def full_id(self): return self.details["fullModelId"] def get_raw_snippet(self): """ Gets the raw dictionary of trained model snippet. The snippet is a lighter version than the details. """ return self.snippet def get_train_info(self): """ Returns various information about the training process (size of the train set, quick description, timing information) :rtype: dict """ return self.details["trainInfo"] def get_user_meta(self): """ Gets the user-accessible metadata (name, description, cluster labels, classification threshold) Returns the original object, not a copy. Changes to the returned object are persisted to DSS by calling :meth:`save_user_meta` """ return self.details["userMeta"] def save_user_meta(self): um = self.details["userMeta"] if self.mltask is not None: self.mltask.client._perform_empty( "PUT", "/projects/%s/models/lab/%s/%s/models/%s/user-meta" % (self.mltask.project_key, self.mltask.analysis_id, self.mltask.mltask_id, self.mltask_model_id), body = um) else: self.saved_model.client._perform_empty( "PUT", "/projects/%s/savedmodels/%s/versions/%s/user-meta" % (self.saved_model.project_key, self.saved_model.sm_id, self.saved_model_version), body = um) def get_origin_analysis_trained_model(self): """ Fetch details about the model in an analysis, this model has been exported from. Returns None if the deployed trained model does not have an origin analysis trained model. :rtype: DSSTrainedModelDetails | None """ if self.saved_model is None: return self else: fmi = self.get_raw().get("smOrigin", {}).get("fullModelId") if fmi is not None: origin_ml_task = DSSMLTask.from_full_model_id(self.saved_model.client, fmi, project_key=self.saved_model.project_key) return origin_ml_task.get_trained_model_details(fmi) def get_diagnostics(self): """ Retrieves diagnostics computed for this trained model :returns: list of diagnostics :rtype: list of type `dataikuapi.dss.ml.DSSMLDiagnostic` """ diagnostics = self.details.get("mlDiagnostics", {}) return [DSSMLDiagnostic(d) for d in diagnostics.get("diagnostics", [])] def generate_documentation(self, folder_id=None, path=None): """ Start the model document generation from a template docx file in a managed folder, or from the default template if no folder id and path are specified. :param folder_id: (optional) the id of the managed folder :param path: (optional) the path to the file from the root of the folder :return: A :class:`~dataikuapi.dss.future.DSSFuture` representing the model document generation process """ if bool(folder_id) != bool(path): raise ValueError("Both folder id and path arguments are required to use a template from folder. Use without argument to generate the model documentation using the default template") template_mode_url = "default-template" if folder_id is None and path is None else "template-in-folder" if self.mltask is not None: f = self.mltask.client._perform_json( "POST", "/projects/%s/models/lab/%s/%s/models/%s/generate-documentation-from-%s" % (self.mltask.project_key, self.mltask.analysis_id, self.mltask.mltask_id, self.mltask_model_id, template_mode_url), params={"folderId": folder_id, "path": path}) return DSSFuture(self.mltask.client, f["jobId"]) else: f = self.saved_model.client._perform_json( "POST", "/projects/%s/savedmodels/%s/versions/%s/generate-documentation-from-%s" % (self.saved_model.project_key, self.saved_model.sm_id, self.saved_model_version, template_mode_url), params={"folderId": folder_id, "path": path}) return DSSFuture(self.saved_model.client, job_id=f["jobId"]) def generate_documentation_from_custom_template(self, fp): """ Start the model document generation from a docx template (as a file object). :param object fp: A file-like object pointing to a template docx file :return: A :class:`~dataikuapi.dss.future.DSSFuture` representing the model document generation process """ files = {'file': fp} if self.mltask is not None: f = self.mltask.client._perform_json( "POST", "/projects/%s/models/lab/%s/%s/models/%s/generate-documentation-from-custom-template" % (self.mltask.project_key, self.mltask.analysis_id, self.mltask.mltask_id, self.mltask_model_id), files=files) return DSSFuture(self.mltask.client, f["jobId"]) else: f = self.saved_model.client._perform_json( "POST", "/projects/%s/savedmodels/%s/versions/%s/generate-documentation-from-custom-template" % (self.saved_model.project_key, self.saved_model.sm_id, self.saved_model_version), files=files) return DSSFuture(self.saved_model.client, job_id=f["jobId"]) def download_documentation_stream(self, export_id): """ Download a model documentation, as a binary stream. Warning: this stream will monopolize the DSSClient until closed. :param export_id: the id of the generated model documentation returned as the result of the future :return: A :class:`~dataikuapi.dss.future.DSSFuture` representing the model document generation process """ if self.mltask is not None: return self.mltask.client._perform_raw( "GET", "/projects/%s/models/lab/documentations/%s" % (self.mltask.project_key, export_id)) else: return self.saved_model.client._perform_raw( "GET", "/projects/%s/savedmodels/documentations/%s" % (self.saved_model.project_key, export_id)) def download_documentation_to_file(self, export_id, path): """ Download a model documentation into the given output file. :param export_id: the id of the generated model documentation returned as the result of the future :param path: the path where to download the model documentation :return: None """ stream = self.download_documentation_stream(export_id) _write_response_content_to_file(stream, path) class DSSMLDiagnostic(object): """ Object that represents a computed Diagnostic on a trained model .. important:: Do not create this object directly, use :meth:`DSSTrainedModelDetails.get_diagnostics()` instead """ def __init__(self, data): self._internal_dict = data def get_raw(self): """ Gets the raw dictionary of the diagnostic :rtype: dict """ return self._internal_dict def get_type(self): """ Returns the base Diagnostic type :rtype: str """ return self._internal_dict["type"] def get_type_pretty(self): """ Returns the Diagnostic type as displayed in the UI :rtype: str """ return self._internal_dict["displayableType"] def get_message(self): """ Returns the message as displayed in the UI :rtype: str """ return self._internal_dict["message"] def __repr__(self): return "{cls}(type={type}, message={msg})".format(cls=self.__class__.__name__, type=self._internal_dict["type"], msg=self._internal_dict["message"]) class DSSMLAssertionsParams(object): """ Object that represents parameters for all assertions of a ml task .. important:: Do not create this object directly, use :meth:`DSSPredictionMLTaskSettings.get_assertions_params` instead """ def __init__(self, data): self._internal_dict = data def __repr__(self): return u"{}({})".format(self.__class__.__name__, self.get_raw()) def get_raw(self): """ Gets the raw dictionary of the assertions parameters :rtype: dict """ return self._internal_dict def get_assertion(self, assertion_name): """ Gets a :class:`DSSMLAssertionParams` representing the parameters of the assertion with the provided name (or None if no assertion has that name) :param str assertion_name: Name of the assertion :rtype: :class:`DSSMLAssertionParams` or None if no assertion has that name """ for assertion_dict in self._internal_dict["assertions"]: if assertion_dict["name"] == assertion_name: return DSSMLAssertionParams(assertion_dict) return None def get_assertions_names(self): """ Gets the list of all assertions' names :return: A list of all assertions' names :rtype: list """ return [assertion_dict["name"] for assertion_dict in self._internal_dict["assertions"]] def add_assertion(self, assertion_params): """ Adds parameters of an assertion to the assertions parameters of the ml task. :param object assertion_params: A :class:`DSSMLAssertionParams` representing parameters of the assertion """ if not isinstance(assertion_params, DSSMLAssertionParams): raise ValueError('Wrong type for assertion parameters: {}'.format(type(assertion_params))) self._internal_dict["assertions"].append(assertion_params.get_raw()) def delete_assertion(self, assertion_name): """ Deletes the assertion parameters of the assertion with the provided name from the assertions parameters of the ml task. Raises a ValueError if no assertion with the provided name was found :param str assertion_name: Name of the assertion """ for idx, assertion_dict in enumerate(self._internal_dict["assertions"]): if assertion_dict["name"] == assertion_name: del self._internal_dict["assertions"][idx] return raise ValueError('No assertion found with name: {}'.format(assertion_name)) class DSSMLAssertionParams(object): """ Object that represents parameters for one assertion .. important:: Do not create this object directly, use :meth:`DSSMLAssertionsParams.get_assertion` or :meth:`from_params` instead """ def __init__(self, data): self._internal_dict = data def __repr__(self): return u"{}({})".format(self.__class__.__name__, self.get_raw()) @staticmethod def from_params(name, a_filter, condition): """ Creates assertion parameters from name, filter and condition :param str name: Name of the assertion :param dict a_filter: A dict representing the filter to select assertion population. You can use a :class:`~dataikuapi.dss.utils.DSSFilterBuilder` to build the settings of the filter :param object condition: A :class:`DSSMLAssertionCondition` for the assertion to be successful :rtype: :class:`DSSMLAssertionParams` """ assertion_params = DSSMLAssertionParams({}) assertion_params.name = name assertion_params.filter = a_filter assertion_params.condition = condition return assertion_params def get_raw(self): """ Gets the raw dictionary of the assertion parameters :rtype: dict """ return self._internal_dict @property def name(self): """ Returns the assertion name :rtype: str """ return self._internal_dict["name"] @name.setter def name(self, name): self._internal_dict["name"] = name @property def filter(self): """ Returns the assertion filter :rtype: dict """ return self._internal_dict["filter"] @filter.setter def filter(self, a_filter): self._internal_dict["filter"] = a_filter @property def condition(self): """ Returns the assertion condition :rtype: :class:`DSSMLAssertionCondition` """ return DSSMLAssertionCondition(self._internal_dict["assertionCondition"]) @condition.setter def condition(self, condition): if not isinstance(condition, DSSMLAssertionCondition): raise ValueError('Wrong type for assertion condition: {}'.format(type(condition))) self._internal_dict["assertionCondition"] = condition.get_raw() class DSSMLAssertionCondition(object): """ Object that represents an assertion condition .. important:: Do not create this object directly, use :meth:`DSSMLAssertionParams.condition`, :meth:`DSSMLAssertionCondition.from_expected_class` or :meth:`DSSMLAssertionCondition.from_expected_range` instead """ def __init__(self, data): self._internal_dict = data def __repr__(self): return u"{}({})".format(self.__class__.__name__, self.get_raw()) @staticmethod def from_expected_class(expected_valid_ratio, expected_class): """ Creates an assertion condition from the expected valid ratio and class :param float expected_valid_ratio: Assertion passes if this ratio of rows predicted as expected_class is attained :param str expected_class: Assertion passes if the ratio of rows predicted as expected_class is attained :rtype: :class:`DSSMLAssertionCondition` """ assertion_condition = DSSMLAssertionCondition({}) assertion_condition.expected_valid_ratio = expected_valid_ratio assertion_condition.expected_class = expected_class return assertion_condition @staticmethod def from_expected_range(expected_valid_ratio, expected_min, expected_max): """ Creates an assertion condition from expected valid ratio and range. The expected range is the interval between expected_min and expected_max (included) for the predictions in which the rows will be considered valid. :param float expected_valid_ratio: Assertion passes if this ratio of rows predicted between expected_min and expected_max (included) is attained :param float expected_min: Min value of the expected range :param float expected_max: Max value of the expected range :rtype: :class:`DSSMLAssertionCondition` """ assertion_condition = DSSMLAssertionCondition({}) assertion_condition.expected_valid_ratio = expected_valid_ratio assertion_condition.expected_min = expected_min assertion_condition.expected_max = expected_max return assertion_condition def get_raw(self): """ Gets the raw dictionary of the condition :rtype: dict """ return self._internal_dict @property def expected_class(self): """ Returns the expected class or None if it is not defined. Assertion passes if the expected_valid_ratio of rows predicted as expected_class is attained. :rtype: str """ if "expectedClass" in self._internal_dict: return self._internal_dict["expectedClass"] else: return None @expected_class.setter def expected_class(self, expected_class): self._internal_dict["expectedClass"] = expected_class @property def expected_valid_ratio(self): """ Returns the ratio of valid rows to exceed for the assertion to pass. A row is considered valid if the prediction is equal to the expected class for classification or in the expected range for regression. :rtype: str """ return self._internal_dict["expectedValidRatio"] @expected_valid_ratio.setter def expected_valid_ratio(self, expected_valid_ratio): self._internal_dict["expectedValidRatio"] = expected_valid_ratio @property def expected_min(self): """ Returns the min (included) of the expected range or None if it is not defined. Assertion passes if the expected_valid_ratio of rows predicted between expected_min and expected_max (included) is attained. :rtype: float """ if "expectedMinValue" in self._internal_dict: return self._internal_dict["expectedMinValue"] else: return None @expected_min.setter def expected_min(self, expected_min): self._internal_dict["expectedMinValue"] = expected_min @property def expected_max(self): """ Returns the max (included) of the expected range or None if it is not defined. Assertion passes if the expected_valid_ratio of rows predicted between expected_min and expected_max (included) is attained. :rtype: float """ if "expectedMaxValue" in self._internal_dict: return self._internal_dict["expectedMaxValue"] else: return None @expected_max.setter def expected_max(self, expected_max): self._internal_dict["expectedMaxValue"] = expected_max class DSSMLAssertionsMetrics(object): """ Object that represents the assertions metrics for all assertions on a trained model .. important:: Do not create this object directly, use :meth:`DSSTrainedPredictionModelDetails.get_assertions_metrics` instead """ def __init__(self, data): self._internal_dict = data def __repr__(self): return u"{}({})".format(self.__class__.__name__, self.get_raw()) def get_raw(self): """ Gets the raw dictionary of the assertions metrics :rtype: dict """ return self._internal_dict def get_metrics(self, assertion_name): """ Retrieves the metrics computed for this trained model for the assertion with the provided name (or None if no assertion with that name exists) :param str assertion_name: Name of the assertion :returns: an object representing assertion metrics or None if no assertion with that name exists :rtype: :class:`DSSMLAssertionMetrics` """ for assertion_metrics_dict in self._internal_dict["perAssertion"]: if assertion_name == assertion_metrics_dict["name"]: return DSSMLAssertionMetrics(assertion_metrics_dict) return None @property def passing_assertions_ratio(self): """ Returns the ratio of passing assertions :rtype: float """ return self._internal_dict['passingAssertionsRatio'] class DSSMLAssertionMetrics(object): """ Object that represents the result of an assertion on a trained model .. important:: Do not create this object directly, use :meth:`DSSMLAssertionMetrics.get_metrics` instead """ def __init__(self, data): self._internal_dict = data def __repr__(self): return u"{}(name='{}', result={}, valid_ratio={}, nb_matching_rows={}," \ u" nb_dropped_rows={})".format(self.__class__.__name__, self.name, self.result, self.valid_ratio, self.nb_matching_rows, self.nb_dropped_rows) def get_raw(self): """ Gets the raw dictionary of metrics of one assertion :rtype: dict """ return self._internal_dict @property def name(self): """ Returns the assertion name :rtype: str """ return self._internal_dict["name"] @property def result(self): """ Returns whether the assertion passes :rtype: bool """ return self._internal_dict["result"] @property def valid_ratio(self): """ Returns the ratio of rows in the assertion population with prediction equals to the expected class for classification or in the expected range for regression :rtype: float """ return self._internal_dict["validRatio"] @property def nb_matching_rows(self): """ Returns the number of rows matching filter :rtype: int """ return self._internal_dict["nbMatchingRows"] @property def nb_dropped_rows(self): """ Returns the number of rows matching filter and dropped by the model's preprocessing :rtype: int """ return self._internal_dict["nbDroppedRows"] class DSSTreeNode(object): def __init__(self, tree, i): self.tree = tree self.i = i def get_left_child(self): """Gets a :class:`dataikuapi.dss.ml.DSSTreeNode` representing the left side of the tree node (or None)""" left = self.tree.tree['leftChild'][self.i] if left < 0: return None else: return DSSTreeNode(self.tree, left) def get_right_child(self): """Gets a :class:`dataikuapi.dss.ml.DSSTreeNode` representing the right side of the tree node (or None)""" left = self.tree.tree['rightChild'][self.i] if left < 0: return None else: return DSSTreeNode(self.tree, left) def get_split_info(self): """Gets the information on the split, as a dict""" info = {} features = self.tree.tree.get("feature", None) probas = self.tree.tree.get("probas", None) leftCategories = self.tree.tree.get("leftCategories", None) impurities = self.tree.tree.get("impurity", None) predicts = self.tree.tree.get("predict", None) thresholds = self.tree.tree.get("threshold", None) nSamples = self.tree.tree.get("nSamples", None) info['feature'] = self.tree.feature_names[features[self.i]] if features is not None else None info['probas'] = probas[self.i] if probas is not None else None info['leftCategories'] = leftCategories[self.i] if leftCategories is not None else None info['impurity'] = impurities[self.i] if impurities is not None else None info['predict'] = predicts[self.i] if predicts is not None else None info['nSamples'] = nSamples[self.i] if nSamples is not None else None info['threshold'] = thresholds[self.i] if thresholds is not None else None return info class DSSTree(object): def __init__(self, tree, feature_names, prediction_type, classes): self.tree = tree self.feature_names = feature_names self.prediction_type = prediction_type self.classes = classes def get_raw(self): """Gets the raw tree data structure""" return self.tree def get_classes(self): if self.prediction_type not in {"BINARY_CLASSIFICATION", "MULTICLASS"}: logger.warning("Regression models do not have a `classes` attribute") return None return self.classes def get_root(self): """Gets a :class:`dataikuapi.dss.ml.DSSTreeNode` representing the root of the tree""" return DSSTreeNode(self, 0) class DSSTreeSet(object): def __init__(self, trees, prediction_type): self.trees = trees self.prediction_type = prediction_type def get_raw(self): """Gets the raw trees data structure""" return self.trees def get_feature_names(self): """Gets the list of feature names (after dummification) """ return self.trees["featureNames"] def get_classes(self): if self.prediction_type not in {"BINARY_CLASSIFICATION", "MULTICLASS"}: logger.warning("Regression models do not have a `classes` attribute") return None else: classes = self.trees.get("classes", None) if classes is None: logger.warning("Models trained on version prior to 13.1 do not have a `classes` attribute") return classes def get_trees(self): """Gets the list of trees as :class:`dataikuapi.dss.ml.DSSTree` """ if self.prediction_type in {"BINARY_CLASSIFICATION", "MULTICLASS"}: classes = self.get_classes() else: classes = None return [DSSTree(t, self.trees["featureNames"], self.prediction_type, classes) for t in self.trees["trees"]] class DSSCoefficientPaths(object): def __init__(self, paths): self.paths = paths def get_raw(self): """Gets the raw paths data structure""" return self.paths def get_feature_names(self): """Get the feature names (after dummification)""" return self.paths['features'] def get_coefficient_path(self, feature, class_index=0): """Get the path of the feature""" i = self.paths['features'].index(feature) if i >= 0 and i < len(self.paths['path'][0][class_index]): n = len(self.paths['path']) return [self.paths['path'][j][class_index][i] for j in range(0, n)] else: return None class DSSScatterPlots(object): def __init__(self, scatters): self.scatters = scatters def get_raw(self): """Gets the raw scatters data structure""" return self.scatters def get_feature_names(self): """Get the feature names (after dummification)""" feature_names = [] for k in self.scatters['features']: feature_names.append(k) return feature_names def get_scatter_plot(self, feature_x, feature_y): """Get the scatter plot between feature_x and feature_y""" ret = {'cluster':self.scatters['cluster'], 'x':self.scatters['features'].get(feature_x, None), 'y':self.scatters['features'].get(feature_x, None)} return ret class DSSTrainedPredictionModelDetails(DSSTrainedModelDetails): """ Object to read details of a trained prediction model .. important:: Do not create this object directly, use :meth:`DSSMLTask.get_trained_model_details()` instead """ def __init__(self, details, snippet, saved_model=None, saved_model_version=None, mltask=None, mltask_model_id=None): super(DSSTrainedPredictionModelDetails, self).__init__(details, snippet, saved_model, saved_model_version, mltask, mltask_model_id) def get_roc_curve_data(self): """ Gets the data used to plot the ROC curve for the model, if it exists. :return: A dictionary containing ROC curve data """ roc = self.details.get("perf", {}).get("rocVizData", {}) if roc is None: raise ValueError("This model does not have ROC visualization data") return roc def get_performance_metrics(self): """ Returns all performance metrics for this model. For binary classification model, this includes both "threshold-independent" metrics like AUC and "threshold-dependent" metrics like precision. Threshold-dependent metrics are returned at the threshold value that was found to be optimal during training. To get access to the per-threshold values, use the following: .. code-block:: python # Returns a list of tested threshold values details.get_performance()["perCutData"]["cut"] # Returns a list of F1 scores at the tested threshold values details.get_performance()["perCutData"]["f1"] # Both lists have the same length If K-Fold cross-test was used, most metrics will have a "std" variant, which is the standard deviation accross the K cross-tested folds. For example, "auc" will be accompanied with "aucstd" :returns: a dict of performance metrics values :rtype: dict """ import copy clean_snippet = copy.deepcopy(self.snippet) for x in ["gridsearchData", "trainDate", "topImportance", "backendType", "userMeta", "sessionDate", "trainInfo", "fullModelId", "gridLength", "algorithm", "sessionId"]: if x in clean_snippet: del clean_snippet[x] return clean_snippet def get_assertions_metrics(self): """ Retrieves assertions metrics computed for this trained model :returns: an object representing assertion metrics :rtype: :class:`DSSMLAssertionsMetrics` """ return DSSMLAssertionsMetrics(self.snippet["assertionsMetrics"]) def get_hyperparameter_search_points(self): """ Gets the list of points in the hyperparameter search space that have been tested. Returns a list of dict. Each entry in the list represents a point. For each point, the dict contains at least: - "score": the average value of the optimization metric over all the folds at this point - "params": a dict of the parameters at this point. This dict has the same structure as the params of the best parameters """ if not "gridCells" in self.details["iperf"]: raise ValueError("No hyperparameter search result, maybe this model did not perform hyperparameter optimization") return self.details["iperf"]["gridCells"] def get_preprocessing_settings(self): """ Gets the preprocessing settings that were used to train this model :rtype: dict """ return self.details["preprocessing"] def get_modeling_settings(self): """ Gets the modeling (algorithms) settings that were used to train this model. .. note:: The structure of this dict is not the same as the modeling params on the ML Task (which may contain several algorithms). :rtype: dict """ return self.details["modeling"] def get_actual_modeling_params(self): """ Gets the actual / resolved parameters that were used to train this model, post hyperparameter optimization. :return: A dictionary, which contains at least a "resolved" key, which is a dict containing the post-optimization parameters :rtype: dict """ return self.details["actualParams"] def get_trees(self): """ Gets the trees in the model (for tree-based models) :return: a DSSTreeSet object to interact with the trees :rtype: :class:`dataikuapi.dss.ml.DSSTreeSet` """ data = self.mltask.client._perform_json( "GET", "/projects/%s/models/lab/%s/%s/models/%s/trees" % (self.mltask.project_key, self.mltask.analysis_id, self.mltask.mltask_id, self.mltask_model_id)) if data is None: raise ValueError("This model has no tree data") prediction_type = self.details.get("coreParams", {}).get("prediction_type") return DSSTreeSet(data, prediction_type) def get_coefficient_paths(self): """ Gets the coefficient paths for Lasso models :return: a DSSCoefficientPaths object to interact with the coefficient paths :rtype: :class:`dataikuapi.dss.ml.DSSCoefficientPaths` """ data = self.mltask.client._perform_json( "GET", "/projects/%s/models/lab/%s/%s/models/%s/coef-paths" % (self.mltask.project_key, self.mltask.analysis_id, self.mltask.mltask_id, self.mltask_model_id)) if data is None: raise ValueError("This model has no coefficient paths") return DSSCoefficientPaths(data) ## Model export def get_scoring_jar_stream(self, model_class="model.Model", include_libs=False): """ Returns a stream of a scoring jar for this trained model. This works provided that you have the license to do so and that the model is compatible with optimized scoring. You need to close the stream after download. Failure to do so will result in the DSSClient becoming unusable. :param str model_class: fully-qualified class name, e.g. "com.company.project.Model" :param bool include_libs: if True, also packs the required dependencies; if False, runtime will require the scoring libs given by :func:`DSSClient.scoring_libs` :returns: a jar file, as a stream :rtype: file-like """ include_libs = "true" if include_libs else "false" if self.mltask is not None: return self.mltask.client._perform_raw( "GET", "/projects/%s/models/lab/%s/%s/models/%s/scoring-jar?fullClassName=%s&includeLibs=%s" % (self.mltask.project_key, self.mltask.analysis_id, self.mltask.mltask_id, self.mltask_model_id, model_class, include_libs)) else: return self.saved_model.client._perform_raw( "GET", "/projects/%s/savedmodels/%s/versions/%s/scoring-jar?fullClassName=%s&includeLibs=%s" % (self.saved_model.project_key, self.saved_model.sm_id, self.saved_model_version, model_class, include_libs)) def get_scoring_pmml_stream(self): """ Returns a stream of a scoring PMML for this trained model. This works provided that you have the license to do so and that the model is compatible with PMML scoring. You need to close the stream after download. Failure to do so will result in the DSSClient becoming unusable. :returns: a PMML file, as a stream :rtype: file-like """ if self.mltask is not None: return self.mltask.client._perform_raw( "GET", "/projects/%s/models/lab/%s/%s/models/%s/scoring-pmml" % (self.mltask.project_key, self.mltask.analysis_id, self.mltask.mltask_id, self.mltask_model_id)) else: return self.saved_model.client._perform_raw( "GET", "/projects/%s/savedmodels/%s/versions/%s/scoring-pmml" % (self.saved_model.project_key, self.saved_model.sm_id, self.saved_model_version)) def get_scoring_python_stream(self): """ Returns a stream of a zip file containing the required data to use this trained model in external python code. See: https://doc.dataiku.com/dss/latest/python-api/ml.html This works provided that you have the license to do so and that the model is compatible with Python scoring. You need to close the stream after download. Failure to do so will result in the DSSClient becoming unusable. :returns: an archive file, as a stream :rtype: file-like """ if self.mltask is not None: return self.mltask.client._perform_raw( "GET", "/projects/%s/models/lab/%s/%s/models/%s/scoring-python?mlflowExport=false" % (self.mltask.project_key, self.mltask.analysis_id, self.mltask.mltask_id, self.mltask_model_id)) else: return self.saved_model.client._perform_raw( "GET", "/projects/%s/savedmodels/%s/versions/%s/scoring-python?mlflowExport=false" % (self.saved_model.project_key, self.saved_model.sm_id, self.saved_model_version)) def get_scoring_python(self, filename): """ Downloads a zip file containing the required data to use this trained model in external python code. See: https://doc.dataiku.com/dss/latest/python-api/ml.html This works provided that you have the license to do so and that the model is compatible with Python scoring. :param str filename: filename of the resulting downloaded file """ with open(filename, "wb") as f: f.write(self.get_scoring_python_stream().content) def get_scoring_mlflow_stream(self): """ Returns a stream of a zip containing this trained model using the MLflow Model format. This works provided that you have the license to do so and that the model is compatible with MLflow scoring. You need to close the stream after download. Failure to do so will result in the DSSClient becoming unusable. :returns: an archive file, as a stream :rtype: file-like """ if self.mltask is not None: return self.mltask.client._perform_raw( "GET", "/projects/%s/models/lab/%s/%s/models/%s/scoring-python?mlflowExport=true" % (self.mltask.project_key, self.mltask.analysis_id, self.mltask.mltask_id, self.mltask_model_id)) else: return self.saved_model.client._perform_raw( "GET", "/projects/%s/savedmodels/%s/versions/%s/scoring-python?mlflowExport=true" % (self.saved_model.project_key, self.saved_model.sm_id, self.saved_model_version)) def get_scoring_mlflow(self, filename): """ Downloads a zip containing data for this trained model, using the MLflow Model format. This works provided that you have the license to do so and that the model is compatible with MLflow scoring. :param str filename: filename to the resulting MLflow Model zip """ with open(filename, "wb") as f: f.write(self.get_scoring_mlflow_stream().content) def export_to_snowflake_function(self, connection_name, function_name, wait=True): """ Exports the model to a Snowflake function. Only works for Saved Model Versions. :param connection_name: Snowflake connection to use :param function_name: Name of the function to create :param wait: a flag to wait for the operation to complete (defaults to **True**) :return: None if wait is True, else a future """ if self.mltask is not None: raise Exception("Only supported for Saved Model Versions") future_response = self.saved_model.client._perform_json( "GET", "/projects/%s/savedmodels/%s/versions/%s/export-to-snowflake-function" % (self.saved_model.project_key, self.saved_model.sm_id, self.saved_model_version), params={"connectionName": connection_name, "functionName": function_name} ) future = DSSFuture.from_resp(self.saved_model.client, future_response) return future.wait_for_result() if wait else future def export_to_databricks_registry(self, connection_name, use_unity_catalog, model_name, experiment_name, wait=True): """ Exports the model as a version of a Registered Model of a Databricks Registry. To do so, the model is exported to the MLflow format, then logged in a run of an experiment, and finally registered in the selected registry. :param connection_name: Databricks Model Deployment Infrastructure connection to use :param use_unity_catalog: exports to a model in the Databricks Workspace registry or in the Databricks Unity Catalog :param model_name: name of the model to add a version to. Restrictions apply on possible name ; please refer to Databricks documentation. The model will be created if needed. :param experiment_name: name of the experiment to use. The experiment will be created if needed. :param wait: a flag to wait for the operation to complete (defaults to **True**) :return: dict if wait is True, else a future """ params = {"connectionName": connection_name, "useUnityCatalog": use_unity_catalog, "modelName": model_name, "experimentName": experiment_name} if self.mltask is not None: future_response = self.mltask.client._perform_json( "POST", "/projects/%s/models/lab/%s/%s/models/%s/export-to-databricks-registry" % (self.mltask.project_key, self.mltask.analysis_id, self.mltask.mltask_id, self.mltask_model_id), params=params ) future = DSSFuture.from_resp(self.mltask.client, future_response) else: future_response = self.saved_model.client._perform_json( "POST", "/projects/%s/savedmodels/%s/versions/%s/export-to-databricks-registry" % (self.saved_model.project_key, self.saved_model.sm_id, self.saved_model_version), params=params ) future = DSSFuture.from_resp(self.saved_model.client, future_response) return future.wait_for_result() if wait else future ## Post-train computations def compute_shapley_feature_importance(self): """ Launches computation of Shapley feature importance for this trained model :returns: A future for the computation task :rtype: :class:`dataikuapi.dss.future.DSSFuture` """ if self.mltask is not None: future_response = self.mltask.client._perform_json( "POST", "/projects/%s/models/lab/%s/%s/models/%s/shapley-feature-importance" % (self.mltask.project_key, self.mltask.analysis_id, self.mltask.mltask_id, self.mltask_model_id), ) future = DSSFuture(self.mltask.client, future_response.get("jobId", None), future_response) else: future_response = self.saved_model.client._perform_json( "POST", "/projects/%s/savedmodels/%s/versions/%s/shapley-feature-importance" % (self.saved_model.project_key, self.saved_model.sm_id, self.saved_model_version), ) future = DSSFuture(self.saved_model.client, future_response.get("jobId", None), future_response) return future def compute_subpopulation_analyses(self, split_by, wait=True, sample_size=1000, random_state=1337, n_jobs=1, debug_mode=False): """ Launch computation of Subpopulation analyses for this trained model. :param list|str split_by: column(s) on which subpopulation analyses are to be computed (one analysis per column) :param bool wait: if True, the call blocks until the computation is finished and returns the results directly :param int sample_size: number of records of the dataset to use for the computation :param int random_state: random state to use to build sample, for reproducibility :param int n_jobs: number of cores used for parallel training. (-1 means 'all cores') :param bool debug_mode: if True, output all logs (slower) :returns: if wait is True, an object containing the Subpopulation analyses, else a future to wait on the result :rtype: Union[:class:`dataikuapi.dss.ml.DSSSubpopulationAnalyses`, :class:`dataikuapi.dss.future.DSSFuture`] """ body = { "features": split_by if isinstance(split_by, list) else [split_by], "computationParams": { "sample_size": sample_size, "random_state": random_state, "n_jobs": n_jobs, "debug_mode": debug_mode, }} if self.mltask is not None: future_response = self.mltask.client._perform_json( "POST", "/projects/%s/models/lab/%s/%s/models/%s/subpopulation-analyses" % (self.mltask.project_key, self.mltask.analysis_id, self.mltask.mltask_id, self.mltask_model_id), body=body ) future = DSSFuture(self.mltask.client, future_response.get("jobId", None), future_response) else: future_response = self.saved_model.client._perform_json( "POST", "/projects/%s/savedmodels/%s/versions/%s/subpopulation-analyses" % (self.saved_model.project_key, self.saved_model.sm_id, self.saved_model_version), body=body ) future = DSSFuture(self.saved_model.client, future_response.get("jobId", None), future_response) if wait: prediction_type = self.details.get("coreParams", {}).get("prediction_type") return DSSSubpopulationAnalyses(future.wait_for_result(), prediction_type) else: return future def get_subpopulation_analyses(self): """ Retrieve all subpopulation analyses computed for this trained model :returns: The subpopulation analyses :rtype: :class:`dataikuapi.dss.ml.DSSSubpopulationAnalyses` """ if self.mltask is not None: data = self.mltask.client._perform_json( "GET", "/projects/%s/models/lab/%s/%s/models/%s/subpopulation-analyses" % (self.mltask.project_key, self.mltask.analysis_id, self.mltask.mltask_id, self.mltask_model_id) ) else: data = self.saved_model.client._perform_json( "GET", "/projects/%s/savedmodels/%s/versions/%s/subpopulation-analyses" % (self.saved_model.project_key, self.saved_model.sm_id, self.saved_model_version), ) prediction_type = self.details.get("coreParams", {}).get("prediction_type") return DSSSubpopulationAnalyses(data, prediction_type) def compute_partial_dependencies(self, features, wait=True, sample_size=1000, random_state=1337, n_jobs=1, debug_mode=False): """ Launch computation of Partial dependencies for this trained model. :param list|str features: feature(s) on which partial dependencies are to be computed :param bool wait: if True, the call blocks until the computation is finished and returns the results directly :param int sample_size: number of records of the dataset to use for the computation :param int random_state: random state to use to build sample, for reproducibility :param int n_jobs: number of cores used for parallel training. (-1 means 'all cores') :param bool debug_mode: if True, output all logs (slower) :returns: if wait is True, an object containing the Partial dependencies, else a future to wait on the result :rtype: Union[:class:`dataikuapi.dss.ml.DSSPartialDependencies`, :class:`dataikuapi.dss.future.DSSFuture`] """ body = { "features": features if isinstance(features, list) else [features], "computationParams": { "sample_size": sample_size, "random_state": random_state, "n_jobs": n_jobs, "debug_mode": debug_mode, }} if self.mltask is not None: future_response = self.mltask.client._perform_json( "POST", "/projects/%s/models/lab/%s/%s/models/%s/partial-dependencies" % (self.mltask.project_key, self.mltask.analysis_id, self.mltask.mltask_id, self.mltask_model_id), body=body ) future = DSSFuture(self.mltask.client, future_response.get("jobId", None), future_response) else: future_response = self.saved_model.client._perform_json( "POST", "/projects/%s/savedmodels/%s/versions/%s/partial-dependencies" % (self.saved_model.project_key, self.saved_model.sm_id, self.saved_model_version), body=body ) future = DSSFuture(self.saved_model.client, future_response.get("jobId", None), future_response) if wait: return DSSPartialDependencies(future.wait_for_result()) else: return future def get_partial_dependencies(self): """ Retrieve all partial dependencies computed for this trained model :returns: The partial dependencies :rtype: :class:`dataikuapi.dss.ml.DSSPartialDependencies` """ if self.mltask is not None: data = self.mltask.client._perform_json( "GET", "/projects/%s/models/lab/%s/%s/models/%s/partial-dependencies" % (self.mltask.project_key, self.mltask.analysis_id, self.mltask.mltask_id, self.mltask_model_id) ) else: data = self.saved_model.client._perform_json( "GET", "/projects/%s/savedmodels/%s/versions/%s/partial-dependencies" % (self.saved_model.project_key, self.saved_model.sm_id, self.saved_model_version), ) return DSSPartialDependencies(data) class DSSSubpopulationGlobal(object): """ Object to read details of performance on global population used for subpopulation analyses. .. important:: Do not create this object directly, use :meth:`DSSSubpopulationAnalyses.get_global()` instead """ def __init__(self, data, prediction_type): self._internal_dict = data self.prediction_type = prediction_type def get_raw(self): """ Gets the raw dictionary of the global subpopulation performance :rtype: dict """ return self._internal_dict def __repr__(self): return "{cls}(prediction_type={type})".format(cls=self.__class__.__name__, type=self.prediction_type) def get_performance_metrics(self): """ Gets the performance results of the global population used for the subpopulation analysis """ return self._internal_dict["performanceMetrics"] def get_prediction_info(self): """ Gets the prediction info of the global population used for the subpopulation analysis """ global_metrics = self._internal_dict["perf"]["globalMetrics"] if self.prediction_type == "BINARY_CLASSIFICATION": return { "predictedPositiveRatio": global_metrics["predictionAvg"][0], "actualPositiveRatio": global_metrics["targetAvg"][0], "testWeight": global_metrics["testWeight"] } elif self.prediction_type == "REGRESSION": return { "predictedAvg":global_metrics["predictionAvg"][0], "predictedStd":global_metrics["predictionStd"][0], "actualAvg":global_metrics["targetAvg"][0], "actualStd":global_metrics["targetStd"][0], "testWeight":global_metrics["testWeight"] } class DSSSubpopulationModality(object): """ Object to read details of a subpopulation analysis modality .. important:: Do not create this object directly, use :meth:`DSSSubpopulationAnalysis.get_modality_data(definition)` instead """ def __init__(self, feature_name, computed_as_type, data, prediction_type): self._internal_dict = data self.prediction_type = prediction_type if computed_as_type == "CATEGORY": self.definition = DSSSubpopulationCategoryModalityDefinition(feature_name, data) elif computed_as_type == "NUMERIC": self.definition = DSSSubpopulationNumericModalityDefinition(feature_name, data) def get_raw(self): """ Gets the raw dictionary of the subpopulation analysis modality :rtype: dict """ return self._internal_dict def __repr__(self): computed_as_type = "CATEGORY" if isinstance(self.definition, DSSSubpopulationCategoryModalityDefinition) else 'NUMERIC' return "{cls}(prediction_type={type}, feature={feature}, computed_as={computed_as_type})".format( cls=self.__class__.__name__, type=self.prediction_type, feature=self.definition.feature_name, computed_as_type=computed_as_type) def get_definition(self): """ Gets the definition of the subpopulation analysis modality :returns: definition :rtype: :class:`dataikuapi.dss.ml.DSSSubpopulationModalityDefinition` """ return self.definition def is_excluded(self): """ Whether modality has been excluded from analysis (e.g. too few rows in the subpopulation) """ return self._internal_dict.get("excluded", False) def get_performance_metrics(self): """ Gets the performance results of the modality """ if self.is_excluded(): raise ValueError("Excluded modalities do not have performance metrics") return self._internal_dict["performanceMetrics"] def get_prediction_info(self): """ Gets the prediction info of the modality """ if self.is_excluded(): raise ValueError("Excluded modalities do not have prediction info") global_metrics = self._internal_dict["perf"]["globalMetrics"] if self.prediction_type == "BINARY_CLASSIFICATION": return { "predictedPositiveRatio": global_metrics["predictionAvg"][0], "actualPositiveRatio": global_metrics["targetAvg"][0], "testWeight": global_metrics["testWeight"] } elif self.prediction_type == "REGRESSION": return { "predictedAvg":global_metrics["predictionAvg"][0], "predictedStd":global_metrics["predictionStd"][0], "actualAvg":global_metrics["targetAvg"][0], "actualStd":global_metrics["targetStd"][0], "testWeight":global_metrics["testWeight"] } class DSSSubpopulationModalityDefinition(object): MISSING_VALUES = "__DSSSubpopulationModalidityDefinition__MISSINGVALUES" def __init__(self, feature_name, data): self.missing_values = data.get("missing_values", False) self.index = data.get("index") self.feature_name = feature_name def is_missing_values(self): return self.missing_values class DSSSubpopulationNumericModalityDefinition(DSSSubpopulationModalityDefinition): def __init__(self, feature_name, data): super(DSSSubpopulationNumericModalityDefinition, self).__init__(feature_name, data) self.lte = data.get("lte", None) self.gt = data.get("gt", None) self.gte = data.get("gte", None) def contains(self, value): lte = self.lte if self.lte is not None else float("inf") gt = self.gt if self.gt is not None else float("-inf") gte = self.gte if self.gte is not None else float("-inf") return not self.missing_values and gt < value and gte <= value and lte >= value def __repr__(self): if self.missing_values: return "DSSSubpopulationNumericModalityDefinition(missing_values)" else: if self.gt is not None: repr_gt = "%s<" % self.gt elif self.gte is not None: repr_gt = "%s<=" % self.gte else: repr_gt = "" if self.lte is not None: repr_lt = "<=%s" % self.lte else: repr_lt = "" return "DSSSubpopulationNumericModalityDefinition(%s%s%s)" % (repr_gt, self.feature_name, repr_lt) class DSSSubpopulationCategoryModalityDefinition(DSSSubpopulationModalityDefinition): def __init__(self, feature_name, data): super(DSSSubpopulationCategoryModalityDefinition, self).__init__(feature_name, data) self.value = data.get("value", None) def contains(self, value): return value == self.value def __repr__(self): if self.missing_values: return "DSSSubpopulationCategoryModalityDefinition(missing_values)" else: return "DSSSubpopulationCategoryModalityDefinition(%s='%s')" % (self.feature_name, self.value) class DSSSubpopulationAnalysis(object): """ Object to read details of a subpopulation analysis of a trained model .. important:: Do not create this object directly, use :meth:`DSSSubpopulationAnalyses.get_analysis(feature)` instead """ def __init__(self, analysis, prediction_type): self._internal_dict = analysis self.computed_as_type = analysis["computed_as_type"] self.modalities = [DSSSubpopulationModality(analysis["feature"], self.computed_as_type, m, prediction_type) for m in analysis.get("modalities", [])] def get_raw(self): """ Gets the raw dictionary of the subpopulation analysis :rtype: dict """ return self._internal_dict def __repr__(self): return "{cls}(computed_as_type={type}, feature={feature}, modalities_count={modalities_count})".format( cls=self.__class__.__name__, type=self.computed_as_type, feature=self._internal_dict["feature"], modalities_count=len(self.modalities)) def get_computation_params(self): """ Gets computation params """ return { "nbRecords": self._internal_dict["nbRecords"], "randomState": self._internal_dict["randomState"], "onSample": self._internal_dict["onSample"] } def list_modalities(self): """ List definitions of modalities """ return [m.definition for m in self.modalities] def get_modality_data(self, definition): """ Retrieves modality from definition :param definition: definition of modality to retrieve. Can be: * :class:`dataikuapi.dss.ml.DSSSubpopulationModalityDefinition` * `dataikuapi.dss.ml.DSSSubpopulationModalityDefinition.MISSING_VALUES` to retrieve modality corresponding to missing values * for category modality, can be a str corresponding to the value of the modality * for numeric modality, can be a number inside the modality :returns: the modality :rtype: :class:`dataikuapi.dss.ml.DSSSubpopulationModality` """ if definition == DSSSubpopulationModalityDefinition.MISSING_VALUES: for m in self.modalities: if m.definition.missing_values: return m raise ValueError("No 'missing values' modality found") if isinstance(definition, DSSSubpopulationModalityDefinition): modality_candidates = [m for m in self.modalities if m.definition.index == definition.index] if len(modality_candidates) == 0: raise ValueError("Modality with index '%s' not found" % definition.index) return modality_candidates[0] for m in self.modalities: if m.definition.contains(definition): return m raise ValueError("Modality not found: %s" % definition) class DSSSubpopulationAnalyses(object): """ Object to read details of subpopulation analyses of a trained model .. important:: Do not create this object directly, use :meth:`DSSTrainedPredictionModelDetails.get_subpopulation_analyses()` instead """ def __init__(self, data, prediction_type): self._internal_dict = data self.prediction_type = prediction_type self.analyses = [] for analysis in data.get("subpopulationAnalyses", []): self.analyses.append(DSSSubpopulationAnalysis(analysis, prediction_type)) def get_raw(self): """ Gets the raw dictionary of subpopulation analyses :rtype: dict """ return self._internal_dict def __repr__(self): return "{cls}(prediction_type={type}, analyses={analyses})".format(cls=self.__class__.__name__, type=self.prediction_type, analyses=self.list_analyses()) def get_global(self): """ Retrieves information and performance on the full dataset used to compute the subpopulation analyses """ return DSSSubpopulationGlobal(self._internal_dict["global"], self.prediction_type) def list_analyses(self): """ Lists all features on which subpopulation analyses have been computed """ return [analysis.get_raw()["feature"] for analysis in self.analyses] def get_analysis(self, feature): """ Retrieves the subpopulation analysis for a particular feature """ try: return next(analysis for analysis in self.analyses if analysis.get_raw()["feature"] == feature) except StopIteration: raise ValueError("Subpopulation analysis for feature '%s' cannot be found" % feature) class DSSPartialDependence(object): """ Object to read details of partial dependence of a trained model .. important:: Do not create this object directly, use :meth:`DSSPartialDependencies.get_partial_dependence(feature)` instead """ def __init__(self, data): self._internal_dict = data def get_raw(self): """ Gets the raw dictionary of the partial dependence :rtype: dict """ return self._internal_dict def __repr__(self): return "{cls}(feature={feature})".format(cls=self.__class__.__name__, feature=self._internal_dict["feature"]) def get_computation_params(self): """ Gets computation params """ return { "nbRecords": self._internal_dict["nbRecords"], "randomState": self._internal_dict["randomState"], "onSample": self._internal_dict["onSample"] } class DSSPartialDependencies(object): """ Object to read details of partial dependencies of a trained model .. important:: Do not create this object directly, use :meth:`DSSTrainedPredictionModelDetails.get_partial_dependencies()` instead """ def __init__(self, data): self._internal_dict = data self.partial_dependencies = [] for pd in data.get("partialDependencies", []): self.partial_dependencies.append(DSSPartialDependence(pd)) def get_raw(self): """ Gets the raw dictionary of partial dependencies :rtype: dict """ return self._internal_dict def __repr__(self): return "{cls}(features={features})".format(cls=self.__class__.__name__, features=self.list_features()) def list_features(self): """ Lists all features on which partial dependencies have been computed """ return [partial_dep.get_raw()["feature"] for partial_dep in self.partial_dependencies] def get_partial_dependence(self, feature): """ Retrieves the partial dependencies for a particular feature """ try: return next(pd for pd in self.partial_dependencies if pd.get_raw()["feature"] == feature) except StopIteration: raise ValueError("Partial dependence for feature '%s' cannot be found" % feature) class DSSClustersFacts(object): def __init__(self, clusters_facts): self.clusters_facts = clusters_facts def get_raw(self): """Gets the raws facts data structure""" return self.clusters_facts def get_cluster_size(self, cluster_index): """Gets the size of a cluster identified by its index""" return self.clusters_facts["clusters"][cluster_index]["size"] def get_facts_for_cluster(self, cluster_index): """ Gets all facts for a cluster identified by its index. Returns a list of dicts :rtype: list """ return self.clusters_facts["clusters"][cluster_index]["facts"] def get_facts_for_cluster_and_feature(self, cluster_index, feature_name): """ Gets all facts for a cluster identified by its index, limited to a feature name. Returns a list of dicts :rtype: list """ return [x for x in self.get_facts_for_cluster(cluster_index) if x["feature_label"] == feature_name] class DSSTrainedClusteringModelDetails(DSSTrainedModelDetails): """ Object to read details of a trained clustering model .. important:: Do not create this class directly, use :meth:`DSSMLTask.get_trained_model_details()` """ def __init__(self, details, snippet, saved_model=None, saved_model_version=None, mltask=None, mltask_model_id=None): super(DSSTrainedClusteringModelDetails, self).__init__(details, snippet, saved_model, saved_model_version, mltask, mltask_model_id) def get_raw(self): """ Gets the raw dictionary of trained model details :returns: A dictionary containing the trained model details :rtype: dict """ return self.details def get_train_info(self): """ Gets various information about the training process. This includes information such as the size of the train set, the quick description and timing information etc. :returns: A dictionary containing the models training information :rtype: dict """ return self.details["trainInfo"] def get_facts(self): """ Gets the 'cluster facts' data. The cluster facts data is the structure behind the screen "for cluster X, average of Y is Z times higher than average". :returns: The clustering facts data :rtype: :class:`DSSClustersFacts` """ return DSSClustersFacts(self.details["facts"]) def get_performance_metrics(self): """ Returns all performance metrics for this clustering model. :returns: A dict of performance metrics values :rtype: dict """ import copy clean_snippet = copy.deepcopy(self.snippet) for x in ["fullModelId", "algorithm", "trainInfo", "userMeta", "backendType", "sessionId", "sessionDate", "facts"]: if x in clean_snippet: del clean_snippet[x] return clean_snippet def get_preprocessing_settings(self): """ Gets the preprocessing settings that were used to train this model :returns: The model preprocessing settings :rtype: dict """ return self.details["preprocessing"] def get_modeling_settings(self): """ Gets the modeling (algorithms) settings that were used to train this model. .. note:: The structure of this dict is not the same as the modeling params on the ML Task (which may contain several algorithms). :returns: The model modeling settings :rtype: dict """ return self.details["modeling"] def get_actual_modeling_params(self): """ Gets the actual / resolved parameters that were used to train this model. :return: A dictionary, which contains at least a "resolved" key :rtype: dict """ return self.details["actualParams"] def get_scatter_plots(self): """ Gets the cluster scatter plot data :return: a DSSScatterPlots object to interact with the scatter plots :rtype: :class:`dataikuapi.dss.ml.DSSScatterPlots` """ scatters = self.mltask.client._perform_json( "GET", "/projects/%s/models/lab/%s/%s/models/%s/scatter-plots" % (self.mltask.project_key, self.mltask.analysis_id, self.mltask.mltask_id, self.mltask_model_id)) return DSSScatterPlots(scatters) class DSSMLTask(object): """ A handle to interact with a ML Task for prediction or clustering in a DSS visual analysis. .. important:: To create a new ML Task, use one of the following methods: :meth:`dataikuapi.dss.project.DSSProject.create_prediction_ml_task`, :meth:`dataikuapi.dss.project.DSSProject.create_clustering_ml_task` or :meth:`dataikuapi.dss.project.DSSProject.create_timeseries_forecasting_ml_task`. """ @staticmethod def from_full_model_id(client, fmi, project_key=None): """ Static method returning a DSSMLTask object representing a pre-existing ML Task :param client: An instantiated DSSClient :type client: :class:`~dataikuapi.dssclient.DSSClient` :param fmi: The Full Model ID of the ML Task :type fmi: str :param project_key: The project key of the project containing the ML Task (defaults to **None**) :type project_key: str, optional :return: The DSSMLTask :rtype: :class:`DSSMLTask` """ match = re.match(r"^A-(\w+)-(\w+)-(\w+)-(s[0-9]+)-(pp[0-9]+(-part-(\w+)|-base)?)-(m[0-9]+)$", fmi) if match is None: return DataikuException("Invalid model id: {}".format(fmi)) else: if project_key is None: project_key = match.group(1) return DSSMLTask(client, project_key, match.group(2), match.group(3)) def __init__(self, client, project_key, analysis_id, mltask_id): self.client = client self.project_key = project_key self.analysis_id = analysis_id self.mltask_id = mltask_id def delete(self): """ Deletes the ML task """ return self.client._perform_json( "DELETE", "/projects/%s/models/lab/%s/%s/" % (self.project_key, self.analysis_id, self.mltask_id)) def wait_guess_complete(self): """ Waits for the ML Task guessing to be complete. This should be called immediately after the creation of a new ML Task if the ML Task was created with ``wait_guess_complete = False``, before calling :meth:`get_settings` or :meth:`train`. """ while True: status = self.get_status() if status.get("guessing", "???") == False: break time.sleep(0.2) def get_status(self): """ Gets the status of this ML Task :return: A dictionary containing the ML Task status :rtype: dict """ return self.client._perform_json( "GET", "/projects/%s/models/lab/%s/%s/status" % (self.project_key, self.analysis_id, self.mltask_id)) def get_settings(self): """ Gets the settings of this ML Task. This should be used whenever you need to modify the settings of an existing ML Task. :return: A DSSMLTaskSettings object. :rtype: :class:`dataikuapi.dss.ml.DSSMLTaskSettings` """ settings = self.client._perform_json( "GET", "/projects/%s/models/lab/%s/%s/settings" % (self.project_key, self.analysis_id, self.mltask_id)) if settings["taskType"] == "PREDICTION": if settings["predictionType"] == DSSTimeseriesForecastingMLTaskSettings.PredictionTypes.TIMESERIES_FORECAST: return DSSTimeseriesForecastingMLTaskSettings(self.client, self.project_key, self.analysis_id, self.mltask_id, settings) elif settings["predictionType"] in [DSSCausalPredictionMLTaskSettings.PredictionTypes.CAUSAL_BINARY_CLASSIFICATION, DSSCausalPredictionMLTaskSettings.PredictionTypes.CAUSAL_REGRESSION]: return DSSCausalPredictionMLTaskSettings(self.client, self.project_key, self.analysis_id, self.mltask_id, settings) else: return DSSPredictionMLTaskSettings(self.client, self.project_key, self.analysis_id, self.mltask_id, settings) else: return DSSClusteringMLTaskSettings(self.client, self.project_key, self.analysis_id, self.mltask_id, settings) def train(self, session_name=None, session_description=None, run_queue=False): """ Trains models for this ML Task. This method waits for training to complete. If you instead want to train asynchronously, use :meth:`start_train` and :meth:`wait_train_complete`. This method returns a list of trained model identifiers. These refer to models that have been trained during this specific training session, rather than all of the trained models available on this ML task. To get all identifiers for all models trained across all training sessions, use :meth:`get_trained_models_ids`. These identifiers can be used for :meth:`get_trained_model_snippet`, :meth:`get_trained_model_details` and :meth:`deploy_to_flow`. :param session_name: Optional name for the session (defaults to **None**) :type session_name: str, optional :param session_description: Optional description for the session (defaults to **None**) :type session_description: str, optional :param bool run_queue: Whether to run any queued sessions after the training completes (defaults to **False**) :return: A list of model identifiers :rtype: list[str] """ train_ret = self.start_train(session_name, session_description, run_queue) self.wait_train_complete() return self.get_trained_models_ids(session_id = train_ret["sessionId"]) def ensemble(self, model_ids, method): """ Creates an ensemble model from a set of models. This method waits for the ensemble training to complete. If you want to train asynchronously, use :meth:`start_ensembling` and :meth:`wait_train_complete`. This method returns the identifier of the trained ensemble. To get all identifiers for all models trained across all training sessions, use :meth:`get_trained_models_ids`. This returned identifier can be used for :meth:`get_trained_model_snippet`, :meth:`get_trained_model_details` and :meth:`deploy_to_flow`. :param model_ids: A list of model identifiers to ensemble (must not be empty) :type model_ids: list[str] :param method: The ensembling method. Must be one of: AVERAGE, PROBA_AVERAGE, MEDIAN, VOTE, LINEAR_MODEL, LOGISTIC_MODEL :type method: str :return: The model identifier of the resulting ensemble model :rtype: str """ train_ret = self.start_ensembling(model_ids, method) self.wait_train_complete() return train_ret def start_train(self, session_name=None, session_description=None, run_queue=False): """ Starts asynchronously a new training session for this ML Task. This returns immediately, before training is complete. To wait for training to complete, use :meth:`wait_train_complete`. :param session_name: Optional name for the session (defaults to **None**) :type session_name: str, optional :param session_description: Optional description for the session (defaults to **None**) :type session_description: str, optional :param bool run_queue: Whether to run any queued sessions after the training completes (defaults to **False**) """ session_info = { "sessionName": session_name, "sessionDescription": session_description, "runQueue": run_queue } return self.client._perform_json( "POST", "/projects/%s/models/lab/%s/%s/train" % (self.project_key, self.analysis_id, self.mltask_id), body=session_info) def start_ensembling(self, model_ids, method): """ Creates asynchronously an ensemble model from a set of models This returns immediately, before training is complete. To wait for training to complete, use :meth:`wait_train_complete` :param model_ids: A list of model identifiers to ensemble (must not be empty) :type model_ids: list[str] :param method: The ensembling method. Must be one of: AVERAGE, PROBA_AVERAGE, MEDIAN, VOTE, LINEAR_MODEL, LOGISTIC_MODEL :type method: str :return: The model identifier of the ensemble :rtype: str """ if not model_ids: raise ValueError("The list of models to ensemble must not be empty") if method not in ["AVERAGE", "PROBA_AVERAGE", "MEDIAN", "VOTE", "LINEAR_MODEL", "LOGISTIC_MODEL"]: raise ValueError("A valid ensembling method must be chosen") ensembling_request = { "method" : method, "modelsIds" : model_ids } return self.client._perform_json( "POST", "/projects/%s/models/lab/%s/%s/ensemble" % (self.project_key, self.analysis_id, self.mltask_id), body=ensembling_request)['id'] def wait_train_complete(self): """ Waits for training to be completed To be used following any asynchronous training started with :meth:`start_train` or :meth:`start_ensembling` """ while True: status = self.get_status() if status.get("training", "???") == False: break time.sleep(2) def get_trained_models_ids(self, session_id=None, algorithm=None): """ Gets the list of trained model identifiers for this ML task. These identifiers can be used for :meth:`get_trained_model_snippet` and :meth:`deploy_to_flow`. The two optional filter params can be used together. :param session_id: Optional filter to return only IDs of models from a specific session. :type session_id: str, optional :param algorithm: Optional filter to return only IDs of models with a specific algorithm. :type algorithm: str, optional :return: A list of model identifiers :rtype: list[str] """ full_model_ids = self.get_status()["fullModelIds"] if session_id is not None: full_model_ids = [fmi for fmi in full_model_ids if fmi.get('fullModelId', {}).get('sessionId', '') == session_id] model_ids = [x["id"] for x in full_model_ids] if algorithm is not None: # algorithm is in the snippets model_ids = [fmi for fmi, s in self.get_trained_model_snippet(ids=model_ids).iteritems() if s.get("algorithm", "") == algorithm] return model_ids def get_trained_model_snippet(self, id=None, ids=None): """ Gets a quick summary of a trained model, as a dict. This method can either be given a single model id, via the `id` param, or a list of model ids, via the `ids` param. For complete model information and a structured object, use :meth:`get_trained_model_details`. :param id: A model id (defaults to **None**) :type id: str, optional :param ids: A list of model ids (defaults to **None**) :type ids: list[str] :return: Either a quick summary of one trained model as a dict, or a list of model summary dicts :rtype: Union[dict, list[dict]] """ if id is not None: obj = { "modelsIds" : [id] } elif ids is not None: obj = { "modelsIds" : ids } else: obj = {} ret = self.client._perform_json( "GET", "/projects/%s/models/lab/%s/%s/models-snippets" % (self.project_key, self.analysis_id, self.mltask_id), body = obj) if id is not None: return ret[id] else: return ret def get_trained_model_details(self, id): """ Gets details for a trained model. :param str id: Identifier of the trained model, as returned by :meth:`get_trained_models_ids` :return: A :class:`DSSTrainedPredictionModelDetails` or :class:`DSSTrainedClusteringModelDetails` representing the details of this trained model. :rtype: Union[:class:`DSSTrainedPredictionModelDetails`, :class:`DSSTrainedClusteringModelDetails`] """ ret = self.client._perform_json( "GET", "/projects/%s/models/lab/%s/%s/models/%s/details" % (self.project_key, self.analysis_id, self.mltask_id,id)) snippet = self.get_trained_model_snippet(id) if "facts" in ret: return DSSTrainedClusteringModelDetails(ret, snippet, mltask=self, mltask_model_id=id) else: return DSSTrainedPredictionModelDetails(ret, snippet, mltask=self, mltask_model_id=id) def delete_trained_model(self, model_id): """ Deletes a trained model :param str model_id: Model identifier, as returned by :meth:`get_trained_models_ids`. """ self.client._perform_empty( "DELETE", "/projects/%s/models/lab/%s/%s/models/%s" % (self.project_key, self.analysis_id, self.mltask_id, model_id)) def train_queue(self): """ Trains each session in this ML Task's queue, or until the queue is paused. :return: A dict including the next sessionID to be trained in the queue :rtype: dict """ return self.client._perform_json( "POST", "/projects/%s/models/lab/%s/%s/actions/train-queue" % (self.project_key, self.analysis_id, self.mltask_id)) def deploy_to_flow(self, model_id, model_name, train_dataset, test_dataset=None, redo_optimization=True): """ Deploys a trained model from this ML Task to the flow. Creates a new saved model and its parent training recipe in the Flow. :param str model_id: Model identifier, as returned by :meth:`get_trained_models_ids` :param str model_name: Name of the saved model when deployed to the Flow :param str train_dataset: Name of the dataset to use as train set. May either be a short name or a PROJECT.name long name (when using a shared dataset) :param test_dataset: Name of the dataset to use as test set. If None (default), the train/test split will be applied over the train set. Only for PREDICTION tasks. May either be a short name or a PROJECT.name long name (when using a shared dataset). :type test_dataset: str, optional :param bool redo_optimization: Whether to redo the hyperparameter optimization phase (defaults to **True**). Only for PREDICTION tasks. :return: A dict containing: "savedModelId" and "trainRecipeName" - Both can be used to obtain further handles :rtype: dict """ obj = { "trainDatasetRef" : train_dataset, "testDatasetRef" : test_dataset, "modelName" : model_name, "redoOptimization": redo_optimization } return self.client._perform_json( "POST", "/projects/%s/models/lab/%s/%s/models/%s/actions/deployToFlow" % (self.project_key, self.analysis_id, self.mltask_id, model_id), body=obj) def redeploy_to_flow(self, model_id, recipe_name=None, saved_model_id=None, activate=True, redo_optimization=False, redo_threshold_optimization=True, fixed_threshold=None): """ Redeploys a trained model from this ML Task to an existing saved model and training recipe in the flow. Either the training recipe `recipe_name` or the `saved_model_id` needs to be specified. :param str model_id: Model identifier, as returned by :meth:`get_trained_models_ids` :param recipe_name: Name of the training recipe to update (defaults to **None**) :type recipe_name: str, optional :param saved_model_id: Name of the saved model to update (defaults to **None**) :type saved_model_id: str, optional :param bool activate: If True (default), make the newly deployed model version become the active version :param bool redo_optimization: Whether to re-run the model optimization (hyperparameter search) on every train :param bool redo_threshold_optimization: Whether to redo the model threshold Optimization on every train (for binary classification models) :param fixed_threshold: Value to use as fixed threshold. Must be set if redoThresholdOptimization is False (for binary classification models) :type fixed_threshold: float, optional :return: A dict containing: "impactsDownstream" - whether the active saved mode version changed and downstream recipes are impacted :rtype: dict """ obj = { "recipeName": recipe_name, "savedModelId": saved_model_id, "activate": activate, "redoOptimization": redo_optimization, "redoThresholdOptimization": redo_threshold_optimization } if fixed_threshold is not None: obj["fixedThreshold"] = fixed_threshold return self.client._perform_json( "POST", "/projects/%s/models/lab/%s/%s/models/%s/actions/redeployToFlow" % (self.project_key, self.analysis_id, self.mltask_id, model_id), body=obj) def remove_unused_splits(self): """ Deletes all stored split data that is no longer in use for this ML Task. You should generally not need to call this method. """ self.client._perform_empty( "POST", "/projects/%s/models/lab/%s/%s/actions/removeUnusedSplits" % (self.project_key, self.analysis_id, self.mltask_id)) def remove_all_splits(self): """ Deletes all stored split data for this ML Task. This operation saves disk space. After performing this operation, it will not be possible anymore to: * Ensemble already trained models * View the "predicted data" or "charts" for already trained models * Resume training of models for which optimization had been previously interrupted Training new models remains possible """ self.client._perform_empty( "POST", "/projects/%s/models/lab/%s/%s/actions/removeAllSplits" % (self.project_key, self.analysis_id, self.mltask_id)) def guess(self, prediction_type=None, reguess_level=None, target_variable=None, timeseries_identifiers=None, time_variable=None, full_reguess=None): """ Reguess the settings of the ML Task. When no optional parameters are given, this will reguess all the settings of the ML Task. For prediction ML tasks only, a new target variable or prediction type can be passed, and this will subsequently reguess the impacted settings. :param prediction_type: The desired prediction type. Only valid for prediction tasks of either `BINARY_CLASSIFICATION`, `MULTICLASS` or `REGRESSION` type, ignored otherwise. Cannot be set if either target_variable, time_variable, or timeseries_identifiers is also specified. (defaults to **None**) :type prediction_type: str, optional :param target_variable: The desired target variable. Only valid for prediction tasks, ignored for clustering. Cannot be set if either prediction_type, time_variable, or timeseries_identifiers is also specified. (defaults to **None**) :type target_variable: str, optional :param timeseries_identifiers: Only valid for time series forecasting tasks. List of columns to be used as time series identifiers. Cannot be set if either prediction_type, target_variable, or time_variable is also specified. (defaults to **None**) :type timeseries_identifiers: list[str], optional :param time_variable: The desired time variable column. Only valid for time series forecasting tasks. Cannot be set if either prediction_type, target_variable, or timeseries_identifiers is also specified. (defaults to **None**) :type time_variable: str, optional :param full_reguess: Scope of the reguess process: whether it should reguess all the settings after changing a core parameter, or only reguess impacted settings (e.g. target remapping when changing the target, metrics when changing the prediction type...). Ignored if no core parameter is given. Only valid for prediction tasks and therefore also ignored for clustering. (defaults to **True**) :type full_reguess: bool, optional :param reguess_level: Deprecated, use `full_reguess` instead. Only valid for prediction tasks. Can be one of the following values: - TARGET_CHANGE: Change the target if target_variable is specified, reguess the target remapping, and clear the model's assertions if any. Equivalent to ``full_reguess=False`` (recommended usage) - FULL_REGUESS: All the settings of the ML task are reguessed. Equivalent to ``full_reguess=True`` (recommended usage) :type reguess_level: str, optional """ obj = {} if prediction_type is not None: obj["predictionType"] = prediction_type if timeseries_identifiers is not None: obj["timeseriesIdentifiers"] = timeseries_identifiers if time_variable is not None: obj["timeVariable"] = time_variable if target_variable is not None: obj["targetVariable"] = target_variable if full_reguess is not None: obj["fullReguess"] = full_reguess if reguess_level is not None: logger.info("'reguess_level' parameter ignored, superseded by 'full_reguess'") if reguess_level is not None and full_reguess is None: logger.info("'reguess_level' is deprecated, use 'full_reguess' instead") if reguess_level == "TARGET_CHANGE": obj["fullReguess"] = False elif reguess_level == "FULL_REGUESS": obj["fullReguess"] = True else: raise ValueError("Unsupported value for 'reguess_level': " + reguess_level) if set(obj.keys()) == {"fullReguess"}: logger.info("No core params given, reguessing all settings") self.client._perform_empty( "PUT", "/projects/{project_key}/models/lab/{analysis_id}/{mltask_id}/guess".format( project_key=self.project_key, analysis_id=self.analysis_id, mltask_id=self.mltask_id ), params=obj ) class DSSMLTaskQueues(object): """ Iterable listing of ML Task queues """ def __init__(self, data): self.data = data def __iter__(self): return self.data["queues"].__iter__() def get_raw(self): return self.data dataiku-api-client-python-13.2.2/dataikuapi/dss/mlflow.py000066400000000000000000000400311471211575100233440ustar00rootroot00000000000000import json import time from datetime import datetime from dataikuapi.dss.savedmodel import ExternalModelVersionHandler from dataikuapi.dss.utils import DSSDatasetSelectionBuilder class DSSMLflowExtension(object): """ A handle to interact with specific endpoints of the DSS MLflow integration. Do not create this directly, use :meth:`dataikuapi.dss.project.DSSProject.get_mlflow_extension` """ def __init__(self, client, project_key): self.client = client self.project = client.get_project(project_key) self.project_key = project_key def list_models(self, run_id): """ Returns the list of models of given run :param run_id: run_id for which to return a list of models :type run_id: str """ response = self.client._perform_http( "GET", "/api/2.0/mlflow/extension/models/{}".format(run_id), headers={"x-dku-mlflow-project-key": self.project_key} ) return response.json() def list_experiments(self, view_type="ACTIVE_ONLY", max_results=1000): """ Returns the list of experiments in the DSS project for which MLflow integration is setup :param view_type: ACTIVE_ONLY, DELETED_ONLY or ALL :type view_type: str :param max_results: max results count :type max_results: int :rtype: dict """ response = self.client._perform_http( "POST", "/api/2.0/mlflow/experiments/search", headers={"x-dku-mlflow-project-key": self.project_key}, body={"view_type": view_type, "max_results": max_results} ) return response.json() def rename_experiment(self, experiment_id, new_name): """ Renames an experiment :param experiment_id: experiment id :type experiment_id: str :param new_name: new name :type new_name: str """ response = self.client._perform_http( "POST", "/api/2.0/mlflow/experiments/update", headers={"x-dku-mlflow-project-key": self.project_key}, body={"experiment_id": experiment_id, "new_name": new_name} ) return response.json() def restore_experiment(self, experiment_id): """ Restores a deleted experiment :param experiment_id: experiment id :type experiment_id: str """ response = self.client._perform_http( "POST", "/api/2.0/mlflow/experiments/restore", headers={"x-dku-mlflow-project-key": self.project_key}, body={"experiment_id": experiment_id} ) return response.json() def restore_run(self, run_id): """ Restores a deleted run :param run_id: run id :type run_id: str """ response = self.client._perform_http( "POST", "/api/2.0/mlflow/runs/restore", headers={"x-dku-mlflow-project-key": self.project_key}, body={"run_id": run_id} ) return response.json() def garbage_collect(self): """ Permanently deletes the experiments and runs marked as "Deleted" """ self.client._perform_http( "POST", "/api/2.0/mlflow/extension/garbage-collect", headers={"x-dku-mlflow-project-key": self.project_key} ) def create_experiment_tracking_dataset(self, dataset_name, experiment_ids=[], view_type="ACTIVE_ONLY", filter_expr="", order_by=[], format="LONG"): """ Creates a virtual dataset exposing experiment tracking data. :param dataset_name: name of the dataset :type dataset_name: str :param experiment_ids: list of ids of experiments to filter on. No filtering if empty :type experiment_ids: list(str) :param view_type: one of ACTIVE_ONLY, DELETED_ONLY and ALL. Default is ACTIVE_ONLY :type view_type: str :param filter_expr: MLflow search expression :type filter_expr: str :param order_by: list of order by clauses. Default is ordered by start_time, then runId :type order_by: list(str) :param format: LONG or JSON. Default is LONG :type format: str """ self.client._perform_http( "POST", "/api/2.0/mlflow/extension/create-project-experiments-dataset", headers={"x-dku-mlflow-project-key": self.project_key}, body={ "datasetName": dataset_name, "experimentIds": experiment_ids, "viewType": view_type, "filter": filter_expr, "orderBy": order_by, "format": format } ) def clean_experiment_tracking_db(self): """ Cleans the experiments, runs, params, metrics, tags, etc. for this project This call requires an API key with admin rights """ self.client._perform_raw("DELETE", "/api/2.0/mlflow/extension/clean-db/%s" % self.project_key) def set_run_inference_info(self, run_id, prediction_type, classes=None, code_env_name=None, target=None): """ Sets the type of the model, and optionally other information useful to deploy or evaluate it. prediction_type must be one of: - REGRESSION - BINARY_CLASSIFICATION - MULTICLASS - OTHER Classes must be specified if and only if the model is a BINARY_CLASSIFICATION or MULTICLASS model. This information is leveraged to filter saved models on their prediction type and prefill the classes when deploying (using the GUI or :meth:`deploy_run_model`) an MLflow model as a version of a DSS Saved Model. :param prediction_type: prediction type (see doc) :type prediction_type: str :param run_id: run_id for which to set the classes :type run_id: str :param classes: ordered list of classes (not for all prediction types, see doc). Every class will be converted by calling str(). The classes must be specified in the same order as learned by the model. Some flavors such as scikit-learn may allow you to build this list from the model itself. :type classes: list :param code_env_name: name of an adequate DSS python code environment :type code_env_name: str :param target: name of the target :type target: str """ if prediction_type not in {"REGRESSION", "BINARY_CLASSIFICATION", "MULTICLASS", "OTHER"}: raise ValueError('Invalid prediction type: {}'.format(prediction_type)) if classes and prediction_type not in {"BINARY_CLASSIFICATION", "MULTICLASS"}: raise ValueError('Classes can be specified only for BINARY_CLASSIFICATION or MULTICLASS prediction types') if prediction_type in {"BINARY_CLASSIFICATION", "MULTICLASS"}: if not classes: raise ValueError('Classes must be specified for {} prediction type'.format(prediction_type)) if not isinstance(classes, list): raise ValueError('Wrong type for classes: {}'.format(type(classes))) for cur_class in classes: if cur_class is None: raise ValueError('class cannot be None') classes = [str(cur_class) for cur_class in classes] if code_env_name and not isinstance(code_env_name, str): raise ValueError('code_env_name must be a string') if target and not isinstance(target, str): raise ValueError('target must be a string') params = { "run_id": run_id, "prediction_type": prediction_type } if classes: params["classes"] = json.dumps(classes) if code_env_name: params["code_env_name"] = code_env_name if target: params["target"] = target self.client._perform_http( "POST", "/api/2.0/mlflow/extension/set-run-inference-info", headers={"x-dku-mlflow-project-key": self.project_key}, body=params ) def deploy_run_model(self, run_id, sm_id, version_id=None, use_inference_info=True, code_env_name=None, evaluation_dataset=None, target_column_name=None, class_labels=None, model_sub_folder=None, selection=None, activate=True, binary_classification_threshold=0.5, use_optimal_threshold=True, skip_expensive_reports=False): """ Deploys a model from an experiment run, with lineage. Simple usage: .. code-block:: python mlflow_ext.set_run_inference_info(run_id, "BINARY_CLASSIFICATION", list_of_classes, code_env_name, target_column_name) sm_id = project.create_mlflow_pyfunc_model("model_name", "BINARY_CLASSIFICATION").id mlflow_extension.deploy_run_model(run_id, sm_id, evaluation_dataset) If the optional `evaluation_dataset` is not set, the model will be deployed but not evaluated: this makes `target_column_name` optional as well in :meth:`set_run_inference_info` :param run_id: The id of the run to deploy :type run_id: str :param sm_id: The id of the saved model to deploy the run to :type sm_id: str :param version_id: [optional] Unique identifier of a Saved Model Version. If it already exists, existing version is overwritten. Whitespaces or dashes are not allowed. If not set, a timestamp will be used as version_id. :type version_id: str :param use_inference_info: [optional] default to True. if set, uses the :meth:`set_inference_info` previously done on the run to retrieve the prediction type of the model, its code environment, classes and target. :type use_inference_info: bool :param evaluation_dataset: [optional] The evaluation dataset, if the deployment of the models can imply an evaluation. :type evaluation_dataset: str :param target_column_name: [optional] The target column of the evaluation dataset. Can be set by :meth:`set_inference_info`. :type target_column_name: str :param class_labels: [optional] The class labels of the target. Can be set by :meth:`set_inference_info`. :type class_labels: list(str) :param code_env_name: [optional] The code environment to be used. Must contain a supported version of the mlflow package and the ML libs used to train the model. Can be set by :meth:`set_inference_info`. :type code_env_name: str :param model_sub_folder: [optional] The name of the subfolder containing the model. Optional if it is unique. Existing values can be retrieved with `project.get_mlflow_extension().list_models(run_id)` :type model_sub_folder: str :param selection: [optional] will default to HEAD_SEQUENTIAL with a maxRecords of 10_000. e.g. * Example 1: ``DSSDatasetSelectionBuilder().with_head_sampling(100)`` * Example 2: ``{"samplingMethod": "HEAD_SEQUENTIAL", "maxRecords": 100}`` :type selection: :class:`DSSDatasetSelectionBuilder` optional sampling parameter for the evaluation or dict :param activate: [optional] True by default. Activate or not the version after deployment :type activate: bool :param binary_classification_threshold: [optional] Threshold (or cut-off) value to override if the model is a binary classification :type binary_classification_threshold: float :param use_optimal_threshold: [optional] Use or not the optimal threshold for the saved model metric computed at evaluation :type use_optimal_threshold: bool :param skip_expensive_reports: [optional] Don't compute expensive report screens (e.g. feature importance). :type skip_expensive_reports: bool :return: a handler in order to interact with the new MLFlow model version :rtype: :class:`dataikuapi.dss.savedmodel.ExternalModelVersionHandler` """ sampling_param = selection.build() if isinstance( selection, DSSDatasetSelectionBuilder) else selection if version_id is None: dt = datetime.fromtimestamp(time.time()) version_id = dt.strftime("%Y_%m_%dT%H_%M_%S") prediction_type = self.project.get_saved_model(sm_id).get_settings().settings["miniTask"].get("predictionType") output_labels = [] if class_labels: for label in class_labels: output_labels.append({'label': label}) model_version_info_obj = {} if evaluation_dataset is not None: model_version_info_obj["gatherFeaturesFromDataset"] = evaluation_dataset if target_column_name is not None: model_version_info_obj["targetColumnName"] = target_column_name if output_labels is not None: model_version_info_obj["classLabels"] = output_labels if code_env_name is not None: model_version_info_obj["pythonCodeEnvName"] = code_env_name if prediction_type is not None: model_version_info_obj["predictionType"] = prediction_type self.client._perform_http("POST", "/api/2.0/mlflow/extension/deploy-run", headers={"x-dku-mlflow-project-key": self.project_key}, params={"projectKey": self.project_key, "runId": run_id, "smId": sm_id, "modelSubfolder": model_sub_folder, "versionId": version_id, "modelVersionInfo": json.dumps(model_version_info_obj), "samplingParam": sampling_param, "activate": activate, "binaryClassificationThreshold": binary_classification_threshold, "useOptimalThreshold": use_optimal_threshold, "skipExpensiveReports": skip_expensive_reports, "useInferenceInfo": use_inference_info } ) return ExternalModelVersionHandler(sm_id, version_id) def import_analyses_models_into_experiment(self, model_ids, experiment_id): """ Import models from a visual ML analysis into an existing experiment. import dataiku Usage example .. code-block:: python # Retrieve all the trained model ids of the first task of the first analysis of a project project = client.get_project("YOUR_PROJECT_ID") first_analysis_id = project.list_analyses()[0]['analysisId'] first_analysis = project.get_analysis(first_analysis_id) first_task_id = first_analysis.list_ml_tasks()['mlTasks'][0]['mlTaskId'] first_task = first_analysis.get_ml_task(first_task_id) full_model_ids = first_task.get_trained_models_ids() # Create a new experiment with project.setup_mlflow(project.create_managed_folder("mlflow")) as mlflow: experiment_id = mlflow.create_experiment("Sample export of DSS visual analysis models") # Export the retrieved model ids to the created experiment project.get_mlflow_extension().import_analyses_models_into_experiment(full_model_ids, experiment_id) :param model_ids: IDs of models from a Visual Analysis. :type model_ids: list of str :param experiment_id: ID of the experiment into which the visual analysis models will be imported. :type experiment_id: str """ self.client._perform_http("POST", "/api/2.0/mlflow/import-analysis-models-into-experiment-tracking", headers={"x-dku-mlflow-project-key": self.project_key}, params={"projectKey": self.project_key, "fullModelIds": model_ids, "experimentId": experiment_id } ) dataiku-api-client-python-13.2.2/dataikuapi/dss/modelcomparison.py000066400000000000000000000163671471211575100252560ustar00rootroot00000000000000import warnings from .discussion import DSSObjectDiscussions import re class DSSModelComparison(object): """ A handle to interact with a model comparison on the DSS instance Do not create this directly, use :meth:`dataikuapi.dss.DSSProject.get_model_comparison` """ def __init__(self, client, project_key, mec_id): self.client = client self.project = client.get_project(project_key) self.project_key = project_key self.mec_id = mec_id @property def id(self): return self.mec_id def get_settings(self): """ Returns the settings of this model comparison :rtype: :class:`dataikuapi.dss.modelcomparison.DSSModelComparisonSettings` """ data = self.client._perform_json( "GET", "/projects/%s/modelcomparisons/%s" % (self.project_key, self.mec_id)) return DSSModelComparisonSettings(self, data) def get_object_discussions(self): """ Get a handle to manage discussions on the model comparison :returns: the handle to manage discussions :rtype: :class:`dataikuapi.discussion.DSSObjectDiscussions` """ return DSSObjectDiscussions(self.client, self.project_key, "MODEL_COMPARISON", self.mec_id) def get_evaluation_like_from_full_id(self, full_id): """ Retrieves a Saved Model from the flow, a Lab Model from an Analysis or a Model Evaluation from a Model Evaluation Store) using its full id. :param string full_id: the full id of the item to retrieve :returns: A handle on the Saved Model, the Model Evaluation or the Lab Model :rtype: :class:`dataikuapi.dss.savedmodel.DSSSavedModel` :rtype: :class:`dataikuapi.dss.modelevaluationstore.DSSModelEvaluation` :rtype: :class:`dataikuapi.dss.ml.DSSTrainedPredictionModelDetails` """ saved_model_pattern = re.compile("^S-(\\w+)-(\\w+)-(\\w+)(?:-part-(\\w+)-(v?\\d+))?$\\Z") analysis_model_pattern = re.compile("^A-(\\w+)-(\\w+)-(\\w+)-(s[0-9]+)-(pp[0-9]+(?:-part-(\\w+)|-base)?)-(m[0-9]+)$\\Z") model_evaluation_pattern = re.compile("^ME-(\\w+)-(\\w+)-(\\w+)$\\Z") if saved_model_pattern.match(full_id): return self.project.get_saved_model(full_id) elif model_evaluation_pattern.match(full_id): mes_id = full_id.split('-')[2] evaluation_id = full_id.split('-')[3] mes = self.project.get_model_evaluation_store(mes_id) return mes.get_model_evaluation(evaluation_id) elif analysis_model_pattern.match(full_id): analysis_id = full_id.split('-')[2] task_id = full_id.split('-')[3] return self.project.get_ml_task(analysis_id, task_id).get_trained_model_details(full_id) raise ValueError("{} is not a valid full model id or full model evaluation id.".format(full_id)) ######################################################## # Deletion ######################################################## def delete(self): """ Delete the model comparison """ return self.client._perform_empty("DELETE", "/projects/%s/modelcomparisons/%s" % (self.project_key, self.mec_id)) class DSSModelComparisonSettings(object): """ A handle on the settings of a model comparison A model comparison has: - a display name ; - a prediction type ; - a list of full ids of items to compare The prediction type can be: - BINARY_CLASSIFICATION, - REGRESSION, - MULTICLASS The full ids are: - the model id of a Lab Model, - the model id of a saved model version, - the model evaluation id of a model evaluation. Do not create this class directly, instead use :meth:`dataikuapi.dss.DSSModelComparison.get_settings` """ def __init__(self, model_comparison, settings): self.model_comparison = model_comparison self.settings = settings def get_raw(self): """ Get raw settings of a model comparison :return: the raw settings of comparison, as a dict. Modifications made to the returned object are reflected when saving :rtype: dict """ return self.settings def add_compared_item(self, full_id): """ Add an item to the list of compared items :param full_id: full id of the item (lab model, saved model version, model evaluation) to add """ if "comparedModels" not in self.settings: self.settings["comparedModels"] = [] self.settings["comparedModels"].append({ "refId": full_id }) def remove_compared_item(self, full_id): """ Remove an item from the list of compared items :param full_id: full id of the item (lab model, saved model version, model evaluation) to remove """ if not self.settings["comparedModels"]: return self.settings["comparedModels"] = list(filter(lambda x: x["refId"] != full_id, self.settings["comparedModels"])) def get_compared_items(self): """ Get the full ids of items compared in this comparison :return: the list of the full ids of compared items :rtype: list[str] """ if not self.settings["comparedModels"]: return [] return [x["refId"] for x in self.settings["comparedModels"]] @property def prediction_type(self): """ Get the prediction type of this comparison :return: str """ warnings.warn("prediction_type field is deprecated. Use model_task_type instead", DeprecationWarning) return self.settings.get("modelTaskType", self.settings.get("predictionType")) @prediction_type.setter def prediction_type(self, prediction_type): """ Set the prediction type of this comparison. Must be consistent with the prediction types of compared items. :param prediction_type: """ warnings.warn("prediction_type field is deprecated. Use model_task_type instead", DeprecationWarning) self.model_task_type = prediction_type @property def model_task_type(self): """ Get the prediction type of this comparison :return: str """ return self.settings.get('modelTaskType') @model_task_type.setter def model_task_type(self, model_task_type): """ Set the initial model task type of this comparison. Must be consistent with the model task types of compared items. :param model_task_type: """ self.settings['modelTaskType'] = model_task_type @property def display_name(self): """ Human readable name of this comparison :return: str """ return self.settings["displayName"] @display_name.setter def display_name(self, display_name): """ Set the human readable name of this comparison :param display_name: """ self.settings["displayName"] = display_name def save(self): """ Save settings modifications """ self.model_comparison.client._perform_empty( "PUT", "/projects/%s/modelcomparisons/%s" % (self.model_comparison.project_key, self.model_comparison.mec_id), body=self.settings) dataiku-api-client-python-13.2.2/dataikuapi/dss/modelevaluationstore.py000066400000000000000000000752661471211575100263330ustar00rootroot00000000000000import json import warnings from io import BytesIO from .metrics import ComputedMetrics from .discussion import DSSObjectDiscussions from .future import DSSFuture try: basestring except NameError: basestring = str class DSSModelEvaluationStore(object): """ A handle to interact with a model evaluation store on the DSS instance. .. warning:: Do not create this directly, use :meth:`dataikuapi.dss.project.DSSProject.get_model_evaluation_store` """ def __init__(self, client, project_key, mes_id): self.client = client self.project = client.get_project(project_key) self.project_key = project_key self.mes_id = mes_id @property def id(self): return self.mes_id def get_settings(self): """ Returns the settings of this model evaluation store. :rtype: DSSModelEvaluationStoreSettings """ data = self.client._perform_json( "GET", "/projects/%s/modelevaluationstores/%s" % (self.project_key, self.mes_id)) return DSSModelEvaluationStoreSettings(self, data) ######################################################## # Misc ######################################################## def get_zone(self): """ Gets the flow zone of this model evaluation store :rtype: :class:`dataikuapi.dss.flow.DSSFlowZone` """ return self.project.get_flow().get_zone_of_object(self) def move_to_zone(self, zone): """ Moves this object to a flow zone :param object zone: a :class:`dataikuapi.dss.flow.DSSFlowZone` where to move the object """ if isinstance(zone, basestring): zone = self.project.get_flow().get_zone(zone) zone.add_item(self) def share_to_zone(self, zone): """ Share this object to a flow zone :param object zone: a :class:`dataikuapi.dss.flow.DSSFlowZone` where to share the object """ if isinstance(zone, basestring): zone = self.project.get_flow().get_zone(zone) zone.add_shared(self) def unshare_from_zone(self, zone): """ Unshare this object from a flow zone :param object zone: a :class:`dataikuapi.dss.flow.DSSFlowZone` from where to unshare the object """ if isinstance(zone, basestring): zone = self.project.get_flow().get_zone(zone) zone.remove_shared(self) def get_usages(self): """ Get the recipes referencing this model evaluation store :return: a list of usages """ return self.client._perform_json("GET", "/projects/%s/modelevaluationstores/%s/usages" % (self.project_key, self.mes_id)) def get_object_discussions(self): """ Get a handle to manage discussions on the model evaluation store :returns: the handle to manage discussions :rtype: :class:`dataikuapi.discussion.DSSObjectDiscussions` """ return DSSObjectDiscussions(self.client, self.project_key, "MODEL_EVALUATION_STORE", self.mes_id) ######################################################## # Deletion ######################################################## def delete(self): """ Delete the model evaluation store """ return self.client._perform_empty("DELETE", "/projects/%s/modelevaluationstores/%s" % (self.project_key, self.mes_id)) ######################################################## # Model evaluations ######################################################## def list_model_evaluations(self): """ List the model evaluations in this model evaluation store. The list is sorted by ME creation date. :returns: The list of the model evaluations :rtype: list of :class:`dataikuapi.dss.modelevaluationstore.DSSModelEvaluation` """ items = self.client._perform_json("GET", "/projects/%s/modelevaluationstores/%s/evaluations/" % (self.project_key, self.mes_id)) return [DSSModelEvaluation(self, item["ref"]["evaluationId"]) for item in items] def get_model_evaluation(self, evaluation_id): """ Get a handle to interact with a specific model evaluation :param string evaluation_id: the id of the desired model evaluation :returns: A :class:`dataikuapi.dss.modelevaluationstore.DSSModelEvaluation` model evaluation handle """ return DSSModelEvaluation(self, evaluation_id) def get_latest_model_evaluation(self): """ Get a handle to interact with the latest model evaluation computed :returns: A :class:`dataikuapi.dss.modelevaluationstore.DSSModelEvaluation` model evaluation handle if the store is not empty, else None """ latest_evaluation_id = self.client._perform_text( "GET", "/projects/%s/modelevaluationstores/%s/latestEvaluationId" % (self.project_key, self.mes_id)) if not latest_evaluation_id: return None return DSSModelEvaluation(self, latest_evaluation_id) def delete_model_evaluations(self, evaluations): """ Remove model evaluations from this store """ obj = [] for evaluation in evaluations: if isinstance(evaluation, DSSModelEvaluation): obj.append(evaluation.evaluation_id) elif isinstance(evaluation, dict): obj.append(evaluation['evaluation_id']) else: obj.append(evaluation) self.client._perform_json( "DELETE", "/projects/%s/modelevaluationstores/%s/evaluations/" % (self.project_key, self.mes_id), body=obj) def build(self, job_type="NON_RECURSIVE_FORCED_BUILD", wait=True, no_fail=False): """ Starts a new job to build this model evaluation store and wait for it to complete. Raises if the job failed. .. code-block:: python job = mes.build() print("Job %s done" % job.id) :param job_type: The job type. One of RECURSIVE_BUILD, NON_RECURSIVE_FORCED_BUILD or RECURSIVE_FORCED_BUILD :param wait: wait for the build to finish before returning :param no_fail: if True, does not raise if the job failed. Valid only when wait is True :return: the :class:`dataikuapi.dss.job.DSSJob` job handle corresponding to the built job :rtype: :class:`dataikuapi.dss.job.DSSJob` """ jd = self.project.new_job(job_type) jd.with_output(self.mes_id, object_type="MODEL_EVALUATION_STORE") if wait: return jd.start_and_wait(no_fail) else: return jd.start(allowFail=not no_fail) ######################################################## # Metrics ######################################################## def get_last_metric_values(self): """ Get the metrics of the latest model evaluation built :returns: a list of metric objects and their value """ return ComputedMetrics(self.client._perform_json( "GET", "/projects/%s/modelevaluationstores/%s/metrics/last" % (self.project_key, self.mes_id))) def get_metric_history(self, metric): """ Get the history of the values of the metric on this model evaluation store :returns: an object containing the values of the metric, cast to the appropriate type (double, boolean,...) """ return self.client._perform_json( "GET", "/projects/%s/modelevaluationstores/%s/metrics/history" % (self.project_key, self.mes_id), params={'metricLookup': metric if isinstance(metric, str)or isinstance(metric, unicode) else json.dumps(metric)}) def compute_metrics(self, metric_ids=None, probes=None): """ Compute metrics on this model evaluation store. If the metrics are not specified, the metrics setup on the model evaluation store are used. """ url = "/projects/%s/modelevaluationstores/%s/actions" % (self.project_key, self.mes_id) if metric_ids is not None: return self.client._perform_json( "POST" , "%s/computeMetricsFromIds" % url, body={"metricIds" : metric_ids}) elif probes is not None: return self.client._perform_json( "POST" , "%s/computeMetrics" % url, body=probes) else: return self.client._perform_json( "POST" , "%s/computeMetrics" % url) def run_checks(self, evaluation_id='', checks=None): """ Run checks on a partition of this model evaluation store. If the checks are not specified, the checks setup on the model evaluation store are used. :param str evaluation_id: (optional) id of evaluation on which checks should be run. Last evaluation is used if not specified. :param list[string] checks: (optional) ids of the checks to run. :returns: a checks computation report, as a dict. :rtype: dict """ if checks is None: return self.client._perform_json( "POST", "/projects/%s/modelevaluationstores/%s/actions/runChecks" %(self.project_key, self.mes_id), params={'evaluationId': evaluation_id}) else: return self.client._perform_json( "POST", "/projects/%s/modelevaluationstores/%s/actions/runChecks" %(self.project_key, self.mes_id), params={'evaluationId': evaluation_id}, body=checks) class MetricDefinition(dict): def __init__(self, code, value, name=None, description=None): dict.__init__(self, {"metricCode": code, "value": value, "name": name, "description": description}) class LabelDefinition(dict): def __init__(self, key, value): dict.__init__(self, {"key": key, "value": value}) def add_custom_model_evaluation(self, metrics, evaluation_id=None, name=None, labels=None, model=None): """ Adds a model evaluation with custom metrics to the model evaluation store. :param list[DSSModelEvaluationStore.MetricDefinition] metrics: the metrics to add. :param str evaluation_id: the id of the evaluation (optional) :param str name: the human-readable name of the evaluation (optional) :param list[DSSModelEvaluationStore.LabelDefinition] labels: labels to set on the model evaluation (optionam). See below. :param model: saved model version (full ID or DSSTrainedPredictionModelDetails) of the evaluated model (optional) :type model: Union[str, DSSTrainedPredictionModelDetails] Code sample: .. code-block:: python import dataiku from dataikuapi.dss.modelevaluationstore import DSSModelEvaluationStore client=dataiku.api_client() project=client.get_default_project() mes=project.get_model_evaluation_store("7vFZWNck") accuracy = DSSModelEvaluationStore.MetricDefinition("accuracy", 0.95, "Accuracy") other = DSSModelEvaluationStore.MetricDefinition("other", 42, "Other", "Other metric desc") label = DSSModelEvaluationStore.LabelDefinition("custom:myLabel", "myValue") mes.add_custom_model_evaluation([accuracy, pouet], labels=[label]) mes.run_checks() """ if hasattr(model, 'full_id'): model = model.full_id url = "/projects/%s/modelevaluationstores/%s/evaluations" % (self.project_key, self.mes_id) return self.client._perform_json( "POST", url, body={ "evaluationId": evaluation_id, "name": name, "metrics": metrics, "labels": labels, "fullModelId": model }) class DSSModelEvaluationStoreSettings: """ A handle on the settings of a model evaluation store .. warning:: Do not create this class directly, instead use :meth:`dataikuapi.dss.modelevaluationstore.DSSModelEvaluationStore.get_settings` """ def __init__(self, model_evaluation_store, settings): self.model_evaluation_store = model_evaluation_store self.settings = settings def get_raw(self): return self.settings def save(self): self.model_evaluation_store.client._perform_empty( "PUT", "/projects/%s/modelevaluationstores/%s" % (self.model_evaluation_store.project_key, self.model_evaluation_store.mes_id), body=self.settings) class DSSModelEvaluation: """ A handle on a model evaluation .. warning:: Do not create this class directly, instead use :meth:`dataikuapi.dss.modelevaluationstore.DSSModelEvaluationStore.get_model_evaluation` """ def __init__(self, model_evaluation_store, evaluation_id): self.model_evaluation_store = model_evaluation_store self.client = model_evaluation_store.client # unpack some fields self.evaluation_id = evaluation_id self.project_key = model_evaluation_store.project_key self.mes_id = model_evaluation_store.mes_id def get_full_info(self): """ Retrieve the model evaluation with its performance data :return: the model evaluation full info, as a :class:`dataikuapi.dss.modelevaluationstore.DSSModelEvaluationFullInfo` """ data = self.client._perform_json( "GET", "/projects/%s/modelevaluationstores/%s/evaluations/%s" % (self.project_key, self.mes_id, self.evaluation_id)) return DSSModelEvaluationFullInfo(self, data) def get_full_id(self): return "ME-{}-{}-{}".format(self.project_key, self.mes_id, self.evaluation_id) def delete(self): """ Remove this model evaluation """ obj = [self.evaluation_id] self.client._perform_json( "DELETE", "/projects/%s/modelevaluationstores/%s/evaluations/" % (self.project_key, self.mes_id), body=obj) @property def full_id(self): return "ME-%s-%s-%s"%(self.project_key, self.mes_id, self.evaluation_id) def compute_data_drift(self, reference=None, data_drift_params=None, wait=True): """ Compute data drift against a reference model or model evaluation. The reference is determined automatically unless specified. .. attention:: Deprecated. Use :meth:`dataikuapi.dss.modelevaluationstore.DSSModelEvaluationStore.compute_drift` instead :param reference: saved model version (full ID or DSSTrainedPredictionModelDetails) or model evaluation (full ID or DSSModelEvaluation) to use as reference (optional) :type reference: Union[str, DSSModelEvaluation, DSSTrainedPredictionModelDetails] :param data_drift_params: data drift computation settings as a :class:`dataikuapi.dss.modelevaluationstore.DataDriftParams` (optional) :type data_drift_params: DataDriftParams :param wait: data drift computation settings (optional) :returns: a :class:`dataikuapi.dss.modelevaluationstore.DataDriftResult` containing data drift analysis results if `wait` is `True`, or a :class:`~dataikuapi.dss.future.DSSFuture` handle otherwise """ warnings.warn("This method is deprecated. Use DSSModelEvaluationStore.compute_drift instead", DeprecationWarning) if hasattr(reference, 'full_id'): reference = reference.full_id if data_drift_params: data_drift_params = data_drift_params.data future_response = self.client._perform_json( "POST", "/projects/%s/modelevaluationstores/%s/evaluations/%s/computeDataDrift" % (self.project_key, self.mes_id, self.evaluation_id), body={ "referenceId": reference, "dataDriftParams": data_drift_params }) future = DSSFuture(self.client, future_response.get('jobId', None), future_response, result_wrapper=DataDriftResult) return future.wait_for_result() if wait else future def compute_drift(self, reference=None, drift_params=None, wait=True): """ Compute drift against a reference model or model evaluation. The reference is determined automatically unless specified. :param reference: saved model version (full ID or DSSTrainedPredictionModelDetails) or model evaluation (full ID or DSSModelEvaluation) to use as reference (optional) :type reference: Union[str, DSSModelEvaluation, DSSTrainedPredictionModelDetails] :param drift_params: drift computation settings as a :class:`dataikuapi.dss.modelevaluationstore.DriftParams` (optional) :type drift_params: DriftParams :param wait: data drift computation settings (optional) :returns: a :class:`dataikuapi.dss.modelevaluationstore.DriftResult` containing data drift analysis results if `wait` is `True`, or a :class:`~dataikuapi.dss.future.DSSFuture` handle otherwise """ if hasattr(reference, 'full_id'): reference = reference.full_id if drift_params: drift_params = drift_params.data future_response = self.client._perform_json( "POST", "/projects/%s/modelevaluationstores/%s/evaluations/%s/computeDrift" % (self.project_key, self.mes_id, self.evaluation_id), body={ "referenceId": reference, "driftParams": drift_params }) future = DSSFuture(self.client, future_response.get('jobId', None), future_response, result_wrapper=DriftResult) return future.wait_for_result() if wait else future def get_metrics(self): """ Get the metrics for this model evaluation. Metrics must be understood here as Metrics in DSS Metrics & Checks :return: the metrics, as a JSON object """ return self.client._perform_json( "GET", "/projects/%s/modelevaluationstores/%s/evaluations/%s/metrics" % (self.project_key, self.mes_id, self.evaluation_id)) def get_sample_df(self): """ Get the sample of the evaluation dataset on which the evaluation was performed :return: the sample content, as a :class:`pandas.DataFrame` """ buf = BytesIO() with self.client._perform_raw( "GET", "/projects/%s/modelevaluationstores/%s/evaluations/%s/sample" % (self.project_key, self.mes_id, self.evaluation_id) ).raw as f: buf.write(f.read()) schema_txt = self.client._perform_raw( "GET", "/projects/%s/modelevaluationstores/%s/evaluations/%s/schema" % (self.project_key, self.mes_id, self.evaluation_id) ).text schema = json.loads(schema_txt) import pandas as pd return pd.read_csv(BytesIO(buf.getvalue()), compression='gzip', sep='\t', header=None, names=[c["name"] for c in schema["columns"]]) class DSSModelEvaluationFullInfo: """ A handle on the full information on a model evaluation. Includes information such as the full id of the evaluated model, the evaluation params, the performance and drift metrics, if any, etc. .. warning:: Do not create this class directly, instead use :meth:`dataikuapi.dss.modelevaluationstore.DSSModelEvaluation.get_full_info` """ def __init__(self, model_evaluation, full_info): self.model_evaluation = model_evaluation self.full_info = full_info self.metrics = self.full_info["metrics"] # type: dict """The performance and data drift metric, if any.""" self.creation_date = self.full_info["evaluation"]["created"] # type: int """The date and time of the creation of the model evaluation, as an epoch.""" self.full_id = self.full_info["evaluation"]["ref"]["fullId"] # type: str if "modelRef" in self.full_info["evaluation"]: self.model_full_id = self.full_info["evaluation"]["modelRef"]["fullId"] # type: str else: self.model_full_id = None self.prediction_type = self.full_info["evaluation"].get("predictionType") # type: str self.prediction_variable = self.full_info["evaluation"].get("predictionVariable") # type: str self.target_variable = self.full_info["evaluation"].get("targetVariable") # type: str self.user_meta = self.full_info["evaluation"]["userMeta"] # type: dict self.has_model = self.full_info["evaluation"].get("hasModel") """The user-accessible metadata (name, labels) Returns the original object, not a copy. Changes to the returned object are persisted to DSS by calling :meth:`save_user_meta`.""" def get_raw(self): return self.full_info def save_user_meta(self): return self.model_evaluation.client._perform_text( "PUT", "/projects/%s/modelevaluationstores/%s/evaluations/%s/user-meta" % (self.model_evaluation.project_key, self.model_evaluation.mes_id, self.model_evaluation.evaluation_id), body=self.user_meta) class DataDriftParams(object): """ Object that represents parameters for data drift computation. .. warning:: Do not create this object directly, use :meth:`dataikuapi.dss.modelevaluationstore.DataDriftParams.from_params` instead. .. attention:: Deprecated. Use :class:`dataikuapi.dss.modelevaluationstore.DriftParams` instead """ def __init__(self, data): self.data = data def __repr__(self): return u"{}({})".format(self.__class__.__name__, self.data) @staticmethod def from_params(per_column_settings, nb_bins=10, compute_histograms=True, confidence_level=0.95): """ Creates parameters for data drift computation from columns, number of bins, compute histograms and confidence level :param dict per_column_settings: A dict representing the per column settings. You should use a :class:`~dataikuapi.dss.modelevaluationstore.PerColumnDriftParamBuilder` to build it. :param int nb_bins: (optional) Nb. bins in histograms (apply to all columns) - default: 10 :param bool compute_histograms: (optional) Enable/disable histograms - default: True :param float confidence_level: (optional) Used to compute confidence interval on drift's model accuracy - default: 0.95 :rtype: :class:`dataikuapi.dss.modelevaluationstore.DataDriftParams` """ warnings.warn("This method is deprecated. Use DriftParams.from_params() instead", DeprecationWarning) return DataDriftParams({ "columns": per_column_settings, "nbBins": nb_bins, "computeHistograms": compute_histograms, "confidenceLevel": confidence_level }) class DriftParams(object): """ Object that represents parameters for drift computation. .. warning:: Do not create this object directly, use :meth:`dataikuapi.dss.modelevaluationstore.DriftParams.from_params` instead. """ def __init__(self, data): self.data = data def __repr__(self): return u"{}({})".format(self.__class__.__name__, self.data) @staticmethod def from_params(per_column_settings, nb_bins=10, compute_histograms=True, confidence_level=0.95): """ Creates parameters for drift computation from columns, number of bins, compute histograms and confidence level :param dict per_column_settings: A dict representing the per column settings. You should use a :class:`~dataikuapi.dss.modelevaluationstore.PerColumnDriftParamBuilder` to build it. :param int nb_bins: (optional) Nb. bins in histograms (apply to all columns) - default: 10 :param bool compute_histograms: (optional) Enable/disable histograms - default: True :param float confidence_level: (optional) Used to compute confidence interval on drift's model accuracy - default: 0.95 :rtype: :class:`dataikuapi.dss.modelevaluationstore.DriftParams` """ return DriftParams({ "columns": per_column_settings, "nbBins": nb_bins, "computeHistograms": compute_histograms, "confidenceLevel": confidence_level }) class PerColumnDriftParamBuilder(object): """ Builder for a map of per column drift params settings. Used as a helper before computing data drift to build columns param expected in :meth:`dataikuapi.dss.modelevaluationstore.DataDriftParams.from_params`. """ def __init__(self): self.columns = {} def build(self): """Returns the built dict for per column drift params settings""" return self.columns def with_column_drift_param(self, name, handling="AUTO", enabled=True): """ Sets the drift params settings for given column name. :param: string name: The name of the column :param: string handling: (optional) The column type, should be either NUMERICAL, CATEGORICAL or AUTO (default: AUTO) :param: bool enabled: (optional) False means the column is ignored in drift computation (default: True) """ self.columns[name] = { "handling": handling, "enabled": enabled } return self class DataDriftResult(object): """ A handle on the data drift result of a model evaluation. .. warning:: Do not create this class directly, instead use :meth:`dataikuapi.dss.modelevaluationstore.DSSModelEvaluation.compute_data_drift` """ def __init__(self, data): self.data = data self.drift_model_result = DriftModelResult(self.data["driftModelResult"]) """Drift analysis based on drift modeling.""" self.univariate_drift_result = UnivariateDriftResult(self.data["univariateDriftResult"]) """Per-column drift analysis based on pairwise comparison of distributions.""" self.per_column_settings = [ColumnSettings(cs) for cs in self.data["perColumnSettings"]] """Information about column handling that has been used (errors, types, etc).""" def get_raw(self): """ :return: the raw data drift result :rtype: dict """ return self.data class DriftResult(object): """ A handle on the drift result of a model evaluation. .. warning:: Do not create this class directly, instead use :meth:`dataikuapi.dss.modelevaluationstore.DSSModelEvaluation.compute_drift` """ def __init__(self, data): self.data = data self.drift_model_result = DriftModelResult(self.data["driftModelResult"]) if "driftModelResult" in self.data else None """Drift analysis based on drift modeling.""" self.univariate_drift_result = UnivariateDriftResult(self.data["univariateDriftResult"]) if "univariateDriftResult" in self.data else None """Per-column drift analysis based on pairwise comparison of distributions.""" self.per_column_settings = [ColumnSettings(cs) for cs in self.data["perColumnSettings"]] if "perColumnSettings" in self.data else None """Information about column handling that has been used (errors, types, etc).""" self.prediction_drift_result = PredictionDriftResult(self.data["predictionDriftResult"]) if "predictionDriftResult" in self.data else None """Drift analysis based on the prediction column""" def get_raw(self): """ :return: the raw data drift result :rtype: dict """ return self.data class DriftModelResult(object): """ A handle on the drift model result. .. warning:: Do not create this class directly, instead use :attr:`dataikuapi.dss.modelevaluationstore.DriftResult.drift_model_result` """ def __init__(self, data): self.data = data self.drift_model_accuracy = DriftModelAccuracy(self.data["driftModelAccuracy"]) self.feature_drift_importance = self.data["driftVersusImportance"] # type: dict def get_raw(self): """ :return: the raw drift model result :rtype: dict """ return self.data class UnivariateDriftResult(object): """ A handle on the univariate data drift. .. warning:: Do not create this class directly, instead use :attr:`dataikuapi.dss.modelevaluationstore.DriftResult.univariate_drift_result` """ def __init__(self, data): self.data = data self.per_column_drift_data = self.data["columns"] # type: dict """Drift data per column, as a dict of column name -> drift data.""" def get_raw(self): """ :return: the raw univariate data drift :rtype: dict """ return self.data class PredictionDriftResult(object): """ A handle on the prediction drift result. .. warning:: Do not create this class directly, instead use :attr:`dataikuapi.dss.modelevaluationstore.DriftResult.prediction_drift_result` """ def __init__(self, data): self.data = data self.chiSquare = self.data.get('chiSquareTestPvalue', None) self.ks = self.data.get('ksTestPvalue', None) self.psi = self.data.get('populationStabilityIndex', None) def get_raw(self): """ :return: the raw prediction drift :rtype: dict """ return self.data class ColumnSettings(object): """ A handle on column handling information. .. warning:: Do not create this class directly, instead use :meth:`dataikuapi.dss.modelevaluationstore.DriftResult.get_per_column_settings` """ def __init__(self, data): self.data = data self.name = self.data["name"] # type: str self.actual_column_handling = self.data["actualHandling"] # type: str """The actual column handling (either forced via drift params or inferred from model evaluation preprocessings). It can be any of NUMERICAL, CATEGORICAL, or IGNORED.""" self.default_column_handling = self.data["defaultHandling"] # type: str """The default column handling (based on model evaluation preprocessing only). It can be any of NUMERICAL, CATEGORICAL, or IGNORED.""" self.error_message = self.data.get("errorMessage", None) def get_raw(self): """ :return: the raw column handling information :rtype: dict """ return self.data class DriftModelAccuracy(object): """ A handle on the drift model accuracy. .. warning:: Do not create this class directly, instead use :attr:`dataikuapi.dss.modelevaluationstore.DriftModelResult.drift_model_accuracy` """ def __init__(self, data): self.data = data self.value = self.data["value"] # type: float self.lower_confidence_interval = self.data["lower"] # type: float self.upper_confidence_interval = self.data["upper"] # type: float self.pvalue = self.data["pvalue"] # type: float def get_raw(self): """ :return: the drift model accuracy data :rtype: dict """ return self.data dataiku-api-client-python-13.2.2/dataikuapi/dss/notebook.py000066400000000000000000000057061471211575100236760ustar00rootroot00000000000000from .discussion import DSSObjectDiscussions class DSSNotebook(object): """ A Python/R/Scala notebook on the DSS instance. .. attention:: Deprecated. Use :class:`dataikuapi.dss.jupyternotebook.DSSJupyterNotebook` """ def __init__(self, client, project_key, notebook_name, state=None): self.client = client self.project_key = project_key self.notebook_name = notebook_name self.state = state self.state_is_peek = True def unload(self, session_id=None): """ Stop this Jupyter notebook and release its resources. .. attention:: Deprecated. Use :class:`dataikuapi.dss.jupyternotebook.DSSJupyterNotebook` """ state = self.get_state() if state is None: raise Exception("Notebook isn't running") if state.get('activeSessions', None) is None: raise Exception("Notebook isn't running") if len(state['activeSessions']) == 0: raise Exception("Notebook isn't running") if session_id is None: if len(state['activeSessions']) > 1: raise Exception("Several sessions of the notebook are running, choose one") else: session_id = state['activeSessions'][0].get('sessionId', None) return self.client._perform_json("DELETE", "/projects/%s/notebooks/" % self.project_key, params={'notebookName' : self.notebook_name, 'sessionId' : session_id}) def get_state(self): """ Get the metadata associated to this Jupyter notebook. .. attention:: Deprecated. Use :class:`dataikuapi.dss.jupyternotebook.DSSJupyterNotebook` """ if self.state is None: self.state = self.client._perform_json("GET", "/projects/%s/notebooks/" % self.project_key, params={'notebookName' : self.notebook_name}) return self.state def get_sessions(self): """ Get the list of running sessions of this Jupyter notebook. .. attention:: Deprecated. Use :class:`dataikuapi.dss.jupyternotebook.DSSJupyterNotebook` """ state = self.get_state() if state is None: raise Exception("Notebook isn't running") if state.get('activeSessions', None) is None: raise Exception("Notebook isn't running") return state['activeSessions'] ######################################################## # Discussions ######################################################## def get_object_discussions(self): """ Get a handle to manage discussions on the notebook. .. attention:: Deprecated. Use :class:`dataikuapi.dss.jupyternotebook.DSSJupyterNotebook` :return: the handle to manage discussions :rtype: :class:`dataikuapi.dss.discussion.DSSObjectDiscussions` """ return DSSObjectDiscussions(self.client, self.project_key, "JUPYTER_NOTEBOOK", self.notebook_name) dataiku-api-client-python-13.2.2/dataikuapi/dss/plugin.py000066400000000000000000001071211471211575100233460ustar00rootroot00000000000000from ..utils import DataikuException from .future import DSSFuture class DSSPluginSettingsBase(object): """ Base class for plugin settings. .. important:: Do not instantiate directly, use either :meth:`DSSPlugin.get_settings` or :meth:`DSSPlugin.get_project_settings`. """ def __init__(self, client, plugin_id, settings, project_key=None): self.client = client self.plugin_id = plugin_id self.settings = settings self.project_key = project_key def get_raw(self): """ Get the raw settings object. .. note:: This method returns a reference to the settings, not a copy. Changing values in the reference then calling :meth:`save()` results in these changes being saved. :return: the settings as a dict. The instance-level settings consist of the plugin code env's name, the presets and the permissions to use the plugin components. The project-level settings consist of the presets and the parameter set descriptions. :rtype: dict """ return self.settings def list_parameter_set_names(self): """ List the names of the parameter sets defined in this plugin. :rtype: list[string] """ return [d["id"] for d in self.settings.get("accessibleParameterSetDescs", [])] def _list_parameter_sets(self): ret = [] for parameter_set_name in self.list_parameter_set_names(): parameter_set = self.get_parameter_set(parameter_set_name) if parameter_set is not None: ret.append(parameter_set) return ret def _get_parameter_set(self, parameter_set_name): desc = None data = None for ps in self.settings.get("accessibleParameterSetDescs", []): if ps["id"] == parameter_set_name: desc = ps for ps in self.settings.get("parameterSets", []): if ps["name"] == parameter_set_name: data = ps if desc is None: return None if data is None: # make a fake one, for example for project-level settings data = {'type':desc["elementType"]} presets_of_parameter_set = [p for p in self.settings["presets"] if p["type"] == data["type"]] return self._make_parameter_set(desc, data, presets_of_parameter_set) def _make_parameter_set(self, desc, data, presets_of_parameter_set): return DSSPluginParameterSet(self, desc, data, presets_of_parameter_set) def save(self): """ Save the settings to DSS. """ self.client._perform_empty("POST", "/plugins/%s/settings" % (self.plugin_id), params={"projectKey": self.project_key}, body=self.settings) class DSSPluginSettings(DSSPluginSettingsBase): """ The settings of a plugin. .. important:: Do not instantiate directly, use :meth:`DSSPlugin.get_settings`. """ def __init__(self, client, plugin_id, settings): super().__init__(client, plugin_id, settings) def set_code_env(self, code_env_name): """ Set the name of the code env to use for this plugin. :param string code_env_name: name of a code env """ self.settings["codeEnvName"] = code_env_name def list_parameter_sets(self): """ List the parameter sets defined in this plugin. :rtype: list[:class:`DSSPluginParameterSet`] """ return self._list_parameter_sets() def get_parameter_set(self, parameter_set_name): """ Get a parameter set in this plugin. :param string parameter_set_name: name of the parameter set :return: a handle on the parameter set :rtype: :class:`DSSPluginParameterSet` """ return self._get_parameter_set(parameter_set_name) def _make_parameter_set(self, desc, data, presets_of_parameter_set): return DSSPluginParameterSet(self, desc, data, presets_of_parameter_set) class DSSPluginProjectSettings(DSSPluginSettingsBase): """ The project-level settings of a plugin. .. important:: Do not instantiate directly, use :meth:`DSSPlugin.get_project_settings`. """ def __init__(self, client, plugin_id, settings, project_key): super().__init__(client, plugin_id, settings, project_key) def start_save(self): """ Save the settings to DSS. Returns with a future representing the post actions done asynchronously (e.g. rebuild cde image for visual recipes) :return: A :class:`~dataikuapi.dss.future.DSSFuture` representing the save post process :rtype: :class:`~dataikuapi.dss.future.DSSFuture` """ resp = self.client._perform_json("POST", "/plugins/%s/settings/future" % (self.plugin_id), body=self.settings) return DSSFuture.from_resp(self.client, resp) def list_parameter_sets(self): """ List the parameter sets defined in this plugin. :rtype: list[:class:`DSSPluginProjectParameterSet`] """ return self._list_parameter_sets() def get_parameter_set(self, parameter_set_name): """ Get a parameter set in this plugin. :param string parameter_set_name: name of the parameter set :return: a handle on the parameter set :rtype: :class:`DSSPluginProjectParameterSet` """ return self._get_parameter_set(parameter_set_name) def _make_parameter_set(self, desc, data, presets_of_parameter_set): return DSSPluginProjectParameterSet(self, desc, data, presets_of_parameter_set) class DSSPluginParameterSetBase(object): """ A parameter set in a plugin. .. important:: Do not instantiate directly, use :meth:`DSSPluginSettings.get_parameter_set()` or :meth:`DSSPluginSettings.list_parameter_sets(). For project-level settings, use :meth:`DSSPluginProjectSettings.get_parameter_set()` or :meth:`DSSPluginProjectSettings.list_parameter_sets()` The values in this class can be modified directly, and changes will be taken into account when calling :meth:`DSSPluginSettings.save()` (or :meth:`DSSPluginProjectSettings.save()` """ def __init__(self, plugin_settings, desc, settings, presets): self._plugin_settings = plugin_settings self._desc = desc self._settings = settings self._presets = presets @property def desc(self): """ Get the raw definition of the parameter set. :return: a parameter set definition, as a dict. The parameter set's contents is a **desc** sub-dict. See `the doc `_ :rtype: dict """ return self._desc @property def settings(self): """ Get the settings of the parameter set. These settings control the behavior of the parameter set, and comprise notably the permissions, but not the presets of this parameter set. :return: the settings of the parameter set, as a dict. The parameter set's settings consist of the permissions controlling whether the presets of the parameter set can be created inline or at the project level. :rtype: dict """ return self._settings def list_preset_names(self): """ List the names of the presets of this parameter set. :rtype: list[string] """ return [p["name"] for p in self._presets] def list_presets(self): """ List the presets of this parameter set. :rtype: list[:class:`DSSPluginPreset`] """ return [DSSPluginPreset(self._plugin_settings, p, self._desc) for p in self._presets] def get_preset(self, preset_name): """ Get a preset of this parameter set. :param string preset_name: name of a preset :return: a handle on the preset definition, or None if the preset doesn't exist :rtype: :class:`DSSPluginPreset` """ for p in self._presets: if p["name"] == preset_name: return DSSPluginPreset(self._plugin_settings, p, self._desc) return None def delete_preset(self, preset_name): """ Remove a preset from this plugin's settings :param string preset_name: name for the preset to remove """ preset = self.get_preset(preset_name) if preset is not None: # note: preset out of get_preset is a DSSPluginPreset, so it's # not the object present in the lists self._plugin_settings.settings["presets"].remove(preset._settings) self._presets.remove(preset._settings) else: raise Exception("Preset '%s' not found" % preset_name) def create_preset(self, preset_name, with_defaults=False): """ Create a new preset of this parameter set in the plugin settings. :param string preset_name: name for the preset to create :param bool with_defaults: if True, fill the new preset with the default value for each parameter :return: a preset definition, as a :class:`DSSPluginPreset` (see :meth:`get_preset()`) :rtype: dict """ for p in self._presets: if p["name"] == preset_name: raise Exception("A preset of the same name already exists") new_preset = {"name":preset_name, "type":self._settings["type"], "config":{}, "pluginConfig":{}, "defaultPermission":{}, "permissions":[]} if with_defaults: for p in self._desc["desc"]["params"]: v = p.get("defaultValue") if v is not None: new_preset["config"][p["name"]] = v for p in self._desc["desc"]["pluginParams"]: v = p.get("defaultValue") if v is not None: new_preset["pluginConfig"][p["name"]] = v self._plugin_settings.settings["presets"].append(new_preset) self._presets.append(new_preset) return self.get_preset(preset_name) def save(self): """ Save the settings to DSS. """ self._plugin_settings.save() class DSSPluginParameterSet(DSSPluginParameterSetBase): """ A parameter set in a plugin. .. important:: Do not instantiate directly, use :meth:`DSSPluginSettings.get_parameter_set()` or :meth:`DSSPluginSettings.list_parameter_sets()`. The values in this class can be modified directly, and changes will be taken into account when calling :meth:`DSSPluginSettings.save()` """ def __init__(self, plugin_settings, desc, settings, presets): self._plugin_settings = plugin_settings self._desc = desc self._settings = settings self._presets = presets @property def definable_inline(self): """ Whether presets for this parameter set can be defined directly in the form of the datasets, recipes, ... :rtype: bool """ return self._settings["defaultPermission"].get("definableInline", False) @definable_inline.setter def definable_inline(self, definable): self._settings["defaultPermission"]['definableInline'] = definable @property def definable_at_project_level(self): """ Whether presets for this parameter set can be defined at the project level :rtype: bool """ return self._settings["defaultPermission"].get("definableAtProjectLevel", False) @definable_at_project_level.setter def definable_at_project_level(self, definable): self._settings["defaultPermission"]['definableAtProjectLevel'] = definable class DSSPluginProjectParameterSet(DSSPluginParameterSetBase): """ A parameter set in a plugin. .. important:: Do not instantiate directly, use :meth:`DSSPluginProjectSettings.get_parameter_set()` or :meth:`DSSPluginProjectSettings.list_parameter_sets()` The values in this class can be modified directly, and changes will be taken into account when calling or :meth:`DSSPluginProjectSettings.save()` """ def __init__(self, plugin_settings, desc, settings, presets): self._plugin_settings = plugin_settings self._desc = desc self._settings = settings self._presets = presets class DSSPluginPreset(dict): """ A preset of a parameter set in a plugin. .. important:: Do not instantiate directly, use :meth:`DSSPluginParameterSet.get_preset()` , :meth:`DSSPluginParameterSet.list_presets()` or :meth:`DSSPluginParameterSet.create_preset()`. For project-level presets, use :meth:`DSSPluginProjectParameterSet.get_preset()` , :meth:`DSSPluginProjectParameterSet.list_presets()` or :meth:`DSSPluginProjectParameterSet.create_preset()` The values in this class can be modified directly, and changes will be taken into account when calling :meth:`DSSPluginSettings.save()`. """ def __init__(self, plugin_settings, settings, parameter_set_desc): super(DSSPluginPreset, self).__init__(settings) self._plugin_settings = plugin_settings self._settings = settings self._parameter_set_desc = parameter_set_desc def __repr__(self): return "DSSPluginPreset(name={}, plugin={}, parameter_set={})".format( self._settings["name"], self._plugin_settings.plugin_id, self._parameter_set_desc["id"] ) def get_raw(self): """ Get the raw settings of the preset object. .. note:: This method returns a reference to the preset, not a copy. Changing values in the reference then calling :meth:`save()` results in these changes being saved. :return: the preset's complete settings :rtype: dict """ return self._settings @property def name(self): """ Get the name of the preset. :return: the name of the preset :rtype: string """ return self._settings["name"] @property def config(self): """ Get the raw config of the preset object. .. note:: This method returns a reference to the preset, not a copy. Changing values in the reference then calling :meth:`save()` results in these changes being saved. :return: the preset's config as a dict. Each parameter of the parameter set is a field in the dict. :rtype: dict """ return self._settings["config"] @property def plugin_config(self): """ Get the raw admin-level config of the preset object. Admin-level config parameters are not shown in the UI to non-admin users. .. note:: This method returns a reference to the preset, not a copy. Changing values in the reference then calling :meth:`save()` results in these changes being saved. :return: the preset's admin config as a dict. Each parameter of the parameter set is a field in the dict. :rtype: dict """ return self._settings["pluginConfig"] @property def owner(self): """ The DSS user that owns this preset :rtype: string """ return self._settings["owner"] @owner.setter def owner(self, login): self._settings["owner"] = login @property def usable_by_all(self): """ Whether the preset is usable by any DSS user :rtype: bool """ return self._settings["defaultPermission"].get("use", False) @usable_by_all.setter def usable_by_all(self, use): self._settings["defaultPermission"]['use'] = use def get_permission_item(self, group): """ Get permissions on the preset for a given group :param string group: the name of the DSS group you want to check permissions for. :return: the permissions as a dict :rtype: dict """ if group is None: return self._settings["defaultPermission"] else: for p in self._settings["permissions"]: if p["group"] == group: return p return None def is_usable_by(self, group): """ Get whether the preset is usable by DSS users in a group :param string group: the name of the DSS group you want to check permissions for. :return: True if the preset can be used by DSS users belonging to *group*. If *group* is None then returns True if the preset can be used by any DSS user (like :meth:`usable_by_all`) :rtype: bool """ permission_item = self.get_permission_item(group) if permission_item is None: return self._settings["defaultPermission"]["use"] else: return permission_item["use"] def set_usable_by(self, group, use): """ Set whether the preset is usable by DSS users in a group :param string group: the name of the DSS group you want to change permissions for. :param bool use: whether the group should be allowed to use the preset or not """ permission_item = self.get_permission_item(group) if permission_item is None: # group can't be None here self._settings["permissions"].append({"group":group, "use":use}) else: permission_item["use"] = use def save(self): """ Save the settings to DSS. """ self._plugin_settings.save() class DSSPlugin(object): """ A plugin on the DSS instance. .. important:: Do not instantiate directly, use :meth:`dataikuapi.DSSClient.get_plugin()` """ def __init__(self, client, plugin_id): self.client = client self.plugin_id = plugin_id ######################################################## # Settings ######################################################## def get_settings(self): """ Get the plugin-level settings. .. note:: This call requires an API key with either: * DSS admin permissions * permission to develop plugins * tied to a user with admin privileges on the plugin :return: a handle on the settings :rtype: :class:`DSSPluginSettings` """ settings = self.client._perform_json("GET", "/plugins/%s/settings" % (self.plugin_id)) return DSSPluginSettings(self.client, self.plugin_id, settings) def get_project_settings(self, project_key): """ Get the project-level settings. .. note:: This call requires an API key with either: * DSS admin permissions * permission to develop plugins * tied to a user with admin privileges on the plugin :return: a handle on the project-level settings :rtype: :class:`DSSPluginProjectSettings` """ settings = self.client._perform_json("GET", "/plugins/%s/settings" % (self.plugin_id), params={"projectKey": project_key}) return DSSPluginProjectSettings(self.client, self.plugin_id, settings, project_key) ######################################################## # Code env ######################################################## def create_code_env(self, python_interpreter=None, conda=False): """ Start the creation of the code env of the plugin. .. note:: This call requires an API key with either: * DSS admin permissions * permission to develop plugins * tied to a user with admin privileges on the plugin If not passing any value to **python_interpreter**, the default defined by the plugin will be used. Usage example: .. code-block:: python # create a default code env for the plugin plugin = client.get_plugin('the-plugin-id') future = plugin.create_code_env() creation = future.wait_for_result() # take the name of the new code env env_name = creation["envName"] # set it as the current plugin code env settings = plugin.get_settings() settings.set_code_env(env_name) settings.save() :param string python_interpreter: which version of python to use. Possible values: PYTHON27, PYTHON34, PYTHON35, PYTHON36, PYTHON37, PYTHON38, PYTHON39, PYTHON310, PYTHON311 :param boolean conda: if True use conda to create the code env, if False use virtualenv and pip. :return: a handle on the operation :rtype: :class:`dataikuapi.dss.future.DSSFuture` """ ret = self.client._perform_json("POST", "/plugins/%s/code-env/actions/create" % (self.plugin_id), body={ "deploymentMode" : "PLUGIN_MANAGED", "conda": conda, "pythonInterpreter": python_interpreter }) return DSSFuture.from_resp(self.client, ret) def update_code_env(self): """ Start an update of the code env of the plugin. .. note:: This call requires an API key with either: * DSS admin permissions * permission to develop plugins * tied to a user with admin privileges on the plugin Usage example: .. code-block:: python # update the plugin code env after updating the plugin plugin = client.get_plugin('the-plugin-id') future = plugin.update_code_env() future.wait_for_result() :return: a handle on the operation :rtype: :class:`dataikuapi.dss.future.DSSFuture` """ ret = self.client._perform_json("POST", "/plugins/%s/code-env/actions/update" % (self.plugin_id)) return DSSFuture.from_resp(self.client, ret) ######################################################## # Plugin update ######################################################## def update_from_zip(self, fp): """ Update the plugin from a plugin archive (as a file object). .. note:: This call requires an API key with either: * DSS admin permissions * permission to develop plugins * tied to a user with admin privileges on the plugin :param file-like fp: A file-like object pointing to a plugin archive zip """ files = {'file': fp } self.client._perform_json("POST", "/plugins/%s/actions/updateFromZip" % (self.plugin_id), files=files) def start_update_from_zip(self, fp): """ Update the plugin from a plugin archive (as a file object). Returns immediately with a future representing the process done asynchronously .. note:: This call requires an API key with either: * DSS admin permissions * permission to develop plugins * tied to a user with admin privileges on the plugin :param file-like fp: A file-like object pointing to a plugin archive zip :return: A :class:`~dataikuapi.dss.future.DSSFuture` representing the update process :rtype: :class:`~dataikuapi.dss.future.DSSFuture` """ files = {'file': fp } f = self.client._perform_json("POST", "/plugins/%s/actions/future/updateFromZip" % (self.plugin_id), files=files) return DSSFuture.from_resp(self.client, f) def update_from_store(self): """ Update the plugin from the Dataiku plugin store. .. note:: This call requires an API key with either: * DSS admin permissions * permission to develop plugins * tied to a user with admin privileges on the plugin Usage example: .. code-block:: python # update a plugin that was installed from the store plugin = client.get_plugin("my-plugin-id") future = plugin.update_from_store() future.wait_for_result() :return: a handle on the operation :rtype: :class:`dataikuapi.dss.future.DSSFuture` """ ret = self.client._perform_json("POST", "/plugins/%s/actions/updateFromStore" % (self.plugin_id)) return DSSFuture.from_resp(self.client, ret) def update_from_git(self, repository_url, checkout = "master", subpath=None): """ Updates the plugin from a Git repository. .. note:: DSS must be setup to allow access to the repository. .. note:: This call requires an API key with either: * DSS admin permissions * permission to develop plugins * tied to a user with admin privileges on the plugin Usage example: .. code-block:: python # update a plugin that was installed from git plugin = client.get_plugin("my-plugin-id") future = plugin.update_from_git("git@github.com:myorg/myrepo") future.wait_for_result() :param string repository_url: URL of a Git remote :param string checkout: branch/tag/SHA1 to commit. For example "master" :param string subpath: Optional, path within the repository to use as plugin. Should contain a 'plugin.json' file :return: a handle on the operation :rtype: :class:`dataikuapi.dss.future.DSSFuture` """ ret = self.client._perform_json("POST", "/plugins/%s/actions/updateFromGit" % (self.plugin_id), body={ "gitRepositoryUrl": repository_url, "gitCheckout" : checkout, "gitSubpath": subpath }) return DSSFuture.from_resp(self.client, ret) ######################################################## # Plugin uninstall/delete ######################################################## def list_usages(self, project_key=None): """ Get the list of usages of the plugin. .. note:: This call requires an API key with either: * DSS admin permissions * permission to develop plugins * tied to a user with admin privileges on the plugin :param string project_key: optional key of project where to look for usages. Default is None and looking in all projects. :rtype: :class:`DSSPluginUsages` """ return DSSPluginUsages( self.client._perform_json("GET", "/plugins/{pluginId}/actions/listUsages".format(pluginId=self.plugin_id), params={"projectKey": project_key}) ) def delete(self, force=False): """ Delete a plugin. .. note:: This call requires an API key with either: * DSS admin permissions * permission to develop plugins * tied to a user with admin privileges on the plugin :param boolean force: if True, plugin will be deleted even if usages are found or errors occurred during usages analysis. Default: False. :rtype: :class:`dataikuapi.dss.future.DSSFuture` """ params = { "force": force } ret = self.client._perform_json("POST", "/plugins/{pluginId}/actions/delete".format(pluginId=self.plugin_id), body=params) return DSSFuture.from_resp(self.client, ret) ######################################################## # Managing the dev plugin's contents ######################################################## def list_files(self): """ Get the hierarchy of files in the plugin. .. note:: Dev plugins only .. note:: This call requires an API key with either: * DSS admin permissions * permission to develop plugins * tied to a user with admin privileges on the plugin :return: list of files or directories, each one a dict. Directories have a **children** field for recursion. Each dict has fields **name** and **path** (the path from the root of the plugin files) :rtype: dict """ return self.client._perform_json("GET", "/plugins/%s/contents" % (self.plugin_id)) def get_file(self, path): """ Get a file from the plugin folder. .. note:: Dev plugins only .. note:: This call requires an API key with either: * DSS admin permissions * permission to develop plugins * tied to a user with admin privileges on the plugin Usage example: .. code-block:: python # read the code env desc of a plugin plugin = client.get_plugin("my-plugin-name") with plugin.get_file('code-env/python/desc.json') as fp: desc = json.load(fp) :param string path: the path of the file, relative to the root of the plugin :return: the file's content :rtype: file-like """ return self.client._perform_raw("GET", "/plugins/%s/contents/%s" % (self.plugin_id, path)).raw def put_file(self, path, f): """ Update a file in the plugin folder. .. note:: Dev plugins only .. note:: This call requires an API key with either: * DSS admin permissions * permission to develop plugins * tied to a user with admin privileges on the plugin :param file-like f: the file contents, as a file-like object :param string path: the path of the file, relative ot the root of the plugin """ file_name = path.split('/')[-1] data = f.read() # eat it all, because making it work with a path variable and a MultifilePart in swing looks complicated self.client._perform_empty("POST", "/plugins/%s/contents/%s" % (self.plugin_id, path), raw_body=data) def rename_file(self, path, new_name): """ Rename a file/folder in the plugin. .. note:: Dev plugins only :param string path: the path of the file/folder, relative ot the root of the plugin :param string new_name: the new name of the file/folder """ self.client._perform_empty("POST", "/plugins/%s/contents-actions/rename" % self.plugin_id, body={"oldPath": path, "newName": new_name}) def move_file(self, path, new_path): """ Move a file/folder in the plugin. .. note:: Dev plugins only .. note:: This call requires an API key with either: * DSS admin permissions * permission to develop plugins * tied to a user with admin privileges on the plugin :param string path: the path of the file/folder, relative ot the root of the plugin :param string new_path: the new path relative at the root of the plugin """ self.client._perform_empty("POST", "/plugins/%s/contents-actions/move" % self.plugin_id, body={"oldPath": path, "newPath": new_path}) class DSSPluginUsage(object): """ Information on a usage of an element of a plugin. .. important:: Do no instantiate directly, use :meth:`dataikuapi.dss.plugin.DSSPlugin.list_usages()` """ def __init__(self, data): self._data = data @property def element_kind(self): """ Get the type of the plugin component. :return: a kind of plugin component, like 'python-clusters', 'python-connectors', 'python-fs-providers', 'webapps', ... :rtype: string """ return self._data["elementKind"] @property def element_type(self): """ Get the identifier of the plugin component. :rtype: string """ return self._data["elementType"] @property def object_id(self): """ Get the identifier of the object using the plugin component. :rtype: string """ return self._data["objectId"] @property def object_type(self): """ Get the type of the object using the plugin component. :return: a type of DSS object, like 'CLUSTER', 'DATASET', 'RECIPE', ... :rtype: string """ return self._data["objectType"] @property def project_key(self): """ Get the project key of the object using the plugin component. :rtype: string """ return self._data["projectKey"] class DSSMissingType(object): """ Information on a type not found while analyzing usages of a plugin. Missing types can occur when plugins stop defining a given component, for example during development, and DSS object still use the now-removed component. .. important:: Do no instantiate directly, use :meth:`dataikuapi.dss.plugin.DSSPlugin.list_usages()` """ def __init__(self, data): self._data = data @property def missing_type(self): """ Get the type of the plugin component. :rtype: string """ return self._data["missingType"] @property def object_id(self): """ Get the identifier of the object using the plugin component. :rtype: string """ return self._data["objectId"] @property def object_type(self): """ Get the type of the object using the plugin component. :return: a type of DSS object, like 'CLUSTER', 'DATASET', 'RECIPE', ... :rtype: string """ return self._data["objectType"] @property def project_key(self): """ Get the project key of the object using the plugin component :rtype: string """ return self._data["projectKey"] class DSSPluginUsages(object): """ Information on the usages of a plugin. .. important:: Do no instantiate directly, use :meth:`dataikuapi.dss.plugin.DSSPlugin.list_usages()` Some custom types may not be found during usage analysis, typically when a plugin was removed but is still used. This prevents some detailed analysis and may hide some uses. This information is provided in :meth:`missing_types()`. """ def __init__(self, data): """ Initialize a DSSPluginUsages from a dict of its properties :param dict data: the usages as json dict """ self._data = data self._usages = [] self._missing_types = [] for json_usage in data.get("usages", []): self._usages.append(DSSPluginUsage(json_usage)) for json_missing_type in data.get("missingTypes"): self._missing_types.append(DSSMissingType(json_missing_type)) def get_raw(self): """ Get the raw plugin usages. :return: a summary of the usages, as a dict with fields **usages** and **missingTypes**. :rtype: dict """ return self._data @property def usages(self): """ Get the list of usages of components of the plugin. :return: list of usages, each a :class:`DSSPluginUsage` :rtype: list """ return self._usages @property def missing_types(self): """ Get the list of usages of missing components of the plugin. :return: list of missing component types, each a :class:`DSSMissingType` :rtype: list """ return self._missing_types def maybe_used(self): """ Whether the plugin maybe in use. :return: True if usages were found, False if errors were encountered during analysis :rtype: boolean """ return not (not self.usages and not self.missing_types) dataiku-api-client-python-13.2.2/dataikuapi/dss/project.py000066400000000000000000004223211471211575100235200ustar00rootroot00000000000000import os.path as osp import warnings from . import recipe from .analysis import DSSAnalysis from .apiservice import DSSAPIService, DSSAPIServiceListItem from .app import DSSAppManifest from .codestudio import DSSCodeStudioObject, DSSCodeStudioObjectListItem from .continuousactivity import DSSContinuousActivity from .dashboard import DSSDashboard, DSSDashboardListItem, DASHBOARDS_URI_FORMAT from .dataset import DSSDataset, DSSDatasetListItem, DSSManagedDatasetCreationHelper from .discussion import DSSObjectDiscussions from .flow import DSSProjectFlow from .future import DSSFuture from .insight import DSSInsight, DSSInsightListItem, INSIGHTS_URI_FORMAT from .job import DSSJob, DSSJobWaiter from .jupyternotebook import DSSJupyterNotebook, DSSJupyterNotebookListItem from .labeling_task import DSSLabelingTask from .macro import DSSMacro from .managedfolder import DSSManagedFolder from .ml import DSSMLTask, DSSMLTaskQueues from .mlflow import DSSMLflowExtension from .modelcomparison import DSSModelComparison from .modelevaluationstore import DSSModelEvaluationStore from .notebook import DSSNotebook from .projectlibrary import DSSLibrary from .recipe import DSSRecipeListItem, DSSRecipe from .savedmodel import DSSSavedModel from .scenario import DSSScenario, DSSScenarioListItem from .sqlnotebook import DSSSQLNotebook, DSSSQLNotebookListItem from .streaming_endpoint import DSSStreamingEndpoint, DSSStreamingEndpointListItem, \ DSSManagedStreamingEndpointCreationHelper from .webapp import DSSWebApp, DSSWebAppListItem from .wiki import DSSWiki from .llm import DSSLLM, DSSLLMListItem from .knowledgebank import DSSKnowledgeBank, DSSKnowledgeBankListItem from ..dss_plugin_mlflow import MLflowHandle class DSSProject(object): """ A handle to interact with a project on the DSS instance. .. important:: Do not create this class directly, instead use :meth:`dataikuapi.DSSClient.get_project` """ def __init__(self, client, project_key): self.client = client self.project_key = project_key def get_summary(self): """ Returns a summary of the project. The summary is a read-only view of some of the state of the project. You cannot edit the resulting dict and use it to update the project state on DSS, you must use the other more specific methods of this :class:`dataikuapi.dss.project.DSSProject` object :returns: a dict containing a summary of the project. Each dict contains at least a **projectKey** field :rtype: dict """ return self.client._perform_json("GET", "/projects/%s" % self.project_key) def get_project_folder(self): """ Get the folder containing this project :rtype: :class:`dataikuapi.dss.projectfolder.DSSProjectFolder` """ root = self.client.get_root_project_folder() def rec(pf): if self.project_key in pf.list_project_keys(): return pf else: for spf in pf.list_child_folders(): found_in_child = rec(spf) if found_in_child: return found_in_child return None found_in = rec(root) if found_in: return found_in else: return root def move_to_folder(self, folder): """ Moves this project to a project folder :param folder: destination folder :type folder: :class:`dataikuapi.dss.projectfolder.DSSProjectFolder` """ current_folder = self.get_project_folder() current_folder.move_project_to(self.project_key, folder) ######################################################## # Project deletion ######################################################## def delete(self, clear_managed_datasets=False, clear_output_managed_folders=False, clear_job_and_scenario_logs=True, **kwargs): """ Delete the project .. attention:: This call requires an API key with admin rights :param bool clear_managed_datasets: Should the data of managed datasets be cleared (defaults to **False**) :param bool clear_output_managed_folders: Should the data of managed folders used as outputs of recipes be cleared (defaults to **False**) :param bool clear_job_and_scenario_logs: Should the job and scenario logs be cleared (defaults to **True**) """ # For backwards compatibility if 'drop_data' in kwargs and kwargs['drop_data']: clear_managed_datasets = True return self.client._perform_empty( "DELETE", "/projects/%s" % self.project_key, params={ "clearManagedDatasets": clear_managed_datasets, "clearOutputManagedFolders": clear_output_managed_folders, "clearJobAndScenarioLogs": clear_job_and_scenario_logs }) ######################################################## # Project export ######################################################## def get_export_stream(self, options=None): """ Return a stream of the exported project .. warning:: You need to close the stream after download. Failure to do so will result in the DSSClient becoming unusable. :param dict options: Dictionary of export options (defaults to **{}**). The following options are available: - **exportUploads** (boolean): Exports the data of Uploaded datasets (default to **False**) - **exportManagedFS** (boolean): Exports the data of managed Filesystem datasets (default to **False**) - **exportAnalysisModels** (boolean): Exports the models trained in analysis (default to **False**) - **exportSavedModels** (boolean): Exports the models trained in saved models (default to **False**) - **exportManagedFolders** (boolean): Exports the data of managed folders (default to **False**) - **exportAllInputDatasets** (boolean): Exports the data of all input datasets (default to **False**) - **exportAllDatasets** (boolean): Exports the data of all datasets (default to **False**) - **exportAllInputManagedFolders** (boolean): Exports the data of all input managed folders (default to **False**) - **exportGitRepository** (boolean): Exports the Git repository history (default to **False**) - **exportInsightsData** (boolean): Exports the data of static insights (default to **False**) :returns: a stream of the export archive :rtype: file-like object """ if options is None: options = {} return self.client._perform_raw( "POST", "/projects/%s/export" % self.project_key, body=options).raw def export_to_file(self, path, options=None): """ Export the project to a file :param str path: the path of the file in which the exported project should be saved :param dict options: Dictionary of export options (defaults to **{}**). The following options are available: * **exportUploads** (boolean): Exports the data of Uploaded datasets (default to **False**) * **exportManagedFS** (boolean): Exports the data of managed Filesystem datasets (default to **False**) * **exportAnalysisModels** (boolean): Exports the models trained in analysis (default to **False**) * **exportSavedModels** (boolean): Exports the models trained in saved models (default to **False**) * **exportModelEvaluationStores** (boolean): Exports the evaluation stores (default to **False**) * **exportManagedFolders** (boolean): Exports the data of managed folders (default to **False**) * **exportAllInputDatasets** (boolean): Exports the data of all input datasets (default to **False**) * **exportAllDatasets** (boolean): Exports the data of all datasets (default to **False**) * **exportAllInputManagedFolders** (boolean): \ Exports the data of all input managed folders (default to **False**) * **exportGitRepository** (boolean): Exports the Git repository history (default to **False**) * **exportInsightsData** (boolean): Exports the data of static insights (default to **False**) """ if options is None: options = {} with open(path, 'wb') as f: export_stream = self.client._perform_raw( "POST", "/projects/%s/export" % self.project_key, body=options) for chunk in export_stream.iter_content(chunk_size=32768): if chunk: f.write(chunk) f.flush() ######################################################## # Project duplicate ######################################################## def duplicate(self, target_project_key, target_project_name, duplication_mode="MINIMAL", export_analysis_models=True, export_saved_models=True, export_git_repository=True, export_insights_data=True, remapping=None, target_project_folder=None): """ Duplicate the project :param str target_project_key: The key of the new project :param str target_project_name: The name of the new project :param str duplication_mode: can be one of the following values: MINIMAL, SHARING, FULL, NONE (defaults to **MINIMAL**) :param bool export_analysis_models: (defaults to **True**) :param bool export_saved_models: (defaults to **True**) :param bool export_git_repository: (defaults to **True**) :param bool export_insights_data: (defaults to **True**) :param dict remapping: dict of connections to be remapped for the new project (defaults to **{}**) :param target_project_folder: the project folder where to put the duplicated project (defaults to **None**) :type target_project_folder: A :class:`dataikuapi.dss.projectfolder.DSSProjectFolder` :returns: A dict containing the original and duplicated project's keys :rtype: dict """ if remapping is None: remapping = {} obj = { "targetProjectName": target_project_name, "targetProjectKey": target_project_key, "duplicationMode": duplication_mode, "exportAnalysisModels": export_analysis_models, "exportSavedModels": export_saved_models, "exportGitRepository": export_git_repository, "exportInsightsData": export_insights_data, "remapping": remapping } if target_project_folder is not None: obj["targetProjectFolderId"] = target_project_folder.project_folder_id ref = self.client._perform_json("POST", "/projects/%s/duplicate/" % self.project_key, body=obj) return ref ######################################################## # Project infos ######################################################## def get_metadata(self): """ Get the metadata attached to this project. The metadata contains label, description checklists, tags and custom metadata of the project. .. note:: For more information on available metadata, please see https://doc.dataiku.com/dss/api/latest/rest/ :returns: the project metadata. :rtype: dict """ return self.client._perform_json("GET", "/projects/%s/metadata" % self.project_key) def set_metadata(self, metadata): """ Set the metadata on this project. Usage example: .. code-block:: python project_metadata = project.get_metadata() project_metadata['tags'] = ['tag1','tag2'] project.set_metadata(project_metadata) :param dict metadata: the new state of the metadata for the project. You should only set a metadata object\ that has been retrieved using the :meth:`get_metadata` call. """ return self.client._perform_empty( "PUT", "/projects/%s/metadata" % self.project_key, body=metadata) def get_settings(self): """ Gets the settings of this project. This does not contain permissions. See :meth:`get_permissions` :returns: a handle to read, modify and save the settings :rtype: :class:`dataikuapi.dss.project.DSSProjectSettings` """ ret = self.client._perform_json("GET", "/projects/%s/settings" % self.project_key) return DSSProjectSettings(self.client, self.project_key, ret) def get_permissions(self): """ Get the permissions attached to this project :returns: A dict containing the owner and the permissions, as a list of pairs of group name and permission type :rtype: dict """ return self.client._perform_json( "GET", "/projects/%s/permissions" % self.project_key) def set_permissions(self, permissions): """ Sets the permissions on this project Usage example: .. code-block:: python project_permissions = project.get_permissions() project_permissions['permissions'].append({'group':'data_scientists', 'readProjectContent': True, 'readDashboards': True}) project.set_permissions(project_permissions) :param dict permissions: a permissions object with the same structure as the one returned by\ :meth:`get_permissions` call """ return self.client._perform_empty( "PUT", "/projects/%s/permissions" % self.project_key, body=permissions) def get_interest(self): """ Get the interest of this project. The interest means the number of watchers and the number of stars. :returns: a dict object containing the interest of the project with two fields: * **starCount**: number of stars for this project * **watchCount**: number of users watching this project :rtype: dict """ return self.client._perform_json("GET", "/projects/%s/interest" % self.project_key) def get_timeline(self, item_count=100): """ Get the timeline of this project. The timeline consists of information about the creation of this project (by whom, and when), the last modification of this project (by whom and when), a list of contributors, and a list of modifications. This list of modifications contains a maximum of **item_count** elements (default to 100). If **item_count** is greater than the real number of modification, **item_count** is adjusted. :param int item_count: maximum number of modifications to retrieve in the items list :returns: a timeline where the top-level fields are : * **allContributors**: all contributors who have been involved in this project * **items**: a history of the modifications of the project * **createdBy**: who created this project * **createdOn**: when the project was created * **lastModifiedBy**: who modified this project for the last time * **lastModifiedOn**: when this modification took place :rtype: dict """ return self.client._perform_json("GET", "/projects/%s/timeline" % self.project_key, params={ "itemCount": item_count }) ######################################################## # Datasets ######################################################## def list_datasets(self, as_type="listitems"): """ List the datasets in this project. :param str as_type: How to return the list. Supported values are "listitems" and "objects" (defaults to **listitems**). :returns: The list of the datasets. If "as_type" is "listitems", each one as a :class:`dataikuapi.dss.dataset.DSSDatasetListItem`. If "as_type" is "objects", each one as a :class:`dataikuapi.dss.dataset.DSSDataset` :rtype: list """ items = self.client._perform_json("GET", "/projects/%s/datasets/" % self.project_key) if as_type == "listitems" or as_type == "listitem": return [DSSDatasetListItem(self.client, item) for item in items] elif as_type == "objects" or as_type == "object": return [DSSDataset(self.client, self.project_key, item["name"]) for item in items] else: raise ValueError("Unknown as_type") def get_dataset(self, dataset_name): """ Get a handle to interact with a specific dataset :param str dataset_name: the name of the desired dataset :returns: A dataset handle :rtype: :class:`dataikuapi.dss.dataset.DSSDataset` """ return DSSDataset(self.client, self.project_key, dataset_name) def create_dataset(self, dataset_name, type, params=None, formatType=None, formatParams=None): """ Create a new dataset in the project, and return a handle to interact with it. The precise structure of **params** and **formatParams** depends on the specific dataset type and dataset format type. To know which fields exist for a given dataset type and format type, create a dataset from the UI, and use :meth:`get_dataset` to retrieve the configuration of the dataset and inspect it. Then reproduce a similar structure in the :meth:`create_dataset` call. Not all settings of a dataset can be set at creation time (for example partitioning). After creation, you'll have the ability to modify the dataset :param str dataset_name: the name of the dataset to create. Must not already exist :param str type: the type of the dataset :param dict params: the parameters for the type, as a python dict (defaults to **{}**) :param str formatType: an optional format to create the dataset with (only for file-oriented datasets) :param dict formatParams: the parameters to the format, as a python dict (only for file-oriented datasets, default to **{}**) :returns: A dataset handle :rtype: :class:`dataikuapi.dss.dataset.DSSDataset` """ if params is None: params = {} if formatParams is None: formatParams = {} obj = { "name": dataset_name, "projectKey": self.project_key, "type": type, "params": params, "formatType": formatType, "formatParams": formatParams } self.client._perform_json("POST", "/projects/%s/datasets/" % self.project_key, body=obj) return DSSDataset(self.client, self.project_key, dataset_name) def create_upload_dataset(self, dataset_name, connection=None): """ Create a new dataset of type 'UploadedFiles' in the project, and return a handle to interact with it. :param str dataset_name: the name of the dataset to create. Must not already exist :param str connection: the name of the upload connection (defaults to **None**) :returns: A dataset handle :rtype: :class:`dataikuapi.dss.dataset.DSSDataset` """ obj = { "name": dataset_name, "projectKey": self.project_key, "type": "UploadedFiles", "params": {} } if connection is not None: obj["params"]["uploadConnection"] = connection self.client._perform_json("POST", "/projects/%s/datasets/" % self.project_key, body=obj) return DSSDataset(self.client, self.project_key, dataset_name) def create_filesystem_dataset(self, dataset_name, connection, path_in_connection): """ Create a new filesystem dataset in the project, and return a handle to interact with it. :param str dataset_name: the name of the dataset to create. Must not already exist :param str connection: the name of the connection :param str path_in_connection: the path of the dataset in the connection :returns: A dataset handle :rtype: :class:`dataikuapi.dss.dataset.DSSDataset` """ return self.create_fslike_dataset(dataset_name, "Filesystem", connection, path_in_connection) def create_s3_dataset(self, dataset_name, connection, path_in_connection, bucket=None): """ Creates a new external S3 dataset in the project and returns a :class:`dataikuapi.dss.dataset.DSSDataset` to interact with it. The created dataset does not have its format and schema initialized, it is recommended to use :meth:`~dataikuapi.dss.dataset.DSSDataset.autodetect_settings` on the returned object :param str dataset_name: the name of the dataset to create. Must not already exist :param str connection: the name of the connection :param str path_in_connection: the path of the dataset in the connection :param str bucket: the name of the s3 bucket (defaults to **None**) :returns: A dataset handle :rtype: :class:`dataikuapi.dss.dataset.DSSDataset` """ extra_params = {} if bucket is not None: extra_params["bucket"] = bucket return self.create_fslike_dataset(dataset_name, "S3", connection, path_in_connection, extra_params) def create_gcs_dataset(self, dataset_name, connection, path_in_connection, bucket=None): """ Creates a new external GCS dataset in the project and returns a :class:`dataikuapi.dss.dataset.DSSDataset` to interact with it. The created dataset does not have its format and schema initialized, it is recommended to use :meth:`~dataikuapi.dss.dataset.DSSDataset.autodetect_settings` on the returned object :param str dataset_name: the name of the dataset to create. Must not already exist :param str connection: the name of the connection :param str path_in_connection: the path of the dataset in the connection :param str bucket: the name of the GCS bucket (defaults to **None**) :returns: A dataset handle :rtype: :class:`dataikuapi.dss.dataset.DSSDataset` """ extra_params = {} if bucket is not None: extra_params["bucket"] = bucket return self.create_fslike_dataset(dataset_name, "GCS", connection, path_in_connection, extra_params) def create_azure_blob_dataset(self, dataset_name, connection, path_in_connection, container=None): """ Creates a new external Azure dataset in the project and returns a :class:`dataikuapi.dss.dataset.DSSDataset` to interact with it. The created dataset does not have its format and schema initialized, it is recommended to use :meth:`~dataikuapi.dss.dataset.DSSDataset.autodetect_settings` on the returned object :param str dataset_name: the name of the dataset to create. Must not already exist :param str connection: the name of the connection :param str path_in_connection: the path of the dataset in the connection :param str container: the name of the storage account container (defaults to **None**) :returns: A dataset handle :rtype: :class:`dataikuapi.dss.dataset.DSSDataset` """ extra_params = {} if container is not None: extra_params["container"] = container return self.create_fslike_dataset(dataset_name, "Azure", connection, path_in_connection, extra_params) def create_fslike_dataset(self, dataset_name, dataset_type, connection, path_in_connection, extra_params=None): """ Create a new file-based dataset in the project, and return a handle to interact with it. :param str dataset_name: the name of the dataset to create. Must not already exist :param str dataset_type: the type of the dataset :param str connection: the name of the connection :param str path_in_connection: the path of the dataset in the connection :param dict extra_params: a python dict of extra parameters (defaults to **None**) :returns: A dataset handle :rtype: :class:`dataikuapi.dss.dataset.DSSDataset` """ body = { "name": dataset_name, "projectKey": self.project_key, "type": dataset_type, "params": { "connection": connection, "path": path_in_connection } } if extra_params is not None: body["params"].update(extra_params) self.client._perform_json("POST", "/projects/%s/datasets/" % self.project_key, body=body) return DSSDataset(self.client, self.project_key, dataset_name) def create_sql_table_dataset(self, dataset_name, type, connection, table, schema, catalog=None): """ Create a new SQL table dataset in the project, and return a handle to interact with it. :param str dataset_name: the name of the dataset to create. Must not already exist :param str type: the type of the dataset :param str connection: the name of the connection :param str table: the name of the table in the connection :param str schema: the schema of the table :param str catalog: [optional] the catalog of the table :returns: A dataset handle :rtype: :class:`dataikuapi.dss.dataset.DSSDataset` """ obj = { "name": dataset_name, "projectKey": self.project_key, "type": type, "params": { "connection": connection, "mode": "table", "table": table, "schema": schema, "catalog": catalog } } self.client._perform_json("POST", "/projects/%s/datasets/" % self.project_key, body=obj) return DSSDataset(self.client, self.project_key, dataset_name) def new_managed_dataset_creation_helper(self, dataset_name): """ .. caution:: Deprecated. Please use :meth:`new_managed_dataset` """ warnings.warn("new_managed_dataset_creation_helper is deprecated, please use new_managed_dataset", DeprecationWarning) return DSSManagedDatasetCreationHelper(self, dataset_name) def new_managed_dataset(self, dataset_name): """ Initializes the creation of a new managed dataset. Returns a :class:`dataikuapi.dss.dataset.DSSManagedDatasetCreationHelper` or one of its subclasses to complete the creation of the managed dataset. Usage example: .. code-block:: python builder = project.new_managed_dataset("my_dataset") builder.with_store_into("target_connection") dataset = builder.create() :param str dataset_name: Name of the dataset to create :returns: An object to create the managed dataset :rtype: :class:`dataikuapi.dss.dataset.DSSManagedDatasetCreationHelper` """ return DSSManagedDatasetCreationHelper(self, dataset_name) ################ # Labeling tasks ################ def get_labeling_task(self, labeling_task_id): """ Get a handle to interact with a specific labeling task :param str labeling_task_id: the id of the desired labeling task :returns: A labeling task handle :rtype: :class:`dataikuapi.dss.labeling_task.DSSLabelingTask` """ return DSSLabelingTask(self.client, self.project_key, labeling_task_id) ######################################################## # Streaming endpoints ######################################################## def list_streaming_endpoints(self, as_type="listitems"): """ List the streaming endpoints in this project. :param str as_type: How to return the list. Supported values are "listitems" and "objects" (defaults to **listitems**). :returns: The list of the streaming endpoints. If "as_type" is "listitems", each one as a :class:`dataikuapi.dss.streaming_endpoint.DSSStreamingEndpointListItem`. If "as_type" is "objects", each one as a :class:`dataikuapi.dss.streaming_endpoint.DSSStreamingEndpoint` :rtype: list """ items = self.client._perform_json("GET", "/projects/%s/streamingendpoints/" % self.project_key) if as_type == "listitems" or as_type == "listitem": return [DSSStreamingEndpointListItem(self.client, item) for item in items] elif as_type == "objects" or as_type == "object": return [DSSStreamingEndpoint(self.client, self.project_key, item["id"]) for item in items] else: raise ValueError("Unknown as_type") def get_streaming_endpoint(self, streaming_endpoint_name): """ Get a handle to interact with a specific streaming endpoint :param str streaming_endpoint_name: the name of the desired streaming endpoint :returns: A streaming endpoint handle :rtype: :class:`dataikuapi.dss.streaming_endpoint.DSSStreamingEndpoint` """ return DSSStreamingEndpoint(self.client, self.project_key, streaming_endpoint_name) def create_streaming_endpoint(self, streaming_endpoint_name, type, params=None): """ Create a new streaming endpoint in the project, and return a handle to interact with it. The precise structure of **params** depends on the specific streaming endpoint type. To know which fields exist for a given streaming endpoint type, create a streaming endpoint from the UI, and use :meth:`get_streaming_endpoint` to retrieve the configuration of the streaming endpoint and inspect it. Then reproduce a similar structure in the :meth:`create_streaming_endpoint` call. Not all settings of a streaming endpoint can be set at creation time (for example partitioning). After creation, you'll have the ability to modify the streaming endpoint. :param str streaming_endpoint_name: the name for the new streaming endpoint :param str type: the type of the streaming endpoint :param dict params: the parameters for the type, as a python dict (defaults to **{}**) :returns: A streaming endpoint handle :rtype: :class:`dataikuapi.dss.streaming_endpoint.DSSStreamingEndpoint` """ if params is None: params = {} obj = { "id": streaming_endpoint_name, "projectKey": self.project_key, "type": type, "params": params } self.client._perform_json("POST", "/projects/%s/streamingendpoints/" % self.project_key, body=obj) return DSSStreamingEndpoint(self.client, self.project_key, streaming_endpoint_name) def create_kafka_streaming_endpoint(self, streaming_endpoint_name, connection=None, topic=None): """ Create a new kafka streaming endpoint in the project, and return a handle to interact with it. :param str streaming_endpoint_name: the name for the new streaming endpoint :param str connection: the name of the kafka connection (defaults to **None**) :param str topic: the name of the kafka topic (defaults to **None**) :returns: A streaming endpoint handle :rtype: :class:`dataikuapi.dss.streaming_endpoint.DSSStreamingEndpoint` """ obj = { "id": streaming_endpoint_name, "projectKey": self.project_key, "type": "kafka", "params": {} } if connection is not None: obj["params"]["connection"] = connection if topic is not None: obj["params"]["topic"] = topic self.client._perform_json("POST", "/projects/%s/streamingendpoints/" % self.project_key, body=obj) return DSSStreamingEndpoint(self.client, self.project_key, streaming_endpoint_name) def create_httpsse_streaming_endpoint(self, streaming_endpoint_name, url=None): """ Create a new https streaming endpoint in the project, and return a handle to interact with it. :param str streaming_endpoint_name: the name for the new streaming endpoint :param str url: the url of the endpoint (defaults to **None**) :returns: A streaming endpoint handle :rtype: :class:`dataikuapi.dss.streaming_endpoint.DSSStreamingEndpoint` """ obj = { "id": streaming_endpoint_name, "projectKey": self.project_key, "type": "httpsse", "params": {} } if url is not None: obj["params"]["url"] = url self.client._perform_json("POST", "/projects/%s/streamingendpoints/" % self.project_key, body=obj) return DSSStreamingEndpoint(self.client, self.project_key, streaming_endpoint_name) def new_managed_streaming_endpoint(self, streaming_endpoint_name, streaming_endpoint_type=None): """ Initializes the creation of a new streaming endpoint. Returns a :class:`dataikuapi.dss.streaming_endpoint.DSSManagedStreamingEndpointCreationHelper` to complete the creation of the streaming endpoint :param str streaming_endpoint_name: Name of the new streaming endpoint - must be unique in the project :param str streaming_endpoint_type: Type of the new streaming endpoint (optional if it can be inferred from a connection type) :returns: An object to create the streaming endpoint :rtype: :class:`~dataikuapi.dss.streaming_endpoint.DSSManagedStreamingEndpointCreationHelper` """ return DSSManagedStreamingEndpointCreationHelper(self, streaming_endpoint_name, streaming_endpoint_type) ######################################################## # Lab and ML # Don't forget to synchronize with DSSDataset.* ######################################################## def create_prediction_ml_task(self, input_dataset, target_variable, ml_backend_type="PY_MEMORY", guess_policy="DEFAULT", prediction_type=None, wait_guess_complete=True): """Creates a new prediction task in a new visual analysis lab for a dataset. :param str input_dataset: the dataset to use for training/testing the model :param str target_variable: the variable to predict :param str ml_backend_type: ML backend to use, one of PY_MEMORY, MLLIB or H2O (defaults to **PY_MEMORY**) :param str guess_policy: Policy to use for setting the default parameters. Valid values are: DEFAULT, SIMPLE_FORMULA, DECISION_TREE, EXPLANATORY and PERFORMANCE (defaults to **DEFAULT**) :param str prediction_type: The type of prediction problem this is. If not provided the prediction type will be guessed. Valid values are: BINARY_CLASSIFICATION, REGRESSION, MULTICLASS (defaults to **None**) :param boolean wait_guess_complete: if False, the returned ML task will be in 'guessing' state, i.e. analyzing the input dataset to determine feature handling and algorithms (defaults to **True**). You should wait for the guessing to be completed by calling **wait_guess_complete** on the returned object before doing anything else (in particular calling **train** or **get_settings**) :returns: A ML task handle of type 'PREDICTION' :rtype: :class:`dataikuapi.dss.ml.DSSMLTask` """ obj = { "inputDataset": input_dataset, "taskType": "PREDICTION", "targetVariable": target_variable, "backendType": ml_backend_type, "guessPolicy": guess_policy } if prediction_type is not None: obj["predictionType"] = prediction_type ref = self.client._perform_json("POST", "/projects/%s/models/lab/" % self.project_key, body=obj) ret = DSSMLTask(self.client, self.project_key, ref["analysisId"], ref["mlTaskId"]) if wait_guess_complete: ret.wait_guess_complete() return ret def create_clustering_ml_task(self, input_dataset, ml_backend_type="PY_MEMORY", guess_policy="KMEANS", wait_guess_complete=True): """Creates a new clustering task in a new visual analysis lab for a dataset. The returned ML task will be in 'guessing' state, i.e. analyzing the input dataset to determine feature handling and algorithms. You should wait for the guessing to be completed by calling **wait_guess_complete** on the returned object before doing anything else (in particular calling **train** or **get_settings**) :param str ml_backend_type: ML backend to use, one of PY_MEMORY, MLLIB or H2O (defaults to **PY_MEMORY**) :param str guess_policy: Policy to use for setting the default parameters. Valid values are: KMEANS and ANOMALY_DETECTION (defaults to **KMEANS**) :param boolean wait_guess_complete: if False, the returned ML task will be in 'guessing' state, i.e. analyzing the input dataset to determine feature handling and algorithms (defaults to **True**). You should wait for the guessing to be completed by calling **wait_guess_complete** on the returned object before doing anything else (in particular calling **train** or **get_settings**) :returns: A ML task handle of type 'CLUSTERING' :rtype: :class:`dataikuapi.dss.ml.DSSMLTask` """ obj = { "inputDataset": input_dataset, "taskType": "CLUSTERING", "backendType": ml_backend_type, "guessPolicy": guess_policy } ref = self.client._perform_json("POST", "/projects/%s/models/lab/" % self.project_key, body=obj) mltask = DSSMLTask(self.client, self.project_key, ref["analysisId"], ref["mlTaskId"]) if wait_guess_complete: mltask.wait_guess_complete() return mltask def create_timeseries_forecasting_ml_task(self, input_dataset, target_variable, time_variable, timeseries_identifiers=None, guess_policy="TIMESERIES_DEFAULT", wait_guess_complete=True): """Creates a new time series forecasting task in a new visual analysis lab for a dataset. :param string input_dataset: The dataset to use for training/testing the model :param string target_variable: The variable to forecast :param string time_variable: Column to be used as time variable. Should be a Date (parsed) column. :param list timeseries_identifiers: List of columns to be used as time series identifiers (when the dataset has multiple series) :param string guess_policy: Policy to use for setting the default parameters. Valid values are: TIMESERIES_DEFAULT, TIMESERIES_STATISTICAL, and TIMESERIES_DEEP_LEARNING :param boolean wait_guess_complete: If False, the returned ML task will be in 'guessing' state, i.e. analyzing the input dataset to determine feature handling and algorithms. You should wait for the guessing to be completed by calling ``wait_guess_complete`` on the returned object before doing anything else (in particular calling ``train`` or ``get_settings``) :return: :class:`dataiku.dss.ml.DSSMLTask` """ obj = { "inputDataset": input_dataset, "taskType": "PREDICTION", "targetVariable": target_variable, "timeVariable": time_variable, "timeseriesIdentifiers": timeseries_identifiers, "backendType": "PY_MEMORY", "guessPolicy": guess_policy, "predictionType": "TIMESERIES_FORECAST" } ref = self.client._perform_json( "POST", "/projects/{project_key}/models/lab/".format(project_key=self.project_key), body=obj ) ret = DSSMLTask(self.client, self.project_key, ref["analysisId"], ref["mlTaskId"]) if wait_guess_complete: ret.wait_guess_complete() return ret def create_causal_prediction_ml_task(self, input_dataset, outcome_variable, treatment_variable, prediction_type=None, wait_guess_complete=True): """Creates a new causal prediction task in a new visual analysis lab for a dataset. :param string input_dataset: The dataset to use for training/testing the model :param string outcome_variable: The outcome to predict. :param string treatment_variable: Column to be used as treatment variable. :param string or None prediction_type: Valid values are: "CAUSAL_BINARY_CLASSIFICATION", "CAUSAL_REGRESSION" or None (in this case prediction_type will be set by the Guesser) :param boolean wait_guess_complete: If False, the returned ML task will be in 'guessing' state, i.e. analyzing the input dataset to determine feature handling and algorithms. You should wait for the guessing to be completed by calling ``wait_guess_complete`` on the returned object before doing anything else (in particular calling ``train`` or ``get_settings``) :return: :class:`dataiku.dss.ml.DSSMLTask` """ obj = { "inputDataset": input_dataset, "taskType": "PREDICTION", "targetVariable": outcome_variable, "treatmentVariable": treatment_variable, "backendType": "PY_MEMORY", "guessPolicy": "CAUSAL_PREDICTION", "predictionType": prediction_type # If None, predictionType will be set by the Guesser } ref = self.client._perform_json( "POST", "/projects/{project_key}/models/lab/".format(project_key=self.project_key), body=obj ) ret = DSSMLTask(self.client, self.project_key, ref["analysisId"], ref["mlTaskId"]) if wait_guess_complete: ret.wait_guess_complete() return ret def list_ml_tasks(self): """ List the ML tasks in this project :returns: the list of the ML tasks summaries, each one as a python dict :rtype: list """ return self.client._perform_json("GET", "/projects/%s/models/lab/" % self.project_key) def get_ml_task(self, analysis_id, mltask_id): """ Get a handle to interact with a specific ML task :param str analysis_id: the identifier of the visual analysis containing the desired ML task :param str mltask_id: the identifier of the desired ML task :returns: A ML task handle :rtype: :class:`dataikuapi.dss.ml.DSSMLTask` """ return DSSMLTask(self.client, self.project_key, analysis_id, mltask_id) def list_mltask_queues(self): """ List non-empty ML task queues in this project :returns: an iterable listing of MLTask queues (each a dict) :rtype: :class:`dataikuapi.dss.ml.DSSMLTaskQueues` """ data = self.client._perform_json("GET", "/projects/%s/models/labs/mltask-queues" % self.project_key) return DSSMLTaskQueues(data) def create_analysis(self, input_dataset): """ Creates a new visual analysis lab for a dataset. :param str input_dataset: the dataset to use for the analysis :returns: A visual analysis handle :rtype: :class:`dataikuapi.dss.analysis.DSSAnalysis` """ obj = { "inputDataset": input_dataset } ref = self.client._perform_json("POST", "/projects/%s/lab/" % self.project_key, body=obj) return DSSAnalysis(self.client, self.project_key, ref["id"]) def list_analyses(self): """ List the visual analyses in this project :returns: the list of the visual analyses summaries, each one as a python dict :rtype: list """ return self.client._perform_json("GET", "/projects/%s/lab/" % self.project_key) def get_analysis(self, analysis_id): """ Get a handle to interact with a specific visual analysis :param str analysis_id: the identifier of the desired visual analysis :returns: A visual analysis handle :rtype: :class:`dataikuapi.dss.analysis.DSSAnalysis` """ return DSSAnalysis(self.client, self.project_key, analysis_id) ######################################################## # Saved models ######################################################## def list_saved_models(self): """ List the saved models in this project :returns: the list of the saved models, each one as a python dict :rtype: list """ return self.client._perform_json( "GET", "/projects/%s/savedmodels/" % self.project_key) def get_saved_model(self, sm_id): """ Get a handle to interact with a specific saved model :param str sm_id: the identifier of the desired saved model :returns: A saved model handle :rtype: :class:`dataikuapi.dss.savedmodel.DSSSavedModel` """ return DSSSavedModel(self.client, self.project_key, sm_id) def create_mlflow_pyfunc_model(self, name, prediction_type=None): """ Creates a new external saved model for storing and managing MLFlow models :param str name: Human readable name for the new saved model in the flow :param str prediction_type: Optional (but needed for most operations). One of BINARY_CLASSIFICATION, MULTICLASS, REGRESSION or None. Defaults to None, standing for other prediction types. If the Saved Model has a None prediction type, scoring, inclusion in a bundle or in an API service will be possible, but features related to performance analysis and explainability will not be available. :returns: The created saved model handle :rtype: :class:`dataikuapi.dss.savedmodel.DSSSavedModel` """ model = { "savedModelType": "MLFLOW_PYFUNC", "predictionType": prediction_type, "name": name } id = self.client._perform_json("POST", "/projects/%s/savedmodels/" % self.project_key, body=model)["id"] return self.get_saved_model(id) def create_finetuned_llm_saved_model(self, name): """ Creates a new finetuned LLM Saved Model for finetuning using Python code :param str name: Human-readable name for the new saved model in the flow :returns: The created saved model handle :rtype: :class:`dataikuapi.dss.savedmodel.DSSSavedModel` """ model = { "savedModelType": "LLM_GENERIC", "name": name } id = self.client._perform_json("POST", "/projects/%s/savedmodels/" % self.project_key, body=model)["id"] return self.get_saved_model(id) def create_external_model(self, name, prediction_type, configuration): """ Creates a new Saved model that can contain external remote endpoints as versions. :param string name: Human-readable name for the new saved model in the flow :param string prediction_type: One of BINARY_CLASSIFICATION, MULTICLASS or REGRESSION :param dict configuration: A dictionary containing the desired external saved model configuration. - For SageMaker, the syntax is: .. code-block:: python configuration = { "protocol": "sagemaker", "region": "" "connection": "" } Where the parameters have the following meaning: - ``region``: The AWS region of the endpoint, e.g. ``eu-west-1`` - ``connection``: (optional) The DSS SageMaker connection to use for authentication. If not defined, credentials will be derived from environment. See the reference documentation for details. - For Databricks, the syntax is: .. code-block:: python configuration = { "protocol": "databricks", "connection": "" } - For AzureML, syntax is: .. code-block:: python configuration = { "protocol": "azure-ml", "connection": "", "subscription_id": "", "resource_group": "", "workspace": "" } Where the parameters have the following meaning: - ``connection``: (optional) The DSS Azure ML connection to use for authentication. If not defined, credentials will be derived from environment. See the reference documentation for details. - ``subscription_id``: The Azure subscription ID - ``resource_group``: The Azure resource group - ``workspace``: The Azure ML workspace - For Vertex AI, syntax is: .. code-block:: python configuration = { "protocol": "vertex-ai", "region": "" "connection": "", "project_id": " or " } Where the parameters have the following meaning: - ``region``: The GCP region of the endpoint, e.g. ``europe-west-1`` - ``connection``: (optional) The DSS Vertex AI connection to use for authentication. If not defined, credentials will be derived from environment. See the reference documentation for details. - ``project_id``: The ID or name of the GCP project - Example: create a saved model for SageMaker endpoints serving binary classification models in region eu-west-1 .. code-block:: python import dataiku client = dataiku.api_client() project = client.get_default_project() configuration = { "protocol": "sagemaker", "region": "eu-west-1" } sm = project.create_external_model("SaveMaker Proxy Model", "BINARY_CLASSIFICATION", configuration) - Example: create a saved model for Vertex AI endpoints serving regression models in region eu-west-1, on project "my-project", performing authentication using DSS connection "vertex_conn" of type "Vertex AI". .. code-block:: python import dataiku client = dataiku.api_client() project = client.get_default_project() configuration = { "protocol": "vertex-ai", "region": "europe-west1", "connection": "vertex_conn" "project_id": "my-project" } sm = project.create_external_model("Vertex AI Proxy Model", "BINARY_CLASSIFICATION", configuration) """ model = { "savedModelType": "PROXY_MODEL", "predictionType": prediction_type, "name": name, "proxyModelConfiguration": configuration } saved_model_id = self.client._perform_json("POST", "/projects/%s/savedmodels/" % self.project_key, body=model)["id"] return self.get_saved_model(saved_model_id) ######################################################## # Managed folders ######################################################## def list_managed_folders(self): """ List the managed folders in this project :returns: the list of the managed folders, each one as a python dict :rtype: list """ return self.client._perform_json( "GET", "/projects/%s/managedfolders/" % self.project_key) def get_managed_folder(self, odb_id): """ Get a handle to interact with a specific managed folder :param str odb_id: the identifier of the desired managed folder :returns: A managed folder handle :rtype: :class:`dataikuapi.dss.managedfolder.DSSManagedFolder` """ return DSSManagedFolder(self.client, self.project_key, odb_id) def create_managed_folder(self, name, folder_type=None, connection_name="filesystem_folders"): """ Create a new managed folder in the project, and return a handle to interact with it :param str name: the name of the managed folder :param str folder_type: type of storage (defaults to **None**) :param str connection_name: the connection name (defaults to **filesystem_folders**) :returns: A managed folder handle :rtype: :class:`dataikuapi.dss.managedfolder.DSSManagedFolder` """ obj = { "name": name, "projectKey": self.project_key, "type": folder_type, "params": { "connection": connection_name, "path": "/${projectKey}/${odbId}" } } res = self.client._perform_json("POST", "/projects/%s/managedfolders/" % self.project_key, body=obj) odb_id = res['id'] return DSSManagedFolder(self.client, self.project_key, odb_id) ######################################################## # Model evaluation stores ######################################################## def list_model_evaluation_stores(self): """ List the model evaluation stores in this project. :returns: The list of the model evaluation stores :rtype: list of :class:`dataikuapi.dss.modelevaluationstore.DSSModelEvaluationStore` """ items = self.client._perform_json("GET", "/projects/%s/modelevaluationstores/" % self.project_key) return [DSSModelEvaluationStore(self.client, self.project_key, item["id"]) for item in items] def get_model_evaluation_store(self, mes_id): """ Get a handle to interact with a specific model evaluation store :param str mes_id: the id of the desired model evaluation store :returns: A model evaluation store handle :rtype: :class:`dataikuapi.dss.modelevaluationstore.DSSModelEvaluationStore` """ return DSSModelEvaluationStore(self.client, self.project_key, mes_id) def create_model_evaluation_store(self, name): """ Create a new model evaluation store in the project, and return a handle to interact with it. :param str name: the name for the new model evaluation store :returns: A model evaluation store handle :rtype: :class:`dataikuapi.dss.modelevaluationstore.DSSModelEvaluationStore` """ obj = { "projectKey": self.project_key, "name": name } res = self.client._perform_json("POST", "/projects/%s/modelevaluationstores/" % self.project_key, body=obj) mes_id = res['id'] return DSSModelEvaluationStore(self.client, self.project_key, mes_id) ######################################################## # Model comparisons ######################################################## def list_model_comparisons(self): """ List the model comparisons in this project. :returns: The list of the model comparisons :rtype: list """ items = self.client._perform_json("GET", "/projects/%s/modelcomparisons/" % self.project_key) return [DSSModelComparison(self.client, self.project_key, item["id"]) for item in items] def get_model_comparison(self, mec_id): """ Get a handle to interact with a specific model comparison :param str mec_id: the id of the desired model comparison :returns: A model comparison handle :rtype: :class:`dataikuapi.dss.modelcomparison.DSSModelComparison` """ return DSSModelComparison(self.client, self.project_key, mec_id) def create_model_comparison(self, name, prediction_type): """ Create a new model comparison in the project, and return a handle to interact with it. :param str name: the name for the new model comparison :param str prediction_type: one of BINARY_CLASSIFICATION, REGRESSION, MULTICLASS, TIMESERIES_FORECAST, CAUSAL_BINARY_CLASSIFICATION, CAUSAL_REGRESSION :returns: A new model comparison handle :rtype: :class:`dataikuapi.dss.modelcomparison.DSSModelComparison` """ obj = { "projectKey": self.project_key, "displayName": name, "predictionType": prediction_type } res = self.client._perform_json("POST", "/projects/%s/modelcomparisons/" % self.project_key, body=obj) mec_id = res['id'] return DSSModelComparison(self.client, self.project_key, mec_id) ######################################################## # Jobs ######################################################## def list_jobs(self): """ List the jobs in this project :returns: a list of the jobs, each one as a python dict, containing both the definition and the state :rtype: list """ return self.client._perform_json( "GET", "/projects/%s/jobs/" % self.project_key) def get_job(self, id): """ Get a handler to interact with a specific job :param str id: the id of the desired job :returns: A job handle :rtype: :class:`dataikuapi.dss.job.DSSJob` """ return DSSJob(self.client, self.project_key, id) def start_job(self, definition): """ Create a new job, and return a handle to interact with it :param dict definition: The definition should contain: * the type of job (RECURSIVE_BUILD, NON_RECURSIVE_FORCED_BUILD, RECURSIVE_FORCED_BUILD,\ RECURSIVE_MISSING_ONLY_BUILD) * a list of outputs to build from the available types: (DATASET, MANAGED_FOLDER, SAVED_MODEL,\ STREAMING_ENDPOINT) * (Optional) a refreshHiveMetastore field (True or False) to specify whether to re-synchronize the Hive\ metastore for recomputed HDFS datasets. :returns: A job handle :rtype: :class:`dataikuapi.dss.job.DSSJob` """ job_def = self.client._perform_json("POST", "/projects/%s/jobs/" % self.project_key, body=definition) return DSSJob(self.client, self.project_key, job_def['id']) def start_job_and_wait(self, definition, no_fail=False): """ Starts a new job and waits for it to complete. :param dict definition: The definition should contain: * the type of job (RECURSIVE_BUILD, NON_RECURSIVE_FORCED_BUILD, RECURSIVE_FORCED_BUILD,\ RECURSIVE_MISSING_ONLY_BUILD) * a list of outputs to build from the available types: (DATASET, MANAGED_FOLDER, SAVED_MODEL,\ STREAMING_ENDPOINT) * (Optional) a refreshHiveMetastore field (True or False) to specify whether to re-synchronize the Hive\ metastore for recomputed HDFS datasets. :param bool no_fail: if true, the function won't fail even if the job fails or aborts (defaults to **False**) :returns: the final status of the job :rtype: str """ job_def = self.client._perform_json("POST", "/projects/%s/jobs/" % self.project_key, body=definition) job = DSSJob(self.client, self.project_key, job_def['id']) waiter = DSSJobWaiter(job) return waiter.wait(no_fail) def new_job(self, job_type='NON_RECURSIVE_FORCED_BUILD'): """ Create a job to be run. You need to add outputs to the job (i.e. what you want to build) before running it. .. code-block:: python job_builder = project.new_job() job_builder.with_output("mydataset") complete_job = job_builder.start_and_wait() print("Job %s done" % complete_job.id) :param str job_type: the type of job (RECURSIVE_BUILD, NON_RECURSIVE_FORCED_BUILD, RECURSIVE_FORCED_BUILD, RECURSIVE_MISSING_ONLY_BUILD) (defaults to `NON_RECURSIVE_FORCED_BUILD`) :returns: A job handle :rtype: :class:`dataikuapi.dss.project.JobDefinitionBuilder` """ return JobDefinitionBuilder(self, job_type) def new_job_definition_builder(self, job_type='NON_RECURSIVE_FORCED_BUILD'): """ .. caution:: Deprecated. Please use :meth:`new_job` """ warnings.warn("new_job_definition_builder is deprecated, please use new_job", DeprecationWarning) return JobDefinitionBuilder(self, job_type) ######################################################## # Jupyter Notebooks ######################################################## def list_jupyter_notebooks(self, active=False, as_type="object"): """ List the jupyter notebooks of a project. :param bool active: if True, only return currently running jupyter notebooks (defaults to **active**). :param string as_type: How to return the list. Supported values are "listitems" and "object" (defaults to **object**). :returns: The list of the notebooks. If "as_type" is "listitems", each one as a :class:`dataikuapi.dss.jupyternotebook.DSSJupyterNotebookListItem`, if "as_type" is "objects", each one as a :class:`dataikuapi.dss.jupyternotebook.DSSJupyterNotebook` :rtype: list of :class:`dataikuapi.dss.jupyternotebook.DSSJupyterNotebookListItem` or list of :class:`dataikuapi.dss.jupyternotebook.DSSJupyterNotebook` """ notebook_items = self.client._perform_json("GET", "/projects/%s/jupyter-notebooks/" % self.project_key, params={"active": active}) if as_type == "listitems" or as_type == "listitem": return [DSSJupyterNotebookListItem(self.client, notebook_item) for notebook_item in notebook_items] elif as_type == "objects" or as_type == "object": return [DSSJupyterNotebook(self.client, self.project_key, notebook_item["name"]) for notebook_item in notebook_items] else: raise ValueError("Unknown as_type") def get_jupyter_notebook(self, notebook_name): """ Get a handle to interact with a specific jupyter notebook :param str notebook_name: The name of the jupyter notebook to retrieve :returns: A handle to interact with this jupyter notebook :rtype: :class:`dataikuapi.dss.jupyternotebook.DSSJupyterNotebook` jupyter notebook handle """ return DSSJupyterNotebook(self.client, self.project_key, notebook_name) def create_jupyter_notebook(self, notebook_name, notebook_content): """ Create a new jupyter notebook and get a handle to interact with it :param str notebook_name: the name of the notebook to create :param dict notebook_content: the data of the notebook to create, as a dict. The data will be converted to a JSON string internally. Use :meth:`DSSJupyterNotebook.get_content()` on a similar existing **DSSJupyterNotebook** object in order to get a sample definition object. :returns: A handle to interact with the newly created jupyter notebook :rtype: dataikuapi.dss.jupyternotebook.DSSJupyterNotebook """ self.client._perform_json("POST", "/projects/%s/jupyter-notebooks/%s" % (self.project_key, notebook_name), body=notebook_content) return self.get_jupyter_notebook(notebook_name) ######################################################## # SQL Notebooks ######################################################## def list_sql_notebooks(self, as_type="listitems"): """ List the SQL notebooks of a project :param string as_type: How to return the list. Supported values are "listitems" and "objects" (defaults to **listitems**) :returns: The list of the notebooks. If "as_type" is "listitems", each one as a :class:`dataikuapi.dss.sqlnotebook.DSSSQLNotebookListItem`, if "as_type" is "objects", each one as a :class:`dataikuapi.dss.sqlnotebook.DSSSQLNotebook` :rtype: List of :class:`dataikuapi.dss.sqlnotebook.DSSSQLNotebookListItem` or list of :class:`dataikuapi.dss.sqlnotebook.DSSSQLNotebook` """ notebook_items = self.client._perform_json("GET", "/projects/%s/sql-notebooks/" % self.project_key) if as_type == "listitems" or as_type == "listitem": return [DSSSQLNotebookListItem(self.client, notebook_item) for notebook_item in notebook_items] elif as_type == "objects" or as_type == "object": return [DSSSQLNotebook(self.client, self.project_key, notebook_item["id"]) for notebook_item in notebook_items] else: raise ValueError("Unknown as_type") def get_sql_notebook(self, notebook_id): """ Get a handle to interact with a specific SQL notebook :param string notebook_id: The id of the SQL notebook to retrieve :returns: A handle to interact with this SQL notebook :rtype: :class:`dataikuapi.dss.sqlnotebook.DSSSQLNotebook` SQL notebook handle """ return DSSSQLNotebook(self.client, self.project_key, notebook_id) def create_sql_notebook(self, notebook_content): """ Create a new SQL notebook and get a handle to interact with it :param dict notebook_content: The data of the notebook to create, as a dict. The data will be converted to a JSON string internally. Use :meth:`DSSSQLNotebook.get_content()` on a similar existing **DSSSQLNotebook** object in order to get a sample definition object :returns: A handle to interact with the newly created SQL notebook :rtype: :class:`dataikuapi.dss.sqlnotebook.DSSSQLNotebook` SQL notebook handle """ notebook = self.client._perform_json("POST", "/projects/%s/sql-notebooks/" % self.project_key, body=notebook_content) return DSSSQLNotebook(self.client, self.project_key, notebook["id"]) ######################################################## # Continuous activities ######################################################## def list_continuous_activities(self, as_objects=True): """ List the continuous activities in this project :param bool as_objects: if True, returns a list of :class:`dataikuapi.dss.continuousactivity.DSSContinuousActivity` objects, else returns a list of python dicts (defaults to **True**) :returns: a list of the continuous activities, each one as a python dict, containing both the definition and the state :rtype: list """ list = self.client._perform_json("GET", "/projects/%s/continuous-activities/" % self.project_key) if as_objects: return [DSSContinuousActivity(self.client, a['projectKey'], a['recipeId']) for a in list] else: return list def get_continuous_activity(self, recipe_id): """ Get a handler to interact with a specific continuous activities :param str recipe_id: the identifier of the recipe controlled by the continuous activity :returns: A job handle :rtype: :class:`dataikuapi.dss.continuousactivity.DSSContinuousActivity` """ return DSSContinuousActivity(self.client, self.project_key, recipe_id) ######################################################## # Variables ######################################################## def get_variables(self): """ Gets the variables of this project. :returns: a dictionary containing two dictionaries : "standard" and "local". "standard" are regular variables, exported with bundles. "local" variables are not part of the bundles for this project :rtype: dict """ return self.client._perform_json( "GET", "/projects/%s/variables/" % self.project_key) def set_variables(self, obj): """ Sets the variables of this project. .. warning:: If executed from a python recipe, the changes made by `set_variables` will not be "seen" in that recipe. Use the internal API dataiku.get_custom_variables() instead if this behavior is needed :param dict obj: must be a modified version of the object returned by get_variables """ if not "standard" in obj: raise ValueError("Missing 'standard' key in argument") if not "local" in obj: raise ValueError("Missing 'local' key in argument") self.client._perform_empty( "PUT", "/projects/%s/variables/" % self.project_key, body=obj) def update_variables(self, variables, type="standard"): """ Updates a set of variables for this project :param variables dict: a dict of variable name -> value to set. Keys of the dict must be strings. Values in the dict can be strings, numbers, booleans, lists or dicts :param type str: Can be "standard" to update regular variables or "local" to update local-only variables that are not part of bundles for this project (defaults to **standard**) """ current_variables = self.get_variables() current_variables[type].update(variables) self.set_variables(current_variables) ######################################################## # API Services ######################################################## def list_api_services(self, as_type="listitems"): """ List the API services in this project :param str as_type: How to return the list. Supported values are "listitems" and "objects" (defaults to **listitems**). :returns: The list of the datasets. If "as_type" is "listitems", each one as a :class:`dataikuapi.dss.apiservice.DSSAPIServiceListItem`. If "as_type" is "objects", each one as a :class:`dataikuapi.dss.apiservice.DSSAPIService` :rtype: list """ items = self.client._perform_json("GET", "/projects/%s/apiservices/" % self.project_key) if as_type == "listitems" or as_type == "listitem": return [DSSAPIServiceListItem(self.client, item) for item in items] elif as_type == "objects" or as_type == "object": return [DSSAPIService(self.client, self.project_key, item["id"]) for item in items] else: raise ValueError("Unknown as_type") def create_api_service(self, service_id): """ Create a new API service, and returns a handle to interact with it. The newly-created service does not have any endpoint. :param str service_id: the ID of the API service to create :returns: A API Service handle :rtype: :class:`dataikuapi.dss.apiservice.DSSAPIService` """ self.client._perform_empty("POST", "/projects/%s/apiservices/%s" % (self.project_key, service_id)) return DSSAPIService(self.client, self.project_key, service_id) def get_api_service(self, service_id): """ Get a handle to interact with a specific API Service from the API Designer :param str service_id: The identifier of the API Designer API Service to retrieve :returns: A handle to interact with this API Service :rtype: :class:`dataikuapi.dss.apiservice.DSSAPIService` """ return DSSAPIService(self.client, self.project_key, service_id) ######################################################## # Bundles / Export and Publish (Design node) ######################################################## def list_exported_bundles(self): """ List all the bundles created in this project on the Design Node. :returns: A dictionary of all bundles for a project on the Design node. :rtype: dict """ return self.client._perform_json("GET", "/projects/%s/bundles/exported" % self.project_key) def export_bundle(self, bundle_id, release_notes=None): """ Creates a new project bundle on the Design node :param str bundle_id: bundle id tag :param str release_notes: important changes introduced in the bundle """ return self.client._perform_json( method="PUT", path="/projects/%s/bundles/exported/%s" % (self.project_key, bundle_id), params={"releaseNotes": release_notes} ) def delete_exported_bundle(self, bundle_id): """ Deletes a project bundle from the Design node :param str bundle_id: bundle id tag """ return self.client._perform_json("DELETE", "/projects/%s/bundles/exported/%s" % (self.project_key, bundle_id)) def get_exported_bundle_archive_stream(self, bundle_id): """ Download a bundle archive that can be deployed in a DSS automation Node, as a binary stream. .. warning:: The stream must be closed after use. Use a **with** statement to handle closing the stream at the end of the block by default. For example: .. code-block:: python with project.get_exported_bundle_archive_stream('v1') as fp: # use fp # or explicitly close the stream after use fp = project.get_exported_bundle_archive_stream('v1') # use fp, then close fp.close() :param str bundle_id: the identifier of the bundle """ return self.client._perform_raw("GET", "/projects/%s/bundles/exported/%s/archive" % (self.project_key, bundle_id)) def download_exported_bundle_archive_to_file(self, bundle_id, path): """ Download a bundle archive that can be deployed in a DSS automation Node into the given output file. :param str bundle_id: the identifier of the bundle :param str path: if "-", will write to /dev/stdout """ if path == "-": path = "/dev/stdout" stream = self.get_exported_bundle_archive_stream(bundle_id) with open(path, 'wb') as f: for chunk in stream.iter_content(chunk_size=10000): if chunk: f.write(chunk) f.flush() stream.close() def publish_bundle(self, bundle_id, published_project_key=None): """ Publish a bundle on the Project Deployer. :param str bundle_id: The identifier of the bundle :param str published_project_key: The key of the project on the Project Deployer where the bundle will be published.A new published project will be created if none matches the key. If the parameter is not set, the key from the current :class:`DSSProject` is used. :returns: a dict with info on the bundle state once published. It contains the keys "publishedOn" for the publish date, "publishedBy" for the user who published the bundle, and "publishedProjectKey" for the key of the Project Deployer project used. :rtype: dict """ params = None if published_project_key is not None: params = {"publishedProjectKey": published_project_key} return self.client._perform_json("POST", "/projects/%s/bundles/%s/publish" % (self.project_key, bundle_id), params=params) ######################################################## # Bundles / Import (Automation node) ######################################################## def list_imported_bundles(self): """ List all the bundles imported for this project, on the Automation node. :returns: a dict containing bundle imports for a project, on the Automation node. :rtype: dict """ return self.client._perform_json("GET", "/projects/%s/bundles/imported" % self.project_key) def import_bundle_from_archive(self, archive_path): """ Imports a bundle from a zip archive path on the Automation node. :param str archive_path: A full path to a zip archive, for example `/home/dataiku/my-bundle-v1.zip` """ return self.client._perform_json("POST", "/projects/%s/bundles/imported/actions/importFromArchive" % (self.project_key), params={"archivePath": osp.abspath(archive_path)}) def import_bundle_from_stream(self, fp): """ Imports a bundle from a file stream, on the Automation node. Usage example: .. code-block:: python project = client.get_project('MY_PROJECT') with open('/home/dataiku/my-bundle-v1.zip', 'rb') as f: project.import_bundle_from_stream(f) :param file-like fp: file handler. """ files = {'file': fp} return self.client._perform_empty("POST", "/projects/%s/bundles/imported/actions/importFromStream" % (self.project_key), files=files) def activate_bundle(self, bundle_id, scenarios_to_enable=None): """ Activates a bundle in this project. :param str bundle_id: The ID of the bundle to activate :param dict scenarios_to_enable: An optional dict of scenarios to enable or disable upon bundle activation. The format of the dict should be scenario IDs as keys with values of True or False (defaults to **{}**). :returns: A report containing any error or warning messages that occurred during bundle activation :rtype: dict """ options = {"scenariosActiveOnActivation": scenarios_to_enable} if scenarios_to_enable else {} return self.client._perform_json("POST", "/projects/%s/bundles/imported/%s/actions/activate" % ( self.project_key, bundle_id), body=options) def preload_bundle(self, bundle_id): """ Preloads a bundle that has been imported on the Automation node :param str bundle_id: the bundle_id for an existing imported bundle """ return self.client._perform_json("POST", "/projects/%s/bundles/imported/%s/actions/preload" % ( self.project_key, bundle_id)) ######################################################## # Scenarios ######################################################## def list_scenarios(self, as_type="listitems"): """ List the scenarios in this project. :param str as_type: How to return the list. Supported values are "listitems" and "objects" (defaults to **listitems**). :returns: The list of the datasets. If "rtype" is "listitems", each one as a :class:`dataikuapi.dss.scenario.DSSScenarioListItem`. If "rtype" is "objects", each one as a :class:`dataikuapi.dss.scenario.DSSScenario` :rtype: list """ items = self.client._perform_json("GET", "/projects/%s/scenarios/" % self.project_key) if as_type == "listitems": return [DSSScenarioListItem(self.client, item) for item in items] elif as_type == "objects": return [DSSScenario(self.client, self.project_key, item["id"]) for item in items] else: raise ValueError("Unknown as_type") def get_scenario(self, scenario_id): """ Get a handle to interact with a specific scenario :param str: scenario_id: the ID of the desired scenario :returns: A scenario handle :rtype: :class:`dataikuapi.dss.scenario.DSSScenario` """ return DSSScenario(self.client, self.project_key, scenario_id) def create_scenario(self, scenario_name, type, definition=None): """ Create a new scenario in the project, and return a handle to interact with it :param str scenario_name: The name for the new scenario. This does not need to be unique (although this is strongly recommended) :param str type: The type of the scenario. MUst be one of 'step_based' or 'custom_python' :param dict definition: the JSON definition of the scenario. Use **get_definition(with_status=False)** on an existing **DSSScenario** object in order to get a sample definition object (defaults to **{'params': {}}**) :returns: a :class:`dataikuapi.dss.scenario.DSSScenario` handle to interact with the newly-created scenario """ if definition is None: definition = {'params': {}} definition['type'] = type definition['name'] = scenario_name scenario_id = self.client._perform_json("POST", "/projects/%s/scenarios/" % self.project_key, body=definition)['id'] return DSSScenario(self.client, self.project_key, scenario_id) ######################################################## # Recipes ######################################################## def list_recipes(self, as_type="listitems"): """ List the recipes in this project :param str as_type: How to return the list. Supported values are "listitems" and "objects" (defaults to **listitems**). :returns: The list of the recipes. If "as_type" is "listitems", each one as a :class:`dataikuapi.dss.recipe.DSSRecipeListItem`. If "as_type" is "objects", each one as a :class:`dataikuapi.dss.recipe.DSSRecipe` :rtype: list """ items = self.client._perform_json("GET", "/projects/%s/recipes/" % self.project_key) if as_type == "listitems" or as_type == "listitem": return [DSSRecipeListItem(self.client, item) for item in items] elif as_type == "objects" or as_type == "object": return [DSSRecipe(self.client, self.project_key, item["name"]) for item in items] else: raise ValueError("Unknown as_type") def get_recipe(self, recipe_name): """ Gets a :class:`dataikuapi.dss.recipe.DSSRecipe` handle to interact with a recipe :param str recipe_name: The name of the recipe :returns: A recipe handle :rtype: :class:`dataikuapi.dss.recipe.DSSRecipe` """ return DSSRecipe(self.client, self.project_key, recipe_name) def create_recipe(self, recipe_proto, creation_settings): """ Create a new recipe in the project, and return a handle to interact with it. We strongly recommend that you use the creator helpers instead of calling this directly. Some recipe types require additional parameters in creation_settings: * 'grouping' : a 'groupKey' column name * 'python', 'sql_query', 'hive', 'impala' : the code of the recipe as a 'payload' string :param dict recipe_proto: a prototype for the recipe object. Must contain at least 'type' and 'name' :param dict creation_settings: recipe-specific creation settings :returns: A recipe handle :rtype: :class:`dataikuapi.dss.recipe.DSSRecipe` """ recipe_proto["projectKey"] = self.project_key definition = {'recipePrototype': recipe_proto, 'creationSettings': creation_settings} recipe_name = self.client._perform_json("POST", "/projects/%s/recipes/" % self.project_key, body=definition)['name'] return DSSRecipe(self.client, self.project_key, recipe_name) def new_recipe(self, type, name=None): """ Initializes the creation of a new recipe. Returns a :class:`dataikuapi.dss.recipe.DSSRecipeCreator` or one of its subclasses to complete the creation of the recipe. Usage example: .. code-block:: python grouping_recipe_builder = project.new_recipe("grouping") grouping_recipe_builder.with_input("dataset_to_group_on") # Create a new managed dataset for the output in the "filesystem_managed" connection grouping_recipe_builder.with_new_output("grouped_dataset", "filesystem_managed") grouping_recipe_builder.with_group_key("column") recipe = grouping_recipe_builder.build() # After the recipe is created, you can edit its settings recipe_settings = recipe.get_settings() recipe_settings.set_column_aggregations("value", sum=True) recipe_settings.save() # And you may need to apply new schemas to the outputs recipe.compute_schema_updates().apply() :param str type: Type of the recipe :param str name: Optional, base name for the new recipe. :returns: A new DSS Recipe Creator handle :rtype: :class:`dataikuapi.dss.recipe.DSSRecipeCreator` """ if type == "generate_features": return recipe.GenerateFeaturesRecipeCreator(name, self) elif type == "grouping": return recipe.GroupingRecipeCreator(name, self) elif type == "window": return recipe.WindowRecipeCreator(name, self) elif type == "sync": return recipe.SyncRecipeCreator(name, self) elif type == "csync": return recipe.ContinuousSyncRecipeCreator(name, self) elif type == "pivot": return recipe.PivotRecipeCreator(name, self) elif type == "sort": return recipe.SortRecipeCreator(name, self) elif type == "topn": return recipe.TopNRecipeCreator(name, self) elif type == "distinct": return recipe.DistinctRecipeCreator(name, self) elif type == "join": return recipe.JoinRecipeCreator(name, self) elif type == "vstack": return recipe.StackRecipeCreator(name, self) elif type == "sampling": return recipe.SamplingRecipeCreator(name, self) elif type == "split": return recipe.SplitRecipeCreator(name, self) elif type == "prepare" or type == "shaker": return recipe.PrepareRecipeCreator(name, self) elif type == "prediction_scoring": return recipe.PredictionScoringRecipeCreator(name, self) elif type == "evaluation": return recipe.EvaluationRecipeCreator(name, self) elif type == "standalone_evaluation": return recipe.StandaloneEvaluationRecipeCreator(name, self) elif type == "nlp_llm_evaluation": return recipe.LLMEvaluationRecipeCreator(name, self) elif type == "clustering_scoring": return recipe.ClusteringScoringRecipeCreator(name, self) elif type == "download": return recipe.DownloadRecipeCreator(name, self) elif type == "sql_query": return recipe.SQLQueryRecipeCreator(name, self) elif type in ["python", "r", "sql_script", "pyspark", "sparkr", "spark_scala", "shell"]: return recipe.CodeRecipeCreator(name, type, self) elif type in ["cpython", "ksql", "streaming_spark_scala"]: return recipe.CodeRecipeCreator(name, type, self) ######################################################## # Flow ######################################################## def get_flow(self): """ :returns: A Flow handle :rtype: A :class:`dataikuapi.dss.flow.DSSProjectFlow` """ return DSSProjectFlow(self.client, self) ######################################################## # Security ######################################################## def sync_datasets_acls(self): """ Resync permissions on HDFS datasets in this project .. attention:: This call requires an API key with admin rights :returns: a handle to the task of resynchronizing the permissions :rtype: :class:`dataikuapi.dss.future.DSSFuture` """ future_response = self.client._perform_json( "POST", "/projects/%s/actions/sync" % (self.project_key)) return DSSFuture(self.client, future_response.get('jobId', None), future_response) ######################################################## # Notebooks ######################################################## def list_running_notebooks(self, as_objects=True): """ .. caution:: Deprecated. Use :meth:`DSSProject.list_jupyter_notebooks` List the currently-running notebooks :returns: list of notebooks. Each object contains at least a 'name' field :rtype: list """ notebook_list = self.client._perform_json("GET", "/projects/%s/notebooks/active" % self.project_key) if as_objects: return [DSSNotebook(self.client, notebook['projectKey'], notebook['name'], notebook) for notebook in notebook_list] else: return notebook_list ######################################################## # Tags ######################################################## def get_tags(self): """ List the tags of this project. :returns: a dictionary containing the tags with a color :rtype: dict """ return self.client._perform_json("GET", "/projects/%s/tags" % self.project_key) def set_tags(self, tags=None): """ Set the tags of this project. :param dict tags: must be a modified version of the object returned by list_tags (defaults to **{}**) """ if tags is None: tags = {} return self.client._perform_empty("PUT", "/projects/%s/tags" % self.project_key, body=tags) ######################################################## # Macros ######################################################## def list_macros(self, as_objects=False): """ List the macros accessible in this project :param as_objects: if True, return the macros as :class:`dataikuapi.dss.macro.DSSMacro` macro handles instead of a list of python dicts (defaults to **False**) :returns: the list of the macros :rtype: list """ macros = self.client._perform_json( "GET", "/projects/%s/runnables/" % self.project_key) if as_objects: return [DSSMacro(self.client, self.project_key, m["runnableType"], m) for m in macros] else: return macros def get_macro(self, runnable_type): """ Get a handle to interact with a specific macro :param str runnable_type: the identifier of a macro :returns: A macro handle :rtype: :class:`dataikuapi.dss.macro.DSSMacro` """ return DSSMacro(self.client, self.project_key, runnable_type) ######################################################## # Wiki ######################################################## def get_wiki(self): """ Get the wiki :returns: the wiki associated to the project :rtype: :class:`dataikuapi.dss.wiki.DSSWiki` """ return DSSWiki(self.client, self.project_key) ######################################################## # Discussions ######################################################## def get_object_discussions(self): """ Get a handle to manage discussions on the project :returns: the handle to manage discussions :rtype: :class:`dataikuapi.dss.discussion.DSSObjectDiscussions` """ return DSSObjectDiscussions(self.client, self.project_key, "PROJECT", self.project_key) ######################################################## # Tables import ######################################################## def init_tables_import(self): """ Start an operation to import Hive or SQL tables as datasets into this project :returns: a :class:`dataikuapi.dss.project.TablesImportDefinition` to add tables to import :rtype: :class:`dataikuapi.dss.project.TablesImportDefinition` """ return TablesImportDefinition(self.client, self.project_key) def list_sql_schemas(self, connection_name): """ Lists schemas from which tables can be imported in a SQL connection :param str connection_name: name of the SQL connection :returns: an array of schemas names :rtype: list """ return self._list_schemas(connection_name) def list_hive_databases(self): """ Lists Hive databases from which tables can be imported :returns: an array of databases names :rtype: list """ return self._list_schemas("@virtual(hive-jdbc):default") def _list_schemas(self, connection_name): return self.client._perform_json("GET", "/projects/%s/datasets/tables-import/actions/list-schemas" % ( self.project_key), params={"connectionName": connection_name}) def list_sql_tables(self, connection_name, schema_name=None): """ Lists tables to import in a SQL connection :param str connection_name: name of the SQL connection :param str schema_name: Optional, name of the schema in the SQL connection in which to list tables. :returns: an array of tables :rtype: list """ ret = self.client._perform_json("GET", "/projects/%s/datasets/tables-import/actions/list-tables" % (self.project_key), params={"connectionName": connection_name, "schemaName": schema_name}) def to_schema_table_pair(x): return {"schema": x.get("schema", None), "table": x["table"]} return [to_schema_table_pair(x) for x in DSSFuture.get_result_wait_if_needed(self.client, ret)['tables']] def list_hive_tables(self, hive_database): """ Lists tables to import in a Hive database :param str hive_database: name of the Hive database :returns: an array of tables :rtype: list """ connection_name = "@virtual(hive-jdbc):" + hive_database ret = self.client._perform_json("GET", "/projects/%s/datasets/tables-import/actions/list-tables" % (self.project_key), params={"connectionName": connection_name}) def to_schema_table_pair(x): return {"schema": x.get("databaseName", None), "table": x["table"]} return [to_schema_table_pair(x) for x in DSSFuture.get_result_wait_if_needed(self.client, ret)['tables']] def list_elasticsearch_indices_or_aliases(self, connection_name): ret = self.client._perform_json("GET", "/projects/%s/datasets/tables-import/actions/list-indices" % self.project_key, params={"connectionName": connection_name}) return DSSFuture.get_result_wait_if_needed(self.client, ret) ######################################################## # App designer ######################################################## def get_app_manifest(self): """ Gets the manifest of the application if the project is an app template or an app instance, fails otherwise. :returns: the manifest of the application associated to the project :rtype: :class:`dataikuapi.dss.app.DSSAppManifest` """ raw_data = self.client._perform_json("GET", "/projects/%s/app-manifest" % self.project_key) return DSSAppManifest(self.client, raw_data, self.project_key) ######################################################## # MLflow experiment tracking ######################################################## def setup_mlflow(self, managed_folder, host=None): """ Set up the dss-plugin for MLflow :param object managed_folder: the managed folder where MLflow artifacts should be stored. Can be either a managed folder id as a string, a :class:`dataikuapi.dss.managedfolder.DSSManagedFolder`, or a :class:`dataiku.Folder` :param str host: setup a custom host if the backend used is not DSS (defaults to **None**). """ return MLflowHandle(client=self.client, project=self, managed_folder=managed_folder, host=host) def get_mlflow_extension(self): """ Get a handle to interact with the extension of MLflow provided by DSS :returns: A Mlflow Extension handle :rtype: :class:`dataikuapi.dss.mlflow.DSSMLflowExtension` """ return DSSMLflowExtension(client=self.client, project_key=self.project_key) ######################################################## # Code studios ######################################################## def list_code_studios(self, as_type="listitems"): """ List the code studio objects in this project :param str as_type: How to return the list. Supported values are "listitems" and "objects" (defaults to **listitems**). :returns: the list of the code studio objects, each one as a python dict :rtype: list """ items = self.client._perform_json( "GET", "/projects/%s/code-studios/" % self.project_key) if as_type == "listitems" or as_type == "listitem": return [DSSCodeStudioObjectListItem(self.client, self.project_key, item) for item in items] elif as_type == "objects" or as_type == "object": return [DSSCodeStudioObject(self.client, self.project_key, item["id"]) for item in items] else: raise ValueError("Unknown as_type") def get_code_studio(self, code_studio_id): """ Get a handle to interact with a specific code studio object :param str code_studio_id: the identifier of the desired code studio object :returns: A code studio object handle :rtype: :class:`dataikuapi.dss.codestudio.DSSCodeStudioObject` """ return DSSCodeStudioObject(self.client, self.project_key, code_studio_id) def create_code_studio(self, name, template_id): """ Create a new code studio object in the project, and return a handle to interact with it :param str name: the name of the code studio object :param str template_id: the identifier of a code studio template :returns: A code studio object handle :rtype: :class:`dataikuapi.dss.codestudio.DSSCodeStudioObject` """ obj = { "name": name, "templateId": template_id } res = self.client._perform_json("POST", "/projects/%s/code-studios/" % self.project_key, body=obj) code_studio_id = res['codeStudio']['id'] return DSSCodeStudioObject(self.client, self.project_key, code_studio_id) ######################################################## # Project libraries ######################################################## def get_library(self): """ Get a handle to manage the project library :returns: A :class:`dataikuapi.dss.projectlibrary.DSSLibrary` handle :rtype: :class:`dataikuapi.dss.projectlibrary.DSSLibrary` """ return DSSLibrary(self.client, self.project_key) ######################################################## # LLM ######################################################## def list_llms(self, purpose="GENERIC_COMPLETION", as_type="listitems"): """ List the LLM usable in this project :param str purpose: Usage purpose of the LLM. Main values are GENERIC_COMPLETION and TEXT_EMBEDDING_EXTRACTION :param str as_type: How to return the list. Supported values are "listitems" and "objects". :returns: The list of LLMs. If "as_type" is "listitems", each one as a :class:`dataikuapi.dss.llm.DSSLLMListItem`. If "as_type" is "objects", each one as a :class:`dataikuapi.dss.llm.DSSLLM` :rtype: list """ llms = self.client._perform_json("GET", "/projects/%s/llms?purpose=%s" % (self.project_key, purpose)) if as_type == "listitems": return [DSSLLMListItem(self.client, self.project_key, item) for item in llms] elif as_type == "objects": return [DSSLLM(self.client, self.project_key, item["id"]) for item in llms] else: raise ValueError("Unknown as_type") def get_llm(self, llm_id): """ Get a handle to interact with a specific LLM :param id: the identifier of an LLM :returns: A :class:`dataikuapi.dss.llm.DSSLLM` LLM handle """ return DSSLLM(self.client, self.project_key, llm_id) def list_knowledge_banks(self, as_type="listitems"): """ List the knowledge banks of this project :param str as_type: How to return the list. Supported values are "listitems" and "objects". :returns: The list of knowledge banks. If "as_type" is "listitems", each one as a :class:`dataikuapi.dss.knowledgebank.DSSKnowledgeBankListItem`. If "as_type" is "objects", each one as a :class:`dataikuapi.dss.knowledgebank.DSSKnowledgeBank` :rtype: list """ kbs = self.client._perform_json("GET", "/projects/%s/knowledge-banks" % (self.project_key)) if as_type == "listitems": return [DSSKnowledgeBankListItem(self.client, item) for item in kbs] elif as_type == "objects": return [DSSKnowledgeBank(self.client, self.project_key, item["id"]) for item in kbs] else: raise ValueError("Unknown as_type") def get_knowledge_bank(self, id): """ Get a handle to interact with a specific knowledge bank :param id: the identifier of a knowledge bank :returns: A :class:`dataikuapi.dss.knowledgebank.DSSKnowledgeBank` knowledge bank handle """ return DSSKnowledgeBank(self.client, self.project_key, id) ######################################################## # Webapps ######################################################## def list_webapps(self, as_type="listitems"): """ List the webapp heads of this project :param str as_type: How to return the list. Supported values are "listitems" and "objects". :returns: The list of the webapps. If "as_type" is "listitems", each one as a :class:`scenario.DSSWebAppListItem`. If "as_type" is "objects", each one as a :class:`scenario.DSSWebApp` :rtype: list """ webapps = self.client._perform_json("GET", "/projects/%s/webapps/" % self.project_key) if as_type == "listitems": return [DSSWebAppListItem(self.client, item) for item in webapps] elif as_type == "objects": return [DSSWebApp(self.client, self.project_key, item["id"]) for item in webapps] else: raise ValueError("Unknown as_type") def get_webapp(self, webapp_id): """ Get a handle to interact with a specific webapp :param webapp_id: the identifier of a webapp :returns: A :class:`dataikuapi.dss.webapp.DSSWebApp` webapp handle """ return DSSWebApp(self.client, self.project_key, webapp_id) ######################################################## # Dashboards ######################################################## def list_dashboards(self, as_type="listitems"): """ List the Dashboards in this project. :returns: The list of the dashboards. :rtype: list """ items = self.client._perform_json("GET", DASHBOARDS_URI_FORMAT % self.project_key) if as_type == "listitems": return [DSSDashboardListItem(self.client, item) for item in items] elif as_type == "objects": return [DSSDashboard(self.client, self.project_key, item["id"]) for item in items] else: raise ValueError("Unknown as_type") def get_dashboard(self, dashboard_id): """ Get a handle to interact with a specific dashboard object :param str dashboard_id: the identifier of the desired dashboard object :returns: A :class:`dataikuapi.dss.dashboard.DSSDashboard` dashboard object handle """ return DSSDashboard(self.client, self.project_key, dashboard_id) def create_dashboard(self, dashboard_name, settings=None): """ Create a new dashboard in the project, and return a handle to interact with it :param str dashboard_name: The name for the new dashboard. This does not need to be unique (although this is strongly recommended) :param dict settings: the JSON definition of the dashboard. Use ``get_settings()`` on an existing ``DSSDashboard`` object in order to get a sample settings object (defaults to `{'pages': []}`) :returns: a :class:`.dashboard.DSSDashboard` handle to interact with the newly-created dashboard """ if settings is None: settings = {'pages': []} settings['name'] = dashboard_name dashboard_id = self.client._perform_json("POST", DASHBOARDS_URI_FORMAT % self.project_key, body = settings)['id'] return DSSDashboard(self.client, self.project_key, dashboard_id) ######################################################## # Insights ######################################################## def list_insights(self, as_type="listitems"): """ List the Insights in this project. :returns: The list of the insights. :rtype: list """ items = self.client._perform_json("GET", INSIGHTS_URI_FORMAT % self.project_key) if as_type == "listitems": return [DSSInsightListItem(self.client, item) for item in items] elif as_type == "objects": return [DSSInsight(self.client, self.project_key, item["id"]) for item in items] else: raise ValueError("Unknown as_type") def get_insight(self, insight_id): """ Get a handle to interact with a specific insight object :param str insight_id: the identifier of the desired insight object :returns: A :class:`dataikuapi.dss.insight.DSSInsight` insight object handle """ return DSSInsight(self.client, self.project_key, insight_id) def create_insight(self, creation_info): """ Create a new insight in the project, and return a handle to interact with it :param dict creation_info: the JSON definition of the insight creation. Use ``get_settings()`` on an existing ``DSSInsight`` object in order to get a sample settings object :returns: a :class:`.insight.DSSInsight` handle to interact with the newly-created insight """ insight_id = self.client._perform_json("POST", INSIGHTS_URI_FORMAT % self.project_key, body = {"insightPrototype": creation_info})['id'] return DSSInsight(self.client, self.project_key, insight_id) ######################################################## # Git ######################################################## def get_project_git(self): """ Gets an handle to perform operations on the project's git repository. :returns: a handle to perform git operations on project. :rtype: :class:`dataikuapi.dss.project.DSSProjectGit` """ return DSSProjectGit(self.client, self.project_key) ######################################################## # Data Quality ######################################################## def get_data_quality_status(self, only_monitored=True): """ Get the aggregated quality status of a project with the list of the datasets and their associated status :param only_monitored: boolean to retrieve only monitored dataset, default to True. :returns: The dict of data quality dataset statuses. :rtype: dict with DATASET_NAME as key """ return self.client._perform_json("GET", "/projects/%s/data-quality/status" % self.project_key, params={"onlyMonitored": only_monitored}) def get_data_quality_timeline(self, min_timestamp=None, max_timestamp=None): """ Get the list of quality status aggregated per day during the timeframe [min_timestamp, max_timestamp]. It includes the current & worst outcome for each days and the details of the datasets runs within the period, also includes previous deleted monitored datasets with the mention "(deleted)" at the end of their id. Default parameters include the timeframe for the last 14 days. :param int min_timestamp: timestamp representing the beginning of the timeframe :param int max_timestamp: timestamp representing the end of the timeframe :returns: list of datasets per day in the timeline :rtype: list of dict """ return self.client._perform_json("GET", "/projects/%s/data-quality/timeline" % self.project_key, params={"minTimestamp": min_timestamp, "maxTimestamp": max_timestamp}) class TablesImportDefinition(object): """ Temporary structure holding the list of tables to import """ def __init__(self, client, project_key): """Do not call this directly, use :meth:`DSSProject.init_tables_import`""" self.client = client self.project_key = project_key self.keys = [] def add_hive_table(self, hive_database, hive_table): """Add a Hive table to the list of tables to import :param str hive_database: the name of the Hive database :param str hive_table: the name of the Hive table """ self.keys.append({ "connectionName": "@virtual(hive-jdbc):" + hive_database, "name": hive_table }) def add_sql_table(self, connection, schema, table, catalog=None): """Add a SQL table to the list of tables to import :param str connection: the name of the SQL connection :param str schema: the schema of the table :param str table: the name of the SQL table :param str catalog: the database of the SQL table. Leave to None to use the default database associated with the connection """ self.keys.append({ "connectionName": connection, "schema": schema, "name": table, "catalog": catalog }) def add_elasticsearch_index_or_alias(self, connection, index_or_alias): """Add an Elastic Search index or alias to the list of tables to import""" self.keys.append({"connectionName": connection, "name": index_or_alias}) def prepare(self): """ Run the first step of the import process. In this step, DSS will check the tables whose import you have requested and prepare dataset names and target connections :returns: an object that allows you to finalize the import process :rtype: :class:`TablesPreparedImport` """ ret = self.client._perform_json("POST", "/projects/%s/datasets/tables-import/actions/prepare-from-keys" % ( self.project_key), body={"keys": self.keys}) future = self.client.get_future(ret["jobId"]) future.wait_for_result() return TablesPreparedImport(self.client, self.project_key, future.get_result()) class TablesPreparedImport(object): """Result of preparing a tables import. Import can now be finished""" def __init__(self, client, project_key, candidates): """Do not call this directly, use :meth:`DSSProject.init_tables_import` and then prepare""" self.client = client self.project_key = project_key self.candidates = candidates def execute(self): """ Starts executing the import in background and returns a :class:`dataikuapi.dss.future.DSSFuture` to wait on the result :returns: a future to wait on the result :rtype: :class:`dataikuapi.dss.future.DSSFuture` """ ret = self.client._perform_json("POST", "/projects/%s/datasets/tables-import/actions/execute-from-candidates" % (self.project_key), body = self.candidates) return DSSFuture.from_resp(self.client, ret) class DSSProjectSettings(object): """Settings of a DSS project""" def __init__(self, client, project_key, settings): """Do not call directly, use :meth:`DSSProject.get_settings`""" self.client = client self.project_key = project_key self.settings = settings def get_raw(self): """Gets all settings as a raw dictionary. This returns a reference to the raw settings, not a copy, so changes made to the returned object will be reflected when saving. :rtype: dict """ return self.settings def set_python_code_env(self, code_env_name): """Sets the default Python code env used by this project :param str code_env_name: Identifier of the code env to use. If None, sets the project to use the builtin Python env """ if code_env_name is None: self.settings["settings"]["codeEnvs"]["python"]["useBuiltinEnv"] = True self.settings["settings"]["codeEnvs"]["python"]["mode"] = "USE_BUILTIN_MODE" else: self.settings["settings"]["codeEnvs"]["python"]["useBuiltinEnv"] = False self.settings["settings"]["codeEnvs"]["python"]["envName"] = code_env_name self.settings["settings"]["codeEnvs"]["python"]["mode"] = "EXPLICIT_ENV" def set_r_code_env(self, code_env_name): """Sets the default R code env used by this project :param str code_env_name: Identifier of the code env to use. If None, sets the project to use the builtin R env """ if code_env_name is None: self.settings["settings"]["codeEnvs"]["r"]["useBuiltinEnv"] = True self.settings["settings"]["codeEnvs"]["r"]["mode"] = "USE_BUILTIN_MODE" else: self.settings["settings"]["codeEnvs"]["r"]["useBuiltinEnv"] = False self.settings["settings"]["codeEnvs"]["r"]["envName"] = code_env_name self.settings["settings"]["codeEnvs"]["r"]["mode"] = "EXPLICIT_ENV" def set_container_exec_config(self, config_name): """Sets the default containerized execution config used by this project :param str config_name: Identifier of the containerized execution config to use. If None, sets the project to use local execution """ if config_name is None: self.settings["settings"]["container"]["containerMode"] = "NONE" else: self.settings["settings"]["container"]["containerMode"] = "EXPLICIT_CONTAINER" self.settings["settings"]["container"]["containerConf"] = config_name def set_k8s_cluster(self, cluster, fallback_cluster=None): """Sets the Kubernetes cluster used by this project :param str cluster: Identifier of the cluster to use. May use variables expansion. If None, sets the project to use the globally-defined cluster :param str fallback_cluster: Identifier of the cluster to use if the variable used for "cluster" does not exist (defaults to **None**) """ if cluster is None: self.settings["settings"]["k8sCluster"]["clusterMode"] = "INHERIT" else: self.settings["settings"]["k8sCluster"]["clusterMode"] = "EXPLICIT_CLUSTER" self.settings["settings"]["k8sCluster"]["clusterId"] = cluster self.settings["settings"]["k8sCluster"]["defaultClusterId"] = fallback_cluster def set_cluster(self, cluster, fallback_cluster=None): """Sets the Hadoop/Spark cluster used by this project :param str cluster: Identifier of the cluster to use. May use variables expansion. If None, sets the project to use the globally-defined cluster :param str fallback_cluster: Identifier of the cluster to use if the variable used for "cluster" does not exist (defaults to **None**) """ if cluster is None: self.settings["settings"]["cluster"]["clusterMode"] = "INHERIT" else: self.settings["settings"]["cluster"]["clusterMode"] = "EXPLICIT_CLUSTER" self.settings["settings"]["cluster"]["clusterId"] = cluster self.settings["settings"]["cluster"]["defaultClusterId"] = fallback_cluster def add_exposed_object(self, object_type, object_id, target_project): """ Exposes an object from this project to another project. Does nothing if the object was already exposed to the target project :param str object_type: type of the object to expose :param str object_id: id of the object to expose :param str target_project: id of the project in which to expose the object """ found_eo = None for eo in self.settings["exposedObjects"]["objects"]: if eo["type"] == object_type and eo["localName"] == object_id: found_eo = eo break if found_eo is None: found_eo = {"type": object_type, "localName": object_id, "rules": []} self.settings["exposedObjects"]["objects"].append(found_eo) already_exists = False for rule in found_eo["rules"]: if rule["targetProject"] == target_project: already_exists = True break if not already_exists: found_eo["rules"].append({"targetProject": target_project}) def save(self): """Saves back the settings to the project""" self.client._perform_empty("PUT", "/projects/%s/settings" % (self.project_key), body=self.settings) class DSSProjectGit(object): """Handle to manage the git repository of a DSS project (fetch, push, pull, ...)""" def __init__(self, client, project_key): """Do not call directly, use :meth:`DSSProject.get_project_git`""" self.client = client self.project_key = project_key def get_status(self): """ Get the current state of the project's git repository :return: A dict containing the following keys: 'currentBranch', 'remotes', 'trackingCount', 'clean', 'hasUncommittedChanges', 'added', 'changed', 'removed', 'missing', 'modified', 'conflicting', 'untracked' and 'untrackedFolders' :rtype: dict """ return self.client._perform_json("GET", "/projects/%s/git/status" % self.project_key) def get_remote(self, name="origin"): """ Get the URL of the remote repository. :param: str name: The name of the remote. Defaults to "origin". :return: The URL of the remote origin if set, None otherwise. :rtype: str or None """ resp = self.client._perform_json("GET", "/projects/%s/git/remotes/%s" % (self.project_key, name)) return resp["url"] if "url" in resp else None def set_remote(self, url, name="origin"): """ Set the URL of the remote repository. :param: str name: The name of the remote to set. Defaults to "origin". :param str url: The URL of the remote repository (git@github.com:user/repo.git). """ self.client._perform_json("POST", "/projects/%s/git/remotes/%s" % (self.project_key, name), body={"url": url}) def remove_remote(self, name="origin"): """ Remove the remote origin of the project's git repository. :param: str name: The name of the remote to remove. Defaults to "origin". """ self.client._perform_json("DELETE", "/projects/%s/git/remotes/%s" % (self.project_key, name)) def list_branches(self, remote=False): """ List all branches (local only or local & remote) of the project's git repository. :param bool remote: Whether to include remote branches. :return: A list of branch names. :rtype: list """ return self.client._perform_json("GET", "/projects/%s/git/branches" % self.project_key, params={"remote": remote}) def create_branch(self, branch_name, commit=None): """ Create a new local branch on the project's git repository and switches to it. :param str branch_name: The name of the branch to create. :param str commit: Hash of a commit to create the branch from (optional). :return: A dict containing keys 'success' and 'output' with information about the command execution. :rtype: dict """ return self.client._perform_json("POST", "/projects/%s/git/branches/" % self.project_key, body={"name": branch_name, "commit": commit}) def delete_branch(self, branch_name, force_delete=False, remote=False, delete_remotely=False): """ Delete a local or remote branch on the project's git repository. :param str branch_name: The name of the branch to delete. :param bool remote: True if the branch to delete is a remote branch; False if it's a local branch. :param bool delete_remotely: True to delete a remote branch both locally and remotely; False to delete the remote branch on the local repository only. :param bool force_delete: True to force the deletion even if some commits have not been pushed; False to fail in case some commits have not been pushed. """ self.client._perform_json("POST", "/projects/%s/git/actions/deleteBranch" % self.project_key, body={"name": branch_name, "remote": remote, "deleteRemotely": delete_remotely, "forceDelete": force_delete}) def get_current_branch(self): """ Get the name of the current branch :return: The name of the current branch :rtype: str """ resp = self.client._perform_json("GET", "/projects/%s/git/current-branch" % self.project_key) return resp["name"] if "name" in resp else None def list_tags(self): """ Lists all existing tags. :return: A list of dict objects, each one containing the following keys: 'name', 'shortName', 'commit' (hash of the commit associated with the tag), 'annotations' and 'readOnly'. :rtype: list """ return self.client._perform_json("GET", "/projects/%s/git/tags" % self.project_key) def create_tag(self, name, reference="HEAD", message=""): """ Create a tag for the specified or current reference. :param str name: The name of the tag to create. :param str reference: ID of a commit to tag. Defaults to HEAD. """ self.client._perform_json("POST", "/projects/%s/git/tags/" % self.project_key, body={"name": name, "reference": reference, "message": message}) def delete_tag(self, name): """ Remove a tag from the local repository :param str name: The name of the tag to delete. """ self.client._perform_json("POST", "/projects/%s/git/actions/deleteTag" % self.project_key, body={"name": name}) def switch(self, branch_name): """ Switch the current repository to the specified branch. :param str branch_name: The name of the branch to switch to. :return: A dict containing keys 'success', 'messages', 'output' with information about the command execution. :rtype: dict """ return self.client._perform_json("POST", "/projects/%s/git/actions/switchBranch" % self.project_key, params={"branchName": branch_name}) def checkout(self, branch_name): """ Switch the current repository to the specified branch (identical to`switch`) :param str branch_name: The name of the branch to checkout. :return: A dict containing keys 'success', 'messages', 'output' with information about the command execution. :rtype: dict """ return self.switch(branch_name) def fetch(self): """ Fetch branches and/or tags (collectively, "refs") from the remote repository to the project's git repository. :return: A dict containing keys 'success', 'logs', 'output' with information about the command execution. :rtype: dict """ return self.client._perform_json("POST", "/projects/%s/git/actions/fetch" % self.project_key) def pull(self, branch_name=None): """ Incorporate changes from a remote repository into the current branch on the project's git repository. :param str branch_name: The name of the branch to pull. If None, pull from the current branch. :return: A dict containing keys 'success', 'logs', 'output' with information about the command execution. :rtype: dict """ return self.client._perform_json("POST", "/projects/%s/git/actions/pullRebase" % self.project_key, params={"branchName": branch_name}) def push(self, branch_name=None): """ Update the remote repository with the project's local commits. :param str branch_name: The name of the branch to push. If None, push commits from the current branch. :return: A dict containing keys 'success', 'logs', 'output' with information about the command execution. :rtype: dict """ return self.client._perform_json("POST", "/projects/%s/git/actions/push" % self.project_key, params={"branchName": branch_name}) def log(self, path=None, start_commit=None, count=1000): """ List commits in the project's git repository. :param str path: Path to filter the logs (optional). If specified, only commits impacting files located in the provided path are returned. :param str start_commit: ID of the first commit to list. Use the value found in the `nextCommit` field from a previous response (optional). :param int count: Maximum number of commits to return (20 by default). :return: A dict containing a key `entries` and optionally a second key `nextCommit` if there are more commits. :rtype: dict """ return self.client._perform_json("GET", "/projects/%s/git/actions/log" % self.project_key, params={"path": path, "startCommit": start_commit, "count": count}) def diff(self, commit_from=None, commit_to=None): """ Show changes between the working copy and the last commit (commit_from=None, commit_to=None), between two commits (commit_from=SOME_ID, commit_to=SOME_ID), or made in a given commit (commit_from=SOME_ID, commit_to=None). :param str commit_from: ID of the first commit or None :param str commit_to: ID of the second commit or None :return: A containing containing the following keys: 'commitFrom' (dict), 'commitTo' (dict), 'addedLines' (int), 'removedLines' (int), 'changedFiles' (int), 'entries' (array of dict) :rtype: dict """ return self.client._perform_json("GET", "/projects/%s/git/actions/diff" % self.project_key, params={"commitFrom": commit_from, "commitTo": commit_to}) def commit(self, message): """ Commit pending changes in the project's git repository with the given message. Note: Untracked tracked are automatically added before committing. :param str message: The commit message. """ self.client._perform_json("POST", "/projects/%s/git/actions/commit" % self.project_key, body={"message": message}) def revert_to_revision(self, commit): """ Revert the project content to the supplied revision. :param str commit: Hash of a valid commit as returned by the log method. :return: A dict containing a key 'success' and optionally a second key 'logs' with information about the command execution. :rtype: dict """ return self.client._perform_json("POST", "/projects/%s/git/actions/revertToRevision" % self.project_key, params={"commit": commit}) def revert_commit(self, commit): """ Revert the changes that the specified commit introduces :param str commit: ID of a valid commit as returned by the log method. :return: A dict containing the following keys: 'success' and optionally 'logs' with information about the command execution. :rtype: dict """ return self.client._perform_json("POST", "/projects/%s/git/actions/revertCommit" % self.project_key, params={"commit": commit}) def reset_to_head(self): """ Drop uncommitted changes in the project's git repository (hard reset to HEAD). """ self.client._perform_json("POST", "/projects/%s/git/actions/resetToLocalHeadState" % self.project_key) def reset_to_upstream(self): """ Drop local changes in the project's git repository and hard reset to the upstream branch. """ self.client._perform_json("POST", "/projects/%s/git/actions/resetToRemoteHeadState" % self.project_key) def drop_and_rebuild(self, i_know_what_i_am_doing=False): """ Fully drop the current git repositoty and rebuild from scratch a new one. CAUTION: ALL HISTORY WILL BE LOST. ONLY CALL THIS METHOD IF YOU KNOW WHAT YOU ARE DOING. :param bool i_know_what_i_m_doing: True if you really want to wipe out all git history for this project. """ if i_know_what_i_am_doing: self.client._perform_json("POST", "/projects/%s/git/actions/dropAndRebuild" % self.project_key, params={"iKnowWhatIAmDoing": True}) else: raise Exception("Set argument 'i_know_what_i_am_doing' to True to confirm you really want to wipe out all git history.") def list_libraries(self): """ Get the list of all external libraries for this project :return: A list of external libraries. :rtype: list """ return self.client._perform_json("GET", "/projects/%s/git/lib-git-refs/" % self.project_key) def add_library(self, repository, local_target_path, checkout, path_in_git_repository="", add_to_python_path=True): """ Add a new external library to the project and pull it. :param str repository: The remote repository. :param str local_target_path: The local target path (relative to root of libraries). :param str checkout: The branch, commit, or tag to check out. :param str path_in_git_repository: The path in the git repository. :param bool add_to_python_path: Whether to add the reference to the Python path. :return: a :class:`dataikuapi.dss.future.DSSFuture` representing the pull process :rtype: :class:`dataikuapi.dss.future.DSSFuture` """ body = { "repository": repository, "pathInGitRepository": path_in_git_repository, "localTargetPath": local_target_path, "checkout": checkout, "addToPythonPath": add_to_python_path } return self.client._perform_json("POST", "/projects/%s/git/lib-git-refs/" % self.project_key, body=body) def set_library(self, git_reference_path, remote, remotePath, checkout): """ Set an existing external library. :param str git_reference_path: The path of the external library. :param str remote: The remote repository. :param str remotePath: The path in the git repository. :param str checkout: The branch, commit, or tag to check out. :return: The path of the external library. :rtype: str """ body = { "repository": remote, "pathInGitRepository": remotePath, "checkout": checkout, } return self.client._perform_json("PUT", "/projects/" + self.project_key + "/git/lib-git-refs/" + git_reference_path, body=body) def remove_library(self, git_reference_path, delete_directory): """ Remove an external library from the project. :param str git_reference_path: The path of the external library. :param bool delete_directory: Whether to delete the local directory associated with the reference. """ params = {"deleteDirectory": delete_directory} self.client._perform_json("DELETE", "/projects/" + self.project_key + "/git/lib-git-refs/" + git_reference_path, params=params) def reset_library(self, git_reference_path): """ Reset changes to HEAD from the external library. :param str git_reference_path: The path of the external library to reset. :return: a :class:`dataikuapi.dss.future.DSSFuture` representing the reset process :rtype: :class:`dataikuapi.dss.future.DSSFuture` """ body = {"gitRef": git_reference_path} future_resp = self.client._perform_json("POST", "/projects/" + self.project_key + "/git/lib-git-refs/action/reset", body=body) return DSSFuture.from_resp(self.client, future_resp) def push_library(self, git_reference_path, commit_message): """ Push changes to the external library :param str git_reference_path: The path of the external library. :param str commit_message: The commit message for the push. :return: a :class:`dataikuapi.dss.future.DSSFuture` representing the push process :rtype: :class:`dataikuapi.dss.future.DSSFuture` """ body = {"gitRef": git_reference_path, "commitMessage": commit_message} future_resp = self.client._perform_json("POST", "/projects/" + self.project_key + "/git/lib-git-refs/action/push", body=body) return DSSFuture.from_resp(self.client, future_resp) def push_all_libraries(self, commit_message): """ Push changes for all libraries in the project. :param str commit_message: The commit message for the push. :return: a :class:`dataikuapi.dss.future.DSSFuture` representing the push process :rtype: :class:`dataikuapi.dss.future.DSSFuture` """ body = {"commitMessage": commit_message} future_resp = self.client._perform_json("POST", "/projects/" + self.project_key + "/git/actions/git-refs/push-all", body=body) return DSSFuture.from_resp(self.client, future_resp) def reset_all_libraries(self): """ Reset changes for all libraries in the project. :return: a :class:`dataikuapi.dss.future.DSSFuture` representing the reset process :rtype: :class:`dataikuapi.dss.future.DSSFuture` """ future_resp = self.client._perform_json("POST", "/projects/" + self.project_key + "/git/actions/git-refs/reset-all") return DSSFuture.from_resp(self.client, future_resp) class JobDefinitionBuilder(object): """ Helper to run a job. Do not create this class directly, use :meth:`DSSProject.new_job` """ def __init__(self, project, job_type="NON_RECURSIVE_FORCED_BUILD"): self.project = project self.definition = {'type': job_type, 'refreshHiveMetastore': False, 'outputs': []} def with_type(self, job_type): """ Sets the build type :param job_type: the build type for the job RECURSIVE_BUILD, NON_RECURSIVE_FORCED_BUILD, RECURSIVE_FORCED_BUILD, RECURSIVE_MISSING_ONLY_BUILD """ self.definition['type'] = job_type return self def with_refresh_metastore(self, refresh_metastore): """ Sets whether the hive tables built by the job should have their definitions refreshed after the corresponding dataset is built :param bool refresh_metastore: """ self.definition['refreshHiveMetastore'] = refresh_metastore return self def with_output(self, name, object_type=None, object_project_key=None, partition=None): """ Adds an item to build in this job :param name: name of the output object :param object_type: type of object to build from: DATASET, MANAGED_FOLDER, SAVED_MODEL, STREAMING_ENDPOINT (defaults to **None**) :param object_project_key: PROJECT_KEY for the project that contains the object to build (defaults to **None**) :param partition: specify partition to build (defaults to **None**) """ self.definition['outputs'].append( {'type': object_type, 'id': name, 'projectKey': object_project_key, 'partition': partition}) return self def get_definition(self): """Gets the internal definition for this job""" return self.definition def start(self): """ Starts the job, and return a :class:`dataikuapi.dss.job.DSSJob` handle to interact with it. You need to wait for the returned job to complete :returns: a job handle :rtype: :class:`dataikuapi.dss.job.DSSJob` """ job_def = self.project.client._perform_json("POST", "/projects/%s/jobs/" % self.project.project_key, body=self.definition) return DSSJob(self.project.client, self.project.project_key, job_def['id']) def start_and_wait(self, no_fail=False): """ Starts the job, waits for it to complete and returns a :class:`dataikuapi.dss.job.DSSJob` handle to interact with it Raises if the job failed. :param no_fail: if True, does not raise if the job failed (defaults to **False**). :returns: A job handle :rtype: :class:`dataikuapi.dss.job.DSSJob` """ job = self.start() waiter = DSSJobWaiter(job) waiter.wait(no_fail) return job dataiku-api-client-python-13.2.2/dataikuapi/dss/projectdeployer.py000066400000000000000000000642171471211575100252720ustar00rootroot00000000000000from .future import DSSFuture class DSSProjectDeployer(object): """ Handle to interact with the Project Deployer. .. important:: Do not instantiate directly, use :meth:`dataikuapi.DSSClient.get_projectdeployer()` """ def __init__(self, client): self.client = client def list_deployments(self, as_objects=True): """ List deployments on the Project Deployer. Usage example: .. code-block:: python # list all deployments with their current state for deployment in deployer.list_deployments(): status = deployment.get_status() print("Deployment %s is %s" % (deployment.id, status.get_health())) :param boolean as_objects: if True, returns a list of :class:`DSSProjectDeployerDeployment`, else returns a list of dict. :returns: list of deployments, either as :class:`DSSProjectDeployerDeployment` or as dict (with fields as in :meth:`DSSProjectDeployerDeploymentStatus.get_light()`) :rtype: list """ l = self.client._perform_json("GET", "/project-deployer/deployments") if as_objects: return [DSSProjectDeployerDeployment(self.client, x["deploymentBasicInfo"]["id"]) for x in l] else: return l def get_deployment(self, deployment_id): """ Get a handle to interact with a deployment. :param string deployment_id: identifier of a deployment :rtype: :class:`DSSProjectDeployerDeployment` """ return DSSProjectDeployerDeployment(self.client, deployment_id) def create_deployment(self, deployment_id, project_key, infra_id, bundle_id, deployed_project_key=None, project_folder_id=None, ignore_warnings=False): """ Create a deployment and return the handle to interact with it. The returned deployment is not yet started and you need to call :meth:`~DSSProjectDeployerDeployment.start_update` Usage example: .. code-block:: python # create and deploy a bundle project = 'my-project' infra = 'my-infra' bundle = 'my-bundle' deployment_id = '%s-%s-on-%s' % (project, bundle, infra) deployment = deployer.create_deployment(deployment_id, project, infra, bundle) update = deployment.start_update() update.wait_for_result() :param string deployment_id: identifier of the deployment to create :param string project_key: key of the published project :param string bundle_id: identifier of the bundle to deploy :param string infra_id: identifier of the infrastructure to use :param string deployed_project_key: The project key to use when deploying this project to the automation node. If not set, the project will be created with the same project key as the published project :param string project_folder_id: The automation node project folder id to deploy this project into. If not set, the project will be created in the root folder :param boolean ignore_warnings: ignore warnings concerning the governance status of the bundle to deploy :return: a new deployment :rtype: :class:`DSSProjectDeployerDeployment` """ settings = { "deploymentId": deployment_id, "publishedProjectKey": project_key, "infraId": infra_id, "bundleId": bundle_id } if deployed_project_key: settings["deployedProjectKey"] = deployed_project_key if project_folder_id: settings["projectFolderId"] = project_folder_id self.client._perform_json("POST", "/project-deployer/deployments", params={"ignoreWarnings": ignore_warnings}, body=settings) return self.get_deployment(deployment_id) def list_stages(self): """ List the possible stages for infrastructures. :return: list of stages. Each stage is returned as a dict with fields: * **id** : identifier of the stage * **desc** : description of the stage :rtype: list[dict] """ return self.client._perform_json("GET", "/project-deployer/stages") def list_infras(self, as_objects=True): """ List the infrastructures on the Project Deployer. Usage example: .. code-block:: python # list infrastructures that the user can deploy to for infrastructure in deployer.list_infras(as_objects=False): if infrastructure.get("canDeploy", False): print("User can deploy to %s" % infrastructure["infraBasicInfo"]["id"]) :param boolean as_objects: if True, returns a list of :class:`DSSProjectDeployerInfra`, else returns a list of dict. :return: list of infrastructures, either as :class:`DSSProjectDeployerInfra` or as dict (with fields as in :meth:`DSSProjectDeployerInfraStatus.get_raw()`) :rtype: list """ l = self.client._perform_json("GET", "/project-deployer/infras") if as_objects: return [DSSProjectDeployerInfra(self.client, x["infraBasicInfo"]["id"]) for x in l] else: return l def create_infra(self, infra_id, stage, govern_check_policy="NO_CHECK"): """ Create a new infrastructure and returns the handle to interact with it. :param string infra_id: unique identifier of the infrastructure to create :param string stage: stage of the infrastructure to create :param string govern_check_policy: what actions with Govern the the deployer will take whe bundles are deployed on this infrastructure. Possible values: PREVENT, WARN, or NO_CHECK :return: a new infrastructure :rtype: :class:`DSSProjectDeployerInfra` """ settings = { "id": infra_id, "stage": stage, "governCheckPolicy": govern_check_policy, } self.client._perform_json("POST", "/project-deployer/infras", body=settings) return self.get_infra(infra_id) def get_infra(self, infra_id): """ Get a handle to interact with an infrastructure. :param string infra_id: identifier of the infrastructure to get :rtype: :class:`DSSProjectDeployerInfra` """ return DSSProjectDeployerInfra(self.client, infra_id) def list_projects(self, as_objects=True): """ List published projects on the Project Deployer. Usage example: .. code-block:: python # list project that the user can deploy bundles from for project in deployer.list_projects(as_objects=False): if project.get("canDeploy", False): print("User can deploy to %s" % project["projectBasicInfo"]["id"]) :param boolean as_objects: if True, returns a list of :class:`DSSProjectDeployerProject`, else returns a list of dict. :return: list of published projects, either as :class:`DSSProjectDeployerProject` or as dict (with fields as in :meth:`DSSProjectDeployerProjectStatus.get_raw()`) :rtype: list """ l = self.client._perform_json("GET", "/project-deployer/projects") if as_objects: return [DSSProjectDeployerProject(self.client, x["projectBasicInfo"]["id"]) for x in l] else: return l def create_project(self, project_key): """ Create a new published project on the Project Deployer and return the handle to interact with it. :param string project_key: key of the project to create :rtype: :class:`DSSProjectDeployerProject` """ settings = { "publishedProjectKey": project_key } self.client._perform_json("POST", "/project-deployer/projects", body=settings) return self.get_project(project_key) def get_project(self, project_key): """ Get a handle to interact with a published project. :param string project_key: key of the project to get :rtype: :class:`DSSProjectDeployerProject` """ return DSSProjectDeployerProject(self.client, project_key) def upload_bundle(self, fp, project_key=None): """ Upload a bundle archive for a project. :param file-like fp: a bundle archive (should be a zip) :param string project_key: key of the published project where the bundle will be uploaded. If the project does not exist, it is created. If not set, the key of the bundle's source project is used. """ if project_key is None: params = None else: params = { "projectKey": project_key, } self.client._perform_empty("POST", "/project-deployer/projects/bundles", params=params, files={"file":fp}) ############################################### # Infrastructures ############################################### class DSSProjectDeployerInfra(object): """ An Automation infrastructure on the Project Deployer. .. important:: Do not instantiate directly, use :meth:`DSSProjectDeployer.get_infra`. """ def __init__(self, client, infra_id): self.client = client self.infra_id = infra_id @property def id(self): """ Get the unique identifier of the infrastructure. :rtype: string """ return self.infra_id def get_status(self): """ Get status information about this infrastructure. :return: the current status :rtype: :class:`DSSProjectDeployerInfraStatus` """ light = self.client._perform_json("GET", "/project-deployer/infras/%s" % (self.infra_id)) return DSSProjectDeployerInfraStatus(self.client, self.infra_id, light) def get_settings(self): """ Get the settings of this infrastructure. :rtype: :class:`DSSProjectDeployerInfraSettings` """ settings = self.client._perform_json( "GET", "/project-deployer/infras/%s/settings" % (self.infra_id)) return DSSProjectDeployerInfraSettings(self.client, self.infra_id, settings) def delete(self): """ Delete this infra. .. note:: You may only delete an infra if there are no deployments using it. """ self.client._perform_empty( "DELETE", "/project-deployer/infras/%s" % (self.infra_id)) class DSSProjectDeployerInfraSettings(object): """ The settings of an Automation infrastructure. .. important:: Do not instantiate directly, use :meth:`DSSProjectDeployerInfra.get_settings` To modify the settings, modify them in the dict returned by :meth:`get_raw()` then call :meth:`save()`. """ def __init__(self, client, infra_id, settings): self.client = client self.infra_id = infra_id self.settings = settings def get_raw(self): """ Get the raw settings of this infrastructure. This returns a reference to the raw settings, not a copy, so changes made to the returned object will be reflected when saving. :return: the settings, as a dict. :rtype: dict """ return self.settings def save(self): """ Save back these settings to the infrastracture. """ self.client._perform_empty( "PUT", "/project-deployer/infras/%s/settings" % (self.infra_id), body = self.settings) class DSSProjectDeployerInfraStatus(object): """ The status of an Automation infrastructure. .. important:: Do not instantiage directly, use :meth:`DSSProjectDeployerInfra.get_status` """ def __init__(self, client, infra_id, light_status): self.client = client self.infra_id = infra_id self.light_status = light_status def get_deployments(self): """ Get the deployments that are deployed on this infrastructure. :return: a list of deployments :rtype: list of :class:`DSSProjectDeployerDeployment` """ return [DSSProjectDeployerDeployment(self.client, deployment["id"]) for deployment in self.light_status["deployments"]] def get_raw(self): """ Get the raw status information. :return: the status, as a dict. The dict contains a list of the bundles currently deployed on the infrastructure as a **deployments** field. :rtype: dict """ return self.light_status ############################################### # Deployments ############################################### class DSSProjectDeployerDeployment(object): """ A deployment on the Project Deployer. .. important:: Do not instantiate directly, use :meth:`DSSProjectDeployer.get_deployment` """ def __init__(self, client, deployment_id): self.client = client self.deployment_id = deployment_id @property def id(self): """ Get the identifier of the deployment. :rtype: string """ return self.deployment_id def get_status(self): """ Get status information about this deployment. :rtype: dataikuapi.dss.apideployer.DSSProjectDeployerDeploymentStatus """ light = self.client._perform_json("GET", "/project-deployer/deployments/%s" % (self.deployment_id)) heavy = self.client._perform_json("GET", "/project-deployer/deployments/%s/status" % (self.deployment_id)) return DSSProjectDeployerDeploymentStatus(self.client, self.deployment_id, light, heavy) def get_governance_status(self, bundle_id=""): """ Get the governance status about this deployment. The infrastructure on which this deployment is running needs to have a Govern check policy of PREVENT or WARN. :param string bundle_id: (Optional) The ID of a specific bundle of the published project to get status from. If empty, the bundle currently used in the deployment. :return: messages about the governance status, as a dict with a **messages** field, itself a list of meassage information, each one a dict of: * **severity** : severity of the error in the message. Possible values are SUCCESS, INFO, WARNING, ERROR * **isFatal** : for ERROR **severity**, whether the error is considered fatal to the operation * **code** : a string with a well-known code documented in `DSS doc `_ * **title** : short message * **message** : the error message * **details** : a more detailed error description :rtype: dict """ return self.client._perform_json("GET", "/project-deployer/deployments/%s/governance-status" % (self.deployment_id), params={ "bundleId": bundle_id }) def get_settings(self): """ Get the settings of this deployment. :rtype: :class:`DSSProjectDeployerDeploymentSettings` """ settings = self.client._perform_json( "GET", "/project-deployer/deployments/%s/settings" % (self.deployment_id)) return DSSProjectDeployerDeploymentSettings(self.client, self.deployment_id, settings) def start_update(self): """ Start an asynchronous update of this deployment. After the update, the deployment should be matching the actual state to the current settings. :returns: a handle on the update operation :rtype: :class:`dataikuapi.dss.future.DSSFuture` """ future_response = self.client._perform_json( "POST", "/project-deployer/deployments/%s/actions/update" % (self.deployment_id)) return DSSFuture(self.client, future_response.get('jobId', None), future_response) def delete(self): """ Deletes this deployment .. note:: You may only delete a deployment if it is disabled and has been updated after disabling it. """ self.client._perform_empty( "DELETE", "/project-deployer/deployments/%s" % (self.deployment_id)) class DSSProjectDeployerDeploymentSettings(object): """ The settings of a Project Deployer deployment. .. important:: Do not instantiate directly, use :meth:`DSSProjectDeployerDeployment.get_settings` To modify the settings, modify them in the dict returned by :meth:`get_raw()`, or change the value of :meth:`bundle_id()`, then call :meth:`save()`. """ def __init__(self, client, deployment_id, settings): self.client = client self.deployment_id = deployment_id self.settings = settings def get_raw(self): """ Get the raw settings of this deployment. This returns a reference to the raw settings, not a copy, so changes made to the returned object will be reflected when saving. :return: the settings, as a dict. Notable fields are: * **id** : identifier of the deployment * **infraId** : identifier of the infrastructure on which the deployment is done * **bundleId** : identifier of the bundle of the published project being deployed :rtype: dict """ return self.settings @property def bundle_id(self): """ Get or set the identifier of the bundle currently used by this deployment. If setting the value, you need to call :meth:`save()` afterward for the change to be effective. """ return self.settings["bundleId"] @bundle_id.setter def bundle_id(self, new_bundle_id): self.settings["bundleId"] = new_bundle_id def save(self, ignore_warnings=False): """ Save back these settings to the deployment. :param boolean ignore_warnings: whether to ignore warnings concerning the governance status of the bundle to deploy """ self.client._perform_empty( "PUT", "/project-deployer/deployments/%s/settings" % (self.deployment_id), params = { "ignoreWarnings" : ignore_warnings }, body = self.settings) class DSSProjectDeployerDeploymentStatus(object): """ The status of a deployment on the Project Deployer. .. important:: Do not instantiate directly, use :meth:`DSSProjectDeployerDeployment.get_status` """ def __init__(self, client, deployment_id, light_status, heavy_status): self.client = client self.deployment_id = deployment_id self.light_status = light_status self.heavy_status = heavy_status def get_light(self): """ Get the 'light' (summary) status. This returns a dictionary with various information about the deployment, but not the actual health of the deployment :returns: a summary, as a dict, with summary information on the deployment, the project from which the deployed bundle originates, and the infrastructure on which it's deployed. :rtype: dict """ return self.light_status def get_heavy(self): """ Get the 'heavy' (full) status. This returns various information about the deployment, notably its health. :return: a status, as a dict. The overall status of the deployment is in a **health** field (possible values: UNKNOWN, ERROR, WARNING, HEALTHY, UNHEALTHY, OUT_OF_SYNC). :rtype: dict """ return self.heavy_status def get_health(self): """ Get the health of this deployment. :returns: possible values are UNKNOWN, ERROR, WARNING, HEALTHY, UNHEALTHY, OUT_OF_SYNC :rtype: string """ return self.heavy_status["health"] def get_health_messages(self): """ Get messages about the health of this deployment :return: a dict with a **messages** field, which is a list of meassage information, each one a dict of: * **severity** : severity of the error in the message. Possible values are SUCCESS, INFO, WARNING, ERROR * **isFatal** : for ERROR **severity**, whether the error is considered fatal to the operation * **code** : a string with a well-known code documented in `DSS doc `_ * **title** : short message * **message** : the error message * **details** : a more detailed error description :rtype: dict """ return self.heavy_status["healthMessages"] ############################################### # Published Project ############################################### class DSSProjectDeployerProject(object): """ A published project on the Project Deployer. .. important:: Do not instantiate directly, use :meth:`DSSProjectDeployer.get_project` """ def __init__(self, client, project_key): self.client = client self.project_key = project_key @property def id(self): """ Get the key of the published project. :rtype: string """ return self.project_key def get_status(self): """ Get status information about this published project. This is used mostly to get information about which versions are available and which deployments are exposing this project :rtype: :class:`DSSProjectDeployerProjectStatus` """ light = self.client._perform_json("GET", "/project-deployer/projects/%s" % (self.project_key)) return DSSProjectDeployerProjectStatus(self.client, self.project_key, light) def get_settings(self): """ Get the settings of this published project. The main things that can be modified in a project settings are permissions :rtype: :class:`DSSProjectDeployerProjectSettings` """ settings = self.client._perform_json( "GET", "/project-deployer/projects/%s/settings" % (self.project_key)) return DSSProjectDeployerProjectSettings(self.client, self.project_key, settings) def delete_bundle(self, bundle_id): """ Delete a bundle from this published project. :param string bundle_id: identifier of the bundle to delete """ self.client._perform_empty( "DELETE", "/project-deployer/projects/%s/bundles/%s" % (self.project_key, bundle_id)) def delete(self): """ Delete this published project. .. note:: You may only delete a published project if there are no deployments using it. """ self.client._perform_empty( "DELETE", "/project-deployer/projects/%s" % (self.project_key)) class DSSProjectDeployerProjectSettings(object): """ The settings of a published project. .. important:: Do not instantiate directly, use :meth:`DSSProjectDeployerProject.get_settings` To modify the settings, modify them in the dict returned by :meth:`get_raw()` then call :meth:`save()`. """ def __init__(self, client, project_key, settings): self.client = client self.project_key = project_key self.settings = settings def get_raw(self): """ Get the raw settings of this published project. This returns a reference to the raw settings, not a copy, so changes made to the returned object will be reflected when saving. :return: the settings, as a dict. :rtype: dict """ return self.settings def save(self): """ Save back these settings to the published project. """ self.client._perform_empty( "PUT", "/project-deployer/projects/%s/settings" % (self.project_key), body = self.settings) class DSSProjectDeployerProjectStatus(object): """ The status of a published project. .. important:: Do not instantiate directly, use :meth:`DSSProjectDeployerProject.get_status` """ def __init__(self, client, project_key, light_status): self.client = client self.project_key = project_key self.light_status = light_status def get_deployments(self, infra_id=None): """ Get the deployments that have been created from this published project. :param string infra_id: (optional) identifier of an infrastructure. When set, only get the deployments deployed on this infrastructure. When not set, the list contains all the deployments using this published project, across every infrastructure of the Project Deployer. :returns: a list of deployments, each a :class:`DSSProjectDeployerDeployment` :rtype: list """ if infra_id is None: return [DSSProjectDeployerDeployment(self.client, deployment["id"]) for deployment in self.light_status["deployments"]] return [DSSProjectDeployerDeployment(self.client, deployment["id"]) for deployment in self.light_status["deployments"] if infra_id == deployment["infraId"]] def get_bundles(self): """ Get the bundles that have been published on this project. Each bundle is a dict that contains at least a "id" field, which is the version identifier :returns: a list of bundles, each one a dict. Each bundle has an **id** field holding its identifier. :rtype: list[dict] """ return self.light_status["packages"] def get_infras(self): """ Get the infrastructures that deployments of this project use. :returns: list of summaries of infrastructures, each a dict. :rtype: list[dict] """ return self.light_status["infras"] def get_raw(self): """ Gets the raw status information. :return: the status, as a dict. A **deployments** sub-field contains a list of the deployments of bundles of this projects. :rtype: dict """ return self.light_status dataiku-api-client-python-13.2.2/dataikuapi/dss/projectfolder.py000066400000000000000000000235671471211575100247250ustar00rootroot00000000000000from .project import DSSProject import warnings class DSSProjectFolder(object): """ A handle for a project folder on the DSS instance. .. important:: Do not instantiate this class directly, instead use :meth:`dataikuapi.DSSClient.get_project_folder` or :meth:`dataikuapi.DSSClient.get_root_project_folder`. """ def __init__(self, client, data): self.client = client self._data = data ######################################################## # Project folder basics ######################################################## @property def id(self): """ :returns: The project folder id. :rtype: string """ return self._data["id"] @property def project_folder_id(self): """ .. caution:: Deprecated. Please use :attr:`dataikuapi.dss.projectfolder.DSSProjectFolder.id`. """ warnings.warn("DSSProjectFolder.project_folder_id is deprecated, please use id", DeprecationWarning) return self.id @property def name(self): """ :returns: The project folder name or :const:`None` for the root project folder. :rtype: string """ return self._data.get("name", None) def get_name(self): """ See :attr:`dataikuapi.dss.projectfolder.DSSProjectFolder.name`. :returns: The project folder name or :const:`None` for the root project folder. :rtype: string """ return self.name def get_path(self): """ :returns: The project folder path from the root project folder (e.g. :const:`'/'` or :const:`'/foo/bar'`). :rtype: string """ parent = self.get_parent() if parent is not None: path = parent.get_path() return ("" if path == "/" else path) + "/" + self.name else: return "/" def get_parent(self): """ :returns: A handle for the parent folder or :const:`None` for the root project folder. :rtype: :class:`dataikuapi.dss.projectfolder.DSSProjectFolder` """ parent_id = self._data.get("parentId", None) if parent_id is None: return None else: return self.client.get_project_folder(parent_id) def list_child_folders(self): """ :returns: Handles for every child project folder. :rtype: list of :class:`dataikuapi.dss.projectfolder.DSSProjectFolder` """ return [self.client.get_project_folder(child_id) for child_id in self._data["childrenIds"]] def list_project_keys(self): """ :returns: The project keys of all projects stored in this project folder. :rtype: list of string """ return self._data["projectKeys"] def list_projects(self): """ :returns: Handles for every project stored in this project folder. :rtype: list of :class:`dataikuapi.dss.project.DSSProject` """ return [self.client.get_project(pkey) for pkey in self.list_project_keys()] ######################################################## # Project folder deletion ######################################################## def delete(self): """ Delete this project folder. .. important:: This project folder must be empty, i.e. contain no project or subfolder. You must move or remove all this project folder content prior to deleting it. .. attention:: This call requires an API key with admin rights. """ return self.client._perform_empty("DELETE", "/project-folders/%s" % self.id) ######################################################## # Project folder settings ######################################################## def get_settings(self): """ :returns: A handle for this project folder settings. :rtype: :class:`dataikuapi.dss.projectfolder.DSSProjectFolderSettings` """ settings = self.client._perform_json("GET", "/project-folders/%s/settings" % self.id) return DSSProjectFolderSettings(self.client, self.id, settings) ######################################################## # Project folder subfolder creation ######################################################## def create_sub_folder(self, name): """ Create a project subfolder inside this project folder. :param str name: The name of the project subfolder to create. :returns: A handle for the created project subfolder. :rtype: :class:`dataikuapi.dss.projectfolder.DSSProjectFolder` """ params = { "name": name } pf = self.client._perform_json("POST", "/project-folders/%s/children" % self.id, params=params) return DSSProjectFolder(self.client, pf) ######################################################## # Project creation ######################################################## def create_project(self, project_key, name, owner, description=None, settings=None): """ Create a new project within this project folder. Return a handle for the created project. .. important:: The provided identifier for the new project must be globally unique. .. attention:: This call requires an API key with admin rights or the right to create a project. :param str project_key: The identifier for the new project. Must be globally unique. :param str name: The displayed name for the new project. :param str owner: The login of the new project owner. :param str description: The description for the new project. :param dict settings: The initial settings for the new project. The settings can be modified later. The exact possible settings are not documented. :returns: A handle for the created project. :rtype: :class:`dataikuapi.dss.project.DSSProject` """ return self.client.create_project(project_key, name, owner, description=description, settings=settings, project_folder_id=self.id) ######################################################## # Project folder move ######################################################## def move_to(self, destination): """ Move this project folder into another project folder. :param destination: The new parent project folder of this project folder. :type destination: :class:`dataikuapi.dss.projectfolder.DSSProjectFolder` """ params = { "destination": destination.id } self.client._perform_empty("POST", "/project-folders/%s/move" % self.id, params=params) ######################################################## # Project move ######################################################## def move_project_to(self, project_key, destination): """ Move a project from this project folder into another project folder. :param str project_key: The identifier of the project to move. :param destination: The new parent project folder of the project. :type destination: :class:`dataikuapi.dss.projectfolder.DSSProjectFolder` """ # Be nice with what people pass if isinstance(project_key, DSSProject): project_key = project_key.project_key params = { "destination": destination.id } self.client._perform_empty("POST", "/project-folders/%s/projects/%s/move" % (self.id, project_key), params=params) class DSSProjectFolderSettings(object): """ A handle for a project folder settings. .. important:: Do not instantiate this class directly, instead use :meth:`dataikuapi.dss.projectfolder.DSSProjectFolder.get_settings`. """ def __init__(self, client, project_folder_id, settings): self.client = client self.id = project_folder_id self.settings = settings def get_raw(self): """ Get the settings of the project folder as a python dict. .. important:: Returns a reference to the raw settings in opposition to a copy. Changes through the reference will be effective upon saving. :returns: The settings of the project folder as a python dict containing the keys: * **name**: the name of the project folder, * **owner**: the login of the project folder owner, * **permissions**: the list of the project folder permissions. :rtype: python dict """ return self.settings def get_name(self): """ :returns: The name of the project folder. :rtype: string """ return self.settings.get("name", None) def set_name(self, name): """ Set the name of the project folder. :param str name: The new name of the project folder. """ self.settings["name"] = name def get_owner(self): """ :returns: The login of the project folder owner. :rtype: string """ return self.settings.get("owner", None) def set_owner(self, owner): """ Set the owner of the project folder. :param str owner: The login of the new project folder owner. """ self.settings["owner"] = owner def get_permissions(self): """ Get the permissions of the project folder. .. important:: Returns a reference to the permissions in opposition to a copy. Changes through the reference will be effective upon saving. :returns: The permissions of the project folder. :rtype: list of string """ return self.settings["permissions"] def save(self): """ Save back the settings to the project folder. """ self.client._perform_empty("PUT", "/project-folders/%s/settings" % (self.id), body = self.settings) dataiku-api-client-python-13.2.2/dataikuapi/dss/projectlibrary.py000066400000000000000000000331471471211575100251110ustar00rootroot00000000000000from ..utils import DataikuException import sys import base64 if sys.version_info >= (3,0): import urllib.parse dku_quote_fn = urllib.parse.quote else: import urllib dku_quote_fn = urllib.quote class DSSLibrary(object): """ A handle to manage the library of a project It saves locally a copy of taxonomy to help navigate in the library All modifications done through this object and related library items are done locally and on remote. .. note:: Taxonomy modifications done outside this library are not reflected locally. You should reload the library in this case. """ def __init__(self, client, project_key): """Do not call directly, use :meth:`dataikuapi.dss.project.DSSProject.get_library`""" self.client = client self.project_key = project_key self.root = self._build_node_("/", None, self.client._perform_json("GET", "/projects/%s/libraries/contents" % (self.project_key))) def _build_node_(self, name, parent, children): """ Helper that builds the tree recursively :param str name: the name of the library item :param parent: the parent of the library item :type parent: :class:`dataikuapi.dss.projectlibrary.DSSLibraryFolder` :param list children: the children of the library item :returns: a new node corresponding to an item in the library :rtype: :class:`dataikuapi.dss.projectlibrary.DSSLibraryItem` """ if children is None: return DSSLibraryFile(self.client, self.project_key, name, parent) item = DSSLibraryFolder(self.client, self.project_key, name, parent, set()) for child in children: item._add_child_(self._build_node_(child["name"], item, child.get("children"))) return item def list(self, folder_path="/"): """ Lists the contents in the given library folder or on the root if no folder is given. :param str folder_path: the folder path (optional). If no path is given, it is defaulted to the root path. :returns: the list of contents in the library folder :rtype: list of :class:`dataikuapi.dss.projectlibrary.DSSLibraryItem` """ return self.get_folder(folder_path).list() def get_file(self, path): """ Retrieves a file in the library :param str path: the file path :returns: the file in the given path :rtype: :class:`dataikuapi.dss.projectlibrary.DSSLibraryFile` """ return self.root.get_file(path) def get_folder(self, path): """ Retrieves a folder in the library :param str path: the folder path :returns: the folder in the given path :rtype: :class:`dataikuapi.dss.projectlibrary.DSSLibraryFolder` """ return self.root.get_folder(path) def add_file(self, file_name): """ Create a file in the library root folder :param str file_name: the file name :returns: the new file :rtype: :class:`dataikuapi.dss.projectlibrary.DSSLibraryFile` """ return self.root.add_file(file_name) def add_folder(self, folder_name): """ Create a folder in the library root folder :param str folder_name: the folder name :returns: the new folder :rtype: :class:`dataikuapi.dss.projectlibrary.DSSLibraryFolder` """ return self.root.add_folder(folder_name) class DSSLibraryItem(object): """ A handle to manage a library item """ def __init__(self, client, project_key, name, parent, children): """Do not call directly, use :meth:`dataikuapi.dss.projectlibrary.DSSLibrary.get_file()` or :meth:`dataikuapi.dss.projectlibrary.DSSLibrary.get_folder()`""" self.client = client self.project_key = project_key self.name = name self.parent = parent self.children = children @property def path(self): if self.parent is None: return self.name return self._clean_path_(self.parent.path + "/" + self.name) def is_root(self): return self.parent is None and self.name == "/" def _clean_path_(self, path): return path.replace("//", "/") def delete(self): """ Deletes this item from library """ if self.is_root(): raise Exception("Cannot delete root folder") self.client._perform_empty("DELETE", "/projects/%s/libraries/contents/%s" % (self.project_key, dku_quote_fn(self.path))) self.parent._remove_child_(self) self.parent = None def rename(self, new_name): """ Rename the folder :param str new_name: the new name of the item """ if self.is_root(): raise Exception("Cannot rename root folder") self.client._perform_empty("POST", "/projects/%s/libraries/contents-actions/rename/" % self.project_key, body={"oldPath": self.path, "newName": new_name}) self.name = new_name def move_to(self, destination_folder): """ Move a library item to another folder :param destination_folder: the folder where we want to move the current item :type destination_folder: :class:`dataikuapi.dss.projectlibrary.DSSLibraryFolder` """ if self.is_root(): raise DataikuException("Cannot move root folder") if destination_folder is None: raise DataikuException("Destination folder cannot be None") if not isinstance(destination_folder, DSSLibraryFolder): raise DataikuException("Destination should be a folder") self.client._perform_empty("POST", "/projects/%s/libraries/contents-actions/move" % self.project_key, body={"oldPath": self.path, "newPath": destination_folder.path}) self.parent._remove_child_(self) destination_folder._add_child_(self) self.parent = destination_folder class DSSLibraryFolder(DSSLibraryItem): """ A handle to manage a library folder .. warning:: Do not call directly, use :class:`dataikuapi.dss.projectlibrary.DSSLibrary.get_folder` """ def __init__(self, client, project_key, name, parent, children): """Do not call directly, use :class:`dataikuapi.dss.projectlibrary.DSSLibrary.get_folder`""" super(DSSLibraryFolder, self).__init__(client, project_key, name, parent, children) def get_child(self, name): """ Retrieve the sub item by its name :param str name: the name of the sub item :returns: the sub item :rtype: :class:`dataikuapi.dss.projectlibrary.DSSLibraryItem` """ for c in self.children: if c.name == name: return c return None def add_file(self, file_name): """ Create a new file in the library folder :param str file_name: the file name :returns: the new file :rtype: :class:`dataikuapi.dss.projectlibrary.DSSLibraryFile` """ file_name = file_name.strip() if self.is_root(): new_path = dku_quote_fn(file_name) else: new_path = dku_quote_fn(self.path + "/" + file_name) try: existing_file = self.client._perform_json("GET", "/projects/%s/libraries/contents/%s" % (self.project_key, new_path)) except: existing_file = None finally: if existing_file is not None: raise DataikuException("File %s already exists" % file_name) self.client._perform_empty("POST", "/projects/%s/libraries/contents/%s" % (self.project_key, new_path)) return self._create_library_item_(self, file_name, True) def add_folder(self, folder_name): """ Create a folder in the library :param str folder_name: the name of the folder to add :returns: the new folder :rtype: :class:`dataikuapi.dss.projectlibrary.DSSLibraryFolder` """ folder_name = folder_name.strip() if self.is_root(): new_path = dku_quote_fn(folder_name) else: new_path = dku_quote_fn(self.path + "/" + folder_name) self.client._perform_empty("POST", "/projects/%s/libraries/folders/%s" % (self.project_key, new_path)) return self._create_library_item_(self, folder_name, False) def _create_library_item_(self, item, path, create_file): if path.startswith("/"): path = path[1:] path_split = path.split('/', 1) if path_split[0] == ".": return self._create_library_item_(item, path_split[1], create_file) if path_split[0] == "..": return self._create_library_item_(item.parent, path_split[1], create_file) if len(path_split) > 1 and path_split[1]: # Check if the path is a child's sub path for c in item.children: if c.name == path_split[0]: return self._create_library_item_(c, path_split[1], create_file) # Path element not present in folder. We should create it folder = DSSLibraryFolder(self.client, self.project_key, path_split[0], item, set()) item.children.add(folder) return self._create_library_item_(folder, path_split[1], create_file) else: # It is a direct child if create_file: new_item = DSSLibraryFile(self.client, self.project_key, path_split[0], item) else: new_item = DSSLibraryFolder(self.client, self.project_key, path_split[0], item, set()) item.children.add(new_item) return new_item def list(self): """ Gets the contents of this folder sorted by name :returns: a sorted list of items :rtype: list of :class:`dataikuapi.dss.projectlibrary.DSSLibraryItem` """ return sorted(list(self.children), key=lambda x: x.name) def _add_child_(self, child): """ Adds the given child from this folder """ self.children.add(child) def _remove_child_(self, child): """ Removes the given child from this folder """ self.children.remove(child) def get_file(self, path): """ Retrieves a file in the library :param str path: the file path :returns: the file in the given path :rtype: :class:`dataikuapi.dss.projectlibrary.DSSLibraryFile` """ item = self._get_item_recursively_(self, path) if item is not None: if item.children is None: return item else: raise DataikuException("The item %s is a folder, not a file " % path) return None def get_folder(self, path): """ Retrieves a folder in the library :param str path: the folder path :returns: the folder in the given path :rtype: :class:`dataikuapi.dss.projectlibrary.DSSLibraryFolder` """ item = self._get_item_recursively_(self, path) if item is not None: if item.children is not None: return item else: raise DataikuException("The item %s is a file, not a folder " % path) return None def _get_item_recursively_(self, item, path): if not path or item is None: return None if path == "/": return item if path.startswith("/"): path = path[1:] path_split = path.split('/', 1) if path_split[0] == ".": if len(path_split) > 1: return self._get_item_recursively_(item, path_split[1]) else: return item if path_split[0] == "..": if item.parent is None: raise DataikuException("Path %s is not valid" % path) if len(path_split) > 1: return self._get_item_recursively_(item.parent, path_split[1]) else: return item.parent if item.children is None: return None for c in item.children: if c.name == path_split[0]: if len(path_split) > 1: return self._get_item_recursively_(c, path_split[1]) else: return c return None class DSSLibraryFile(DSSLibraryItem): """ A handle to manage a library file .. warning:: Do not call directly, use :meth:`dataikuapi.dss.projectlibrary.DSSLibrary.get_file` """ def __init__(self, client, project_key, name, parent): """Do not call directly, use :meth:`dataikuapi.dss.projectlibrary.DSSLibrary.get_file`""" super(DSSLibraryFile, self).__init__(client, project_key, name, parent, None) def read(self, as_type="str"): """ Get the file contents from DSS :param str as_type: specify whether you want to read the file in text mode (as_type='str') or in binary mode (as_type='bytes'). Defaults to text mode. :returns: the contents of the file as a string """ url = "/projects/%s/libraries/contents/%s" % (self.project_key, dku_quote_fn(self.path)) if as_type == "str": return self.client._perform_json("GET", url)["data"] elif as_type == "bytes": return base64.b64decode(self.client._perform_json("GET", url, params={"dataEncoding": "base64"})["data"]) else: raise ValueError("Unknown as_type") def write(self, data): """ Updates the contents of the file with the given data """ self.client._perform_empty("POST", "/projects/%s/libraries/contents/%s" % (self.project_key, dku_quote_fn(self.path)), raw_body=data) dataiku-api-client-python-13.2.2/dataikuapi/dss/recipe.py000066400000000000000000002624251471211575100233300ustar00rootroot00000000000000from ..utils import DataikuException from .utils import DSSTaggableObjectSettings from .discussion import DSSObjectDiscussions import json, logging, warnings from .utils import DSSTaggableObjectListItem, DSSTaggableObjectSettings try: basestring except NameError: basestring = str class DSSRecipeListItem(DSSTaggableObjectListItem): """ An item in a list of recipes. .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.list_recipes()` """ def __init__(self, client, data): super(DSSRecipeListItem, self).__init__(data) self.client = client def to_recipe(self): """ Gets a handle corresponding to this recipe. :rtype: :class:`DSSRecipe` """ return DSSRecipe(self.client, self._data["projectKey"], self._data["name"]) @property def name(self): """ Get the name of the recipe. :rtype: string """ return self._data["name"] @property def id(self): """ Get the identifier of the recipe. For recipes, the name is the identifier. :rtype: string """ return self._data["name"] @property def type(self): """ Get the type of the recipe. :return: a recipe type, for example 'sync' or 'join' :rtype: string """ return self._data["type"] class DSSRecipe(object): """ A handle to an existing recipe on the DSS instance. .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.get_recipe()` """ def __init__(self, client, project_key, recipe_name): self.client = client self.project_key = project_key self.recipe_name = recipe_name @property def id(self): """ Get the identifier of the recipe. For recipes, the name is the identifier. :rtype: string """ return self.recipe_name @property def name(self): """ Get the name of the recipe. :rtype: string """ return self.recipe_name def compute_schema_updates(self): """ Computes which updates are required to the outputs of this recipe. This method only computes which changes would be needed to make the schema of the outputs of the recipe match the actual schema that the recipe will produce. To effectively apply these changes to the outputs, you can use the :meth:`~RequiredSchemaUpdates.apply()` on the returned object. .. note:: Not all recipe types can compute automatically the schema of their outputs. Code recipes like Python recipes, notably can't. This method raises an exception in these cases. Usage example: .. code-block:: python required_updates = recipe.compute_schema_updates() if required_updates.any_action_required(): print("Some schemas will be updated") # Note that you can call apply even if no changes are required. This will be noop required_updates.apply() :return: an object containing the required updates :rtype: :class:`RequiredSchemaUpdates` """ data = self.client._perform_json( "GET", "/projects/%s/recipes/%s/schema-update" % (self.project_key, self.recipe_name)) return RequiredSchemaUpdates(self, data) def run(self, job_type="NON_RECURSIVE_FORCED_BUILD", partitions=None, wait=True, no_fail=False): """ Starts a new job to run this recipe and wait for it to complete. Raises if the job failed. .. code-block:: python job = recipe.run() print("Job %s done" % job.id) :param string job_type: job type. One of RECURSIVE_BUILD, NON_RECURSIVE_FORCED_BUILD or RECURSIVE_FORCED_BUILD :param string partitions: if the outputs are partitioned, a partition spec. A spec is a comma-separated list of partition identifiers, and a partition identifier is a pipe-separated list of values for the partitioning dimensions :param boolean no_fail: if True, does not raise if the job failed :param boolean wait: if True, the method waits for the job complettion. If False, the method returns immediately :return: a job handle corresponding to the recipe run :rtype: :class:`dataikuapi.dss.job.DSSJob` """ project = self.client.get_project(self.project_key) outputs = project.get_flow().get_graph().get_successor_computables(self) if len(outputs) == 0: raise Exception("recipe has no outputs, can't run it") first_output = outputs[0] object_type_map = { "COMPUTABLE_DATASET": "DATASET", "COMPUTABLE_FOLDER": "MANAGED_FOLDER", "COMPUTABLE_SAVED_MODEL": "SAVED_MODEL", "COMPUTABLE_STREAMING_ENDPOINT": "STREAMING_ENDPOINT", "COMPUTABLE_MODEL_EVALUATION_STORE": "MODEL_EVALUATION_STORE" } if first_output["type"] in object_type_map: jd = project.new_job(job_type) jd.with_output(first_output["ref"], object_type=object_type_map[first_output["type"]], partition=partitions) else: raise Exception("Recipe has unsupported output type {}, can't run it".format(first_output["type"])) if wait: return jd.start_and_wait(no_fail) else: return jd.start() def delete(self): """ Delete the recipe. """ return self.client._perform_empty( "DELETE", "/projects/%s/recipes/%s" % (self.project_key, self.recipe_name)) def rename(self, new_name): """ Rename the recipe with the new specified name :param str new_name: the new name of the recipe """ if self.recipe_name == new_name: raise ValueError("Recipe name is already " + new_name) obj = { "oldName": self.recipe_name, "newName": new_name } self.client._perform_empty("POST", "/projects/%s/actions/renameRecipe" % self.project_key, body=obj) self.recipe_name = new_name def get_settings(self): """ Get the settings of the recipe, as a :class:`DSSRecipeSettings` or one of its subclasses. Some recipes have a dedicated class for the settings, with additional helpers to read and modify the settings Once you are done modifying the returned settings object, you can call :meth:`~DSSRecipeSettings.save` on it in order to save the modifications to the DSS recipe. :rtype: :class:`DSSRecipeSettings` or a subclass """ data = self.client._perform_json( "GET", "/projects/%s/recipes/%s" % (self.project_key, self.recipe_name)) type = data["recipe"]["type"] if type == "generate_features": return GenerateFeaturesRecipeSettings(self, data) if type == "grouping": return GroupingRecipeSettings(self, data) elif type == "window": return WindowRecipeSettings(self, data) elif type == "sync": return SyncRecipeSettings(self, data) elif type == "pivot": return PivotRecipeSettings(self, data) elif type == "sort": return SortRecipeSettings(self, data) elif type == "topn": return TopNRecipeSettings(self, data) elif type == "distinct": return DistinctRecipeSettings(self, data) elif type == "join": return JoinRecipeSettings(self, data) elif type == "vstack": return StackRecipeSettings(self, data) elif type == "sampling": return SamplingRecipeSettings(self, data) elif type == "split": return SplitRecipeSettings(self, data) elif type == "prepare" or type == "shaker": return PrepareRecipeSettings(self, data) #elif type == "prediction_scoring": #elif type == "clustering_scoring": elif type == "download": return DownloadRecipeSettings(self, data) #elif type == "sql_query": # return WindowRecipeSettings(self, data) elif type in ["python", "r", "sql_script", "pyspark", "sparkr", "spark_scala", "shell"]: return CodeRecipeSettings(self, data) else: return DSSRecipeSettings(self, data) def get_definition_and_payload(self): """ Get the definition of the recipe. .. attention:: Deprecated. Use :meth:`get_settings` :return: an object holding both the raw definition of the recipe (the type, which inputs and outputs, engine settings...) and the payload (SQL script, Python code, join definition,... depending on type) :rtype: :class:`DSSRecipeDefinitionAndPayload` """ warnings.warn("Recipe.get_definition_and_payload is deprecated, please use get_settings", DeprecationWarning) data = self.client._perform_json( "GET", "/projects/%s/recipes/%s" % (self.project_key, self.recipe_name)) return DSSRecipeDefinitionAndPayload(self, data) def set_definition_and_payload(self, definition): """ Set the definition of the recipe. .. attention:: Deprecated. Use :meth:`get_settings` then :meth:`DSSRecipeSettings.save()` .. important:: The **definition** parameter should come from a call to :meth:`get_definition()` :param object definition: a recipe definition, as returned by :meth:`get_definition()` """ warnings.warn("Recipe.set_definition_and_payload is deprecated, please use get_settings", DeprecationWarning) definition._payload_to_str() return self.client._perform_json( "PUT", "/projects/%s/recipes/%s" % (self.project_key, self.recipe_name), body=definition.data) def get_status(self): """ Gets the status of this recipe. The status of a recipe is made of messages from checks performed by DSS on the recipe, of messages related to engines availability for the recipe, of messages about testing the recipe on the engine, ... :return: an object to interact with the status :rtype: :class:`dataikuapi.dss.recipe.DSSRecipeStatus` """ data = self.client._perform_json( "GET", "/projects/%s/recipes/%s/status" % (self.project_key, self.recipe_name)) return DSSRecipeStatus(self.client, data) def get_metadata(self): """ Get the metadata attached to this recipe. The metadata contains label, description checklists, tags and custom metadata of the recipe :return: the metadata as a dict, with fields: * **label** : label of the object (not defined for recipes) * **description** : description of the object (not defined for recipes) * **checklists** : checklists of the object, as a dict with a **checklists** field, which is a list of checklists, each a dict. * **tags** : list of tags, each a string * **custom** : custom metadata, as a dict with a **kv** field, which is a dict with any contents the user wishes * **customFields** : dict of custom field info (not defined for recipes) :rtype: dict """ return self.client._perform_json( "GET", "/projects/%s/recipes/%s/metadata" % (self.project_key, self.recipe_name)) def set_metadata(self, metadata): """ Set the metadata on this recipe. .. important:: You should only set a **metadata** object that has been retrieved using :meth:`get_metadata()`. :params dict metadata: the new state of the metadata for the recipe. """ return self.client._perform_json( "PUT", "/projects/%s/recipes/%s/metadata" % (self.project_key, self.recipe_name), body=metadata) def get_input_refs(self): inputs = self.get_settings().data["recipe"]["inputs"]["main"]["items"] return [inp["ref"] for inp in inputs] def get_object_discussions(self): """ Get a handle to manage discussions on the recipe. :return: the handle to manage discussions :rtype: :class:`dataikuapi.dss.discussion.DSSObjectDiscussions` """ return DSSObjectDiscussions(self.client, self.project_key, "RECIPE", self.recipe_name) def get_continuous_activity(self): """ Get a handle on the associated continuous activity. .. note:: Should only be used on continuous recipes. :rtype: :class:`dataikuapi.dss.continuousactivity.DSSContinuousActivity` """ from .continuousactivity import DSSContinuousActivity return DSSContinuousActivity(self.client, self.project_key, self.recipe_name) def move_to_zone(self, zone): """ Move this object to a flow zone. :param object zone: a :class:`dataikuapi.dss.flow.DSSFlowZone` where to move the object """ if isinstance(zone, basestring): zone = self.client.get_project(self.project_key).get_flow().get_zone(zone) zone.add_item(self) class DSSRecipeStatus(object): """ Status of a recipe. .. important:: Do not instantiate directly, use :meth:`DSSRecipe.get_status` """ def __init__(self, client, data): self.client = client self.data = data def get_selected_engine_details(self): """ Get the selected engine for this recipe. This method will raise if there is no selected engine, whether it's because the present recipe type has no notion of engine, or because DSS couldn't find any viable engine for running the recipe. :return: a dict of the details of the selected engine. The type of engine is in a **type** field. Depending on the type, additional field will give more details, like whether some aggregations are possible. :rtype: dict """ if not "selectedEngine" in self.data: raise ValueError("This recipe doesn't have a selected engine") return self.data["selectedEngine"] def get_engines_details(self): """ Get details about all possible engines for this recipe. This method will raise if there is no engine, whether it's because the present recipe type has no notion of engine, or because DSS couldn't find any viable engine for running the recipe. :return: a list of dict of the details of each possible engine. See :meth:`get_selected_engine_details()` for the fields of each dict. :rtype: list[dict] """ if not "engines" in self.data: raise ValueError("This recipe doesn't have engines") return self.data["engines"] def get_status_severity(self): """ Get the overall status of the recipe. This is the final result of checking the different parts of the recipe, and depends on the recipe type. Examples of checks done include: - checking the validity of the formulas in computed columns or filters - checking if some of the input columns retrieved by joins overlap - checking against the SQL database if the generated SQL is valid :return: SUCCESS, WARNING, ERROR or INFO. None if the status has no message at all. :rtype: string """ return self.data["allMessagesForFrontend"].get("maxSeverity") def get_status_messages(self, as_objects=False): """ Returns status messages for this recipe. :param boolean as_objects: if True, return a list of :class:`dataikuapi.dss.utils.DSSInfoMessage`. If False, as a list of raw dicts. :return: if **as_objects** is True, a list of :class:`dataikuapi.dss.utils.DSSInfoMessage`, otherwise a list of message information, each one a dict of: * **severity** : severity of the error in the message. Possible values are SUCCESS, INFO, WARNING, ERROR * **isFatal** : for ERROR **severity**, whether the error is considered fatal to the operation * **code** : a string with a well-known code documented in `DSS doc `_ * **title** : short message * **message** : the error message * **details** : a more detailed error description :rtype: list """ if as_objects: return [DSSInfoMessage(message) for message in self.data["allMessagesForFrontend"].get("messages", [])] else: return self.data["allMessagesForFrontend"]["messages"] class DSSRecipeSettings(DSSTaggableObjectSettings): """ Settings of a recipe. .. important:: Do not instantiate directly, use :meth:`DSSRecipe.get_settings` """ def __init__(self, recipe, data): super(DSSRecipeSettings, self).__init__(data["recipe"]) self.recipe = recipe self.data = data self.recipe_settings = self.data["recipe"] self._str_payload = self.data.get("payload", None) self._obj_payload = None def save(self): """ Save back the recipe in DSS. """ self._payload_to_str() return self.recipe.client._perform_json( "PUT", "/projects/%s/recipes/%s" % (self.recipe.project_key, self.recipe.recipe_name), body=self.data) @property def type(self): """ Get the type of the recipe. :return: a type, like 'sync', 'python' or 'join' :rtype: string """ return self.recipe_settings["type"] @property def str_payload(self): """ The raw "payload" of the recipe. This is exactly the data persisted on disk. :return: for code recipes, the payload will be the script of the recipe. For visual recipes, the payload is a JSON of settings that are specific to the recipe type, like the definitions of the aggregations for a grouping recipe. :rtype: string """ self._payload_to_str() return self._str_payload @str_payload.setter def str_payload(self, payload): self._str_payload = payload self._obj_payload = None @property def obj_payload(self): """ The "payload" of the recipe, parsed from JSON. .. note:: Do not use on code recipes, their payload isn't JSON-encoded. :return: settings that are specific to the recipe type, like the definitions of the aggregations for a grouping recipe. :rtype: dict """ self._payload_to_obj() return self._obj_payload @property def raw_params(self): """ The non-payload settings of the recipe. :return: recipe type-specific settings that aren't stored in the payload. Typically this comprises engine settings. :rtype: dict """ return self.recipe_settings["params"] def _payload_to_str(self): if self._obj_payload is not None: self._str_payload = json.dumps(self._obj_payload) self._obj_payload = None if self._str_payload is not None: self.data["payload"] = self._str_payload def _payload_to_obj(self): if self._str_payload is not None: self._obj_payload = json.loads(self._str_payload) self._str_payload = None def get_recipe_raw_definition(self): """ Get the recipe definition. :return: the part of the recipe's settings that aren't stored in the payload, as a dict. Notable fields are: * **name** and **projectKey** : identifiers of the recipe * **type** : type of the recipe * **params** : type-specific parameters of the recipe (on top of what is in the payload) * **inputs** : input roles to the recipe, as a dict of role name to role, where a role is a dict with an **items** field consisting of a list of one dict per input object. Each individual input has fields: * **ref** : a dataset name or a managed folder id or a saved model id. Should be prefixed by the project key for exposed items, like in "PROJECT_KEY.dataset_name" * **deps** : for partitioned inputs, a list of partition dependencies mapping output dimensions to dimensions in this input. Each partition dependency is a dict. * **outputs** : output roles to the recipe, as a dict of role name to role, where a role is a dict with a **items** field consisting of a list of one dict per output object. Each individual output has fields: * **ref** : a dataset name or a managed folder id or a saved model id. Should be prefixed by the project key for exposed items, like in "PROJECT_KEY.dataset_name" * **appendMode** : if True, the recipe should append into the output; if False, the recipe should overwrite the output when running :rtype: dict """ return self.recipe_settings def get_recipe_inputs(self): """ Get the inputs to this recipe. :rtype: dict """ return self.recipe_settings.get('inputs') def get_recipe_outputs(self): """ Get the outputs of this recipe. :rtype: dict """ return self.recipe_settings.get('outputs') def get_recipe_params(self): """ The non-payload settings of the recipe. :return: recipe type-specific settings that aren't stored in the payload. Typically this comprises engine settings. :rtype: dict """ return self.recipe_settings.get('params') def get_payload(self): """ The raw "payload" of the recipe. This is exactly the data persisted on disk. :return: for code recipes, the payload will be the script of the recipe. For visual recipes, the payload is a JSON of settings that are specific to the recipe type, like the definitions of the aggregations for a grouping recipe. :rtype: string """ self._payload_to_str() return self._str_payload def get_json_payload(self): """ The "payload" of the recipe, parsed from JSON. .. note:: Do not use on code recipes, their payload isn't JSON-encoded. :return: settings that are specific to the recipe type, like the definitions of the aggregations for a grouping recipe. :rtype: dict """ self._payload_to_obj() return self._obj_payload def set_payload(self, payload): """ Set the payload of this recipe. :param string payload: the payload, as a string """ self._str_payload = payload self._obj_payload = None def set_json_payload(self, payload): """ Set the payload of this recipe. :param dict payload: the payload, as a dict. Will be converted to JSON internally. """ self._str_payload = None self._obj_payload = payload def has_input(self, input_ref): """ Whether a ref is part of the recipe's inputs. :param string input_ref: a ref to an object in DSS, i.e. a dataset name or a managed folder id or a saved model id. Should be prefixed by the project key for exposed items, like in "PROJECT_KEY.dataset_name" :rtype: boolean """ inputs = self.get_recipe_inputs() for (input_role_name, input_role) in inputs.items(): for item in input_role.get("items", []): if item.get("ref", None) == input_ref: return True return False def has_output(self, output_ref): """ Whether a ref is part of the recipe's outputs. :param string output_ref: a ref to an object in DSS, i.e. a dataset name or a managed folder id or a saved model id. Should be prefixed by the project key for exposed items, like in "PROJECT_KEY.dataset_name" :rtype: boolean """ outputs = self.get_recipe_outputs() for (output_role_name, output_role) in outputs.items(): for item in output_role.get("items", []): if item.get("ref", None) == output_ref: return True return False def replace_input(self, current_input_ref, new_input_ref): """ Replaces an input of this recipe by another. If the **current_input_ref** isn't part of the recipe's inputs, this method has no effect. :param string current_input_ref: a ref to an object in DSS, i.e. a dataset name or a managed folder id or a saved model id, that is currently input to the recipe :param string new_input_ref: a ref to an object in DSS, i.e. a dataset name or a managed folder id or a saved model id, that **current_input_ref** should be replaced with. """ inputs = self.get_recipe_inputs() for (input_role_name, input_role) in inputs.items(): for item in input_role.get("items", []): if item.get("ref", None) == current_input_ref: item["ref"] = new_input_ref def replace_output(self, current_output_ref, new_output_ref): """ Replaces an output of this recipe by another. If the **current_output_ref** isn't part of the recipe's outputs, this method has no effect. :param string current_output_ref: a ref to an object in DSS, i.e. a dataset name or a managed folder id or a saved model id, that is currently output to the recipe :param string new_output_ref: a ref to an object in DSS, i.e. a dataset name or a managed folder id or a saved model id, that **current_output_ref** should be replaced with. """ outputs = self.get_recipe_outputs() for (output_role_name, output_role) in outputs.items(): for item in output_role.get("items", []): if item.get("ref", None) == current_output_ref: item["ref"] = new_output_ref def add_input(self, role, ref, partition_deps=None): """ Add an input to the recipe. For most recipes, there is only one role, named "main". Some few recipes have additional roles, like scoring recipes which have a "model" role. Check the roles known to the recipe with :meth:`get_recipe_inputs()`. :param string role: name of the role of the recipe in which to add **ref** as input :param string ref: a ref to an object in DSS, i.e. a dataset name or a managed folder id or a saved model id :param list partition_deps: if **ref** points to a partitioned object, a list of partition dependencies, one per dimension in the partitioning scheme """ if partition_deps is None: partition_deps = [] self._get_or_create_input_role(role)["items"].append({"ref": ref, "deps": partition_deps}) def add_output(self, role, ref, append_mode=False): """ Add an output to the recipe. For most recipes, there is only one role, named "main". Some few recipes have additional roles, like evaluation recipes which have a "metrics" role. Check the roles known to the recipe with :meth:`get_recipe_outputs()`. :param string role: name of the role of the recipe in which to add **ref** as input :param string ref: a ref to an object in DSS, i.e. a dataset name or a managed folder id or a saved model id :param list partition_deps: if **ref** points to a partitioned object, a list of partition dependencies, one per dimension in the partitioning scheme """ self._get_or_create_output_role(role)["items"].append({"ref": ref, "appendMode": append_mode}) def _get_or_create_input_role(self, role): inputs = self.get_recipe_inputs() if not role in inputs: role_obj = {"items": []} inputs[role] = role_obj return inputs[role] def _get_or_create_output_role(self, role): outputs = self.get_recipe_outputs() if not role in outputs: role_obj = {"items": []} outputs[role] = role_obj return outputs[role] def _get_flat_inputs(self): ret = [] for role_key, role_obj in self.get_recipe_inputs().items(): for item in role_obj["items"]: ret.append((role_key, item)) return ret def _get_flat_outputs(self): ret = [] for role_key, role_obj in self.get_recipe_outputs().items(): for item in role_obj["items"]: ret.append((role_key, item)) return ret def get_flat_input_refs(self): """ List all input refs of this recipe, regardless of the input role. :return: a list of refs, i.e. of dataset names or managed folder ids or saved model ids :rtype: list[string] """ ret = [] for role_key, role_obj in self.get_recipe_inputs().items(): for item in role_obj["items"]: ret.append(item["ref"]) return ret def get_flat_output_refs(self): """ List all output refs of this recipe, regardless of the input role. :return: a list of refs, i.e. of dataset names or managed folder ids or saved model ids :rtype: list[string] """ ret = [] for role_key, role_obj in self.get_recipe_outputs().items(): for item in role_obj["items"]: ret.append(item["ref"]) return ret class DSSRecipeDefinitionAndPayload(DSSRecipeSettings): """ Settings of a recipe. .. note:: Deprecated. Alias to :class:`DSSRecipeSettings`, use :meth:`DSSRecipe.get_settings()` instead. """ pass class RequiredSchemaUpdates(object): """ Handle on a set of required updates to the schema of the outputs of a recipe. .. important:: Do not instantiate directly, use :meth:`DSSRecipe.compute_schema_updates()` For example, changes can be new columns in the output of a Group recipe when new aggregates are activated in the recipe's settings. """ def __init__(self, recipe, data): self.recipe = recipe self.data = data self.drop_and_recreate = True self.synchronize_metastore = True def any_action_required(self): """ Whether there are changes at all. :rtype: boolean """ return self.data["totalIncompatibilities"] > 0 def apply(self): """ Apply the changes. All the updates found to be required are applied, for each of the recipe's outputs. """ results = [] for computable in self.data["computables"]: osu = { "computableType": computable["type"], # dirty "computableId": computable["type"] == "DATASET" and computable["datasetName"] or computable["id"], "newSchema": computable["newSchema"], "dropAndRecreate": self.drop_and_recreate, "synchronizeMetastore" : self.synchronize_metastore } results.append(self.recipe.client._perform_json("POST", "/projects/%s/recipes/%s/actions/updateOutputSchema" % (self.recipe.project_key, self.recipe.recipe_name), body=osu)) return results ##################################################### # Recipes creation infrastructure ##################################################### class DSSRecipeCreator(object): """ Helper to create new recipes. .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.new_recipe()` instead. """ def __init__(self, type, name, project): self.project = project self.recipe_proto = { "inputs" : {}, "outputs" : {}, "type" : type, "name" : name } self.creation_settings = { } def set_name(self, name): """ Set the name of the recipe-to-be-created. :param string name: a recipe name. Should only use alphanum letters and underscores. Cannot contain dots. """ self.recipe_proto["name"] = name def _build_ref(self, object_id, project_key=None): if project_key is not None and project_key != self.project.project_key: return project_key + '.' + object_id else: return object_id def _with_input(self, dataset_name, project_key=None, role="main"): role_obj = self.recipe_proto["inputs"].get(role, None) if role_obj is None: role_obj = { "items" : [] } self.recipe_proto["inputs"][role] = role_obj role_obj["items"].append({'ref':self._build_ref(dataset_name, project_key)}) return self def _with_output(self, dataset_name, append=False, role="main"): role_obj = self.recipe_proto["outputs"].get(role, None) if role_obj is None: role_obj = { "items" : [] } self.recipe_proto["outputs"][role] = role_obj role_obj["items"].append({'ref':self._build_ref(dataset_name, None), 'appendMode': append}) return self def _get_input_refs(self): ret = [] for role_key, role_obj in self.recipe_proto["inputs"].items(): for item in role_obj["items"]: ret.append(item["ref"]) return ret def with_input(self, input_id, project_key=None, role="main"): """ Add an existing object as input to the recipe-to-be-created. :param string input_id: name of the dataset, or identifier of the managed folder or identifier of the saved model :param string project_key: project containing the object, if different from the one where the recipe is created :param string role: the role of the recipe in which the input should be added. Most recipes only have one role named "main". """ return self._with_input(input_id, project_key, role) def with_output(self, output_id, append=False, role="main"): """ Add an existing object as output to the recipe-to-be-created. The output dataset must already exist. :param string output_id: name of the dataset, or identifier of the managed folder or identifier of the saved model :param boolean append: whether the recipe should append or overwrite the output when running (note: not available for all dataset types) :param string role: the role of the recipe in which the input should be added. Most recipes only have one role named "main". """ return self._with_output(output_id, append, role) def build(self): """ Create the recipe. .. note:: Deprecated. Alias to :meth:`create()` """ return self.create() def create(self): """ Creates the new recipe in the project, and return a handle to interact with it. :rtype: :class:`dataikuapi.dss.recipe.DSSRecipe` """ self._finish_creation_settings() return self.project.create_recipe(self.recipe_proto, self.creation_settings) def set_raw_mode(self): """ Activate raw creation mode. .. caution:: For advanced uses only. In this mode, the field "recipe_proto" of this recipe creator is used as-is to create the recipe, and if it exists, the value of creation_settings["rawPayload"] is used as the payload of the created recipe. No checks of existence or validity of the inputs or outputs are done, and no output is auto-created. """ self.creation_settings["rawCreation"] = True def _finish_creation_settings(self): pass class SingleOutputRecipeCreator(DSSRecipeCreator): """ Create a recipe that has a single output. .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.new_recipe()` instead. """ def __init__(self, type, name, project): DSSRecipeCreator.__init__(self, type, name, project) self.create_output_dataset = None self.output_dataset_settings = None self.create_output_folder = None self.output_folder_settings = None def with_existing_output(self, output_id, append=False): """ Add an existing object as output to the recipe-to-be-created. The output dataset must already exist. :param string output_id: name of the dataset, or identifier of the managed folder or identifier of the saved model :param boolean append: whether the recipe should append or overwrite the output when running (note: not available for all dataset types) """ assert self.create_output_dataset is None self.create_output_dataset = False self._with_output(output_id, append) return self def with_new_output(self, name, connection, type=None, format=None, override_sql_schema=None, partitioning_option_id=None, append=False, object_type='DATASET', overwrite=False, **kwargs): """ Create a new dataset or managed folder as output to the recipe-to-be-created. The dataset or managed folder is not created immediately, but when the recipe is created (ie in the create() method). Whether a dataset is created or a managed folder is created, depends on the recipe type. :param string name: name of the dataset or identifier of the managed folder :param string connection: name of the connection to create the dataset or managed folder on :param string type: sub-type of dataset or managed folder, for connections where the type could be ambiguous. Typically applies to SSH connections, where sub-types can be SCP or SFTP :param string format: name of a format preset relevant for the dataset type. Possible values are: CSV_ESCAPING_NOGZIP_FORHIVE, CSV_UNIX_GZIP, CSV_EXCEL_GZIP, CSV_EXCEL_GZIP_BIGQUERY, CSV_NOQUOTING_NOGZIP_FORPIG, PARQUET_HIVE, AVRO, ORC :param boolean override_sql_schema: schema to force dataset, for SQL dataset. If left empty, will be autodetected :param string partitioning_option_id: to copy the partitioning schema of an existing dataset 'foo', pass a value of 'copy:dataset:foo'. If unset, then the output will be non-partitioned :param boolean append: whether the recipe should append or overwrite the output when running (note: not available for all dataset types) :param string object_type: DATASET or MANAGED_FOLDER :param boolean overwrite: If the dataset being created already exists, overwrite it (and delete data) """ for k in kwargs: #for retrop comp if k == "connection_id": connection = kwargs.get("connection_id") elif k == "format_option_id": format = kwargs.get("format_option_id") elif k == "typeOptionId": type = kwargs.get("typeOptionId") elif k == "type_option_id": type = kwargs.get("type_option_id") else: raise Exception("Unknown argument '{}'".format(k)) if object_type == 'DATASET': assert self.create_output_dataset is None dataset = self.project.get_dataset(name) if overwrite and dataset.exists(): dataset.delete(drop_data=True) self.create_output_dataset = True self.output_dataset_settings = {'connectionId':connection,'typeOptionId':type,'specificSettings':{'formatOptionId':format, 'overrideSQLSchema':override_sql_schema},'partitioningOptionId':partitioning_option_id} self._with_output(name, append) elif object_type == 'MANAGED_FOLDER': assert self.create_output_folder is None self.create_output_folder = True self.output_folder_settings = {'connectionId':connection,'typeOptionId':type,'partitioningOptionId':partitioning_option_id} self._with_output(name, append) return self def with_output(self, output_id, append=False): """ Add an existing object as output to the recipe-to-be-created. .. note:: Alias of :meth:`with_existing_output()` """ return self.with_existing_output(output_id, append) def _finish_creation_settings(self): self.creation_settings['createOutputDataset'] = self.create_output_dataset self.creation_settings['outputDatasetSettings'] = self.output_dataset_settings self.creation_settings['createOutputFolder'] = self.create_output_folder self.creation_settings['outputFolderSettings'] = self.output_folder_settings class VirtualInputsSingleOutputRecipeCreator(SingleOutputRecipeCreator): """ Create a recipe that has a single output and several inputs. """ def __init__(self, type, name, project): SingleOutputRecipeCreator.__init__(self, type, name, project) self.virtual_inputs = [] def with_input(self, input_id, project_key=None): """ Add an existing object as input to the recipe-to-be-created. :param string input_id: name of the dataset :param string project_key: project containing the object, if different from the one where the recipe is created """ self.virtual_inputs.append(self._build_ref(input_id, project_key)) return self def _finish_creation_settings(self): super(VirtualInputsSingleOutputRecipeCreator, self)._finish_creation_settings() self.creation_settings['virtualInputs'] = self.virtual_inputs ##################################################### # Per-recipe-type classes: Visual recipes ##################################################### class GenerateFeaturesRecipeSettings(DSSRecipeSettings): """ Settings of a Generate features recipe. .. important:: Do not instantiate directly, use :meth:`DSSRecipe.get_settings()` """ pass class GenerateFeaturesRecipeCreator(VirtualInputsSingleOutputRecipeCreator): """ Create a generate features recipes .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.new_recipe()` instead. """ def __init__(self, name, project): VirtualInputsSingleOutputRecipeCreator.__init__(self, 'generate_features', name, project) class GroupingRecipeSettings(DSSRecipeSettings): """ Settings of a grouping recipe. .. important:: Do not instantiate directly, use :meth:`DSSRecipe.get_settings()` """ def clear_grouping_keys(self): """ Clear all grouping keys. """ self.obj_payload["keys"] = [] def add_grouping_key(self, column): """ Adds grouping on a column. :param string column: column to group on """ self.obj_payload["keys"].append({"column":column}) def set_global_count_enabled(self, enabled): """ Activate computing the count of records per group. :param boolean enabled: True if the global count should be activated """ self.obj_payload["globalCount"] = enabled def get_or_create_column_settings(self, column): """ Get a dict representing the aggregations to perform on a column. If the column has no aggregation on it yet, the dict is created and added to the settings. :param string column: name of the column to aggregate on :return: the settings of the aggregations on a particular column, as a dict. The name of the column to perform aggregates on is in a **column** field, and the aggregates are toggled on or off with boolean fields. :rtype: dict """ found = None for gv in self.obj_payload["values"]: if gv["column"] == column: found = gv break if found is None: found = {"column" : column} self.obj_payload["values"].append(found) return found def set_column_aggregations(self, column, type=None, min=False, max=False, count=False, count_distinct=False, sum=False,concat=False,stddev=False,avg=False): """ Set the basic aggregations on a column. .. note:: Not all aggregations may be possible. For example string-typed columns don't have a mean or standard deviation, and some SQL databases can't compute the exact standard deviation. The method returns a reference to the settings of the column, not a copy. Modifying the dict returned by the method, then calling :meth:`DSSRecipeSettings.save()` will commit the changes. Usage example: .. code-block:: # activate the concat aggregate on a column, and set optional parameters # pertaining to concatenation settings = recipe.get_settings() column_settings = settings.set_column_aggregations("my_column_name", concat=True) column_settings["concatDistinct"] = True column_settings["concatSeparator"] = ', ' settings.save() :param string column: The column name :param string type: The type of the column (as a DSS schema type name) :param boolean min: whether the min aggregate is computed :param boolean max: whether the max aggregate is computed :param boolean count: whether the count aggregate is computed :param boolean count_distinct: whether the count distinct aggregate is computed :param boolean sum: whether the sum aggregate is computed :param boolean concat: whether the concat aggregate is computed :param boolean avg: whether the mean aggregate is computed :param boolean stddev: whether the standard deviation aggregate is computed :return: the settings of the aggregations on a the column, as a dict. The name of the column is in a **column** field. :rtype: dict """ cs = self.get_or_create_column_settings(column) if type is not None: cs["type"] = type cs["min"] = min cs["max"] = max cs["count"] = count cs["countDistinct"] = count_distinct cs["sum"] = sum cs["concat"] = concat cs["stddev"] = stddev return cs class GroupingRecipeCreator(SingleOutputRecipeCreator): """ Create a Group recipe. .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.new_recipe()` instead. """ def __init__(self, name, project): SingleOutputRecipeCreator.__init__(self, 'grouping', name, project) self.group_key = None def with_group_key(self, group_key): """ Set a column as the first grouping key. Only a single grouping key may be set at recipe creation time. To add more grouping keys, get the recipe settings and use :meth:`GroupingRecipeSettings.add_grouping_key()`. To have no grouping keys at all, get the recipe settings and use :meth:`GroupingRecipeSettings.clear_grouping_keys()`. :param string group_key: name of a column in the input dataset :return: self :rtype: :class:`GroupingRecipeCreator` """ self.group_key = group_key return self def _finish_creation_settings(self): super(GroupingRecipeCreator, self)._finish_creation_settings() if self.group_key is not None: self.creation_settings['groupKey'] = self.group_key class WindowRecipeSettings(DSSRecipeSettings): """ Settings of a Window recipe. .. important:: Do not instantiate directly, use :meth:`DSSRecipe.get_settings()` """ pass # TODO: Write helpers for window class WindowRecipeCreator(SingleOutputRecipeCreator): """ Create a Window recipe .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.new_recipe()` instead. """ def __init__(self, name, project): SingleOutputRecipeCreator.__init__(self, 'window', name, project) class SyncRecipeSettings(DSSRecipeSettings): """ Settings of a Sync recipe. .. important:: Do not instantiate directly, use :meth:`DSSRecipe.get_settings()` """ pass # TODO: Write helpers for sync class SyncRecipeCreator(SingleOutputRecipeCreator): """ Create a Sync recipe .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.new_recipe()` instead. """ def __init__(self, name, project): SingleOutputRecipeCreator.__init__(self, 'sync', name, project) class ContinuousSyncRecipeCreator(SingleOutputRecipeCreator): """ Create a continuous Sync recipe .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.new_recipe()` instead. """ def __init__(self, name, project): SingleOutputRecipeCreator.__init__(self, 'csync', name, project) class SortRecipeSettings(DSSRecipeSettings): """ Settings of a Sort recipe. .. important:: Do not instantiate directly, use :meth:`DSSRecipe.get_settings()` """ pass # TODO: Write helpers for sort class SortRecipeCreator(SingleOutputRecipeCreator): """ Create a Sort recipe .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.new_recipe()` instead. """ def __init__(self, name, project): SingleOutputRecipeCreator.__init__(self, 'sort', name, project) class PivotRecipeSettings(DSSRecipeSettings): """ Settings of a Pivot recipe. .. important:: Do not instantiate directly, use :meth:`DSSRecipe.get_settings()` """ pass # TODO: Write helpers for sort class PivotRecipeCreator(SingleOutputRecipeCreator): """ Create a Pivot recipe .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.new_recipe()` instead. """ def __init__(self, name, project): SingleOutputRecipeCreator.__init__(self, 'pivot', name, project) class TopNRecipeSettings(DSSRecipeSettings): """ Settings of a TopN recipe. .. important:: Do not instantiate directly, use :meth:`DSSRecipe.get_settings()` """ pass # TODO: Write helpers for topn class TopNRecipeCreator(DSSRecipeCreator): """ Create a TopN recipe .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.new_recipe()` instead. """ def __init__(self, name, project): DSSRecipeCreator.__init__(self, 'topn', name, project) class DistinctRecipeSettings(DSSRecipeSettings): """ Settings of a Distinct recipe. .. important:: Do not instantiate directly, use :meth:`DSSRecipe.get_settings()` """ pass # TODO: Write helpers for distinct class DistinctRecipeCreator(SingleOutputRecipeCreator): """ Create a Distinct recipe .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.new_recipe()` instead. """ def __init__(self, name, project): SingleOutputRecipeCreator.__init__(self, 'distinct', name, project) class PrepareRecipeSettings(DSSRecipeSettings): """ Settings of a Prepare recipe. .. important:: Do not instantiate directly, use :meth:`DSSRecipe.get_settings()` """ pass @property def raw_steps(self): """ Get the list of the steps of this prepare recipe. This method returns a reference to the list of steps, not a copy. Modifying the list then calling :meth:`DSSRecipeSettings.save()` commits the changes. :return: list of steps, each step as a dict. The precise settings for each step are not documented, but each dict has at least fields: * **metaType** : one of PROCESSOR or GROUP. If GROUP, there there is a field **steps** with a sub-list of steps. * **type** : type of the step, for example FillEmptyWithValue or ColumnRenamer (there are many types of steps) * **params** : dict of the step's own parameters. Each step type has its own parameters. * **disabled** : whether the step is disabled * **name** : label of the step :rtype: list[dict] """ return self.obj_payload["steps"] def add_processor_step(self, type, params): """ Add a step in the script. :param string type: type of the step, for example FillEmptyWithValue or ColumnRenamer (there are many types of steps) :param dict params: dict of the step's own parameters. Each step type has its own parameters. """ step = { "metaType": "PROCESSOR", "type": type, "params": params } self.raw_steps.append(step) class PrepareRecipeCreator(SingleOutputRecipeCreator): """ Create a Prepare recipe .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.new_recipe()` instead. """ def __init__(self, name, project): SingleOutputRecipeCreator.__init__(self, 'shaker', name, project) class JoinRecipeSettings(DSSRecipeSettings): """ Settings of a join recipe. .. important:: Do not instantiate directly, use :meth:`DSSRecipe.get_settings()` In order to enable self-joins, join recipes are based on a concept of "virtual inputs". Every join, computed pre-join column, pre-join filter, ... is based on one virtual input, and each virtual input references an input of the recipe, by index For example, if a recipe has inputs A and B and declares two joins: - A->B - A->A (based on a computed column) There are 3 virtual inputs: * 0: points to recipe input 0 (i.e. dataset A) * 1: points to recipe input 1 (i.e. dataset B) * 2: points to recipe input 0 (i.e. dataset A) and includes the computed column * The first join is between virtual inputs 0 and 1 * The second join is between virtual inputs 0 and 2 """ pass # TODO: Write helpers for join @property def raw_virtual_inputs(self): """ Get the list of virtual inputs. This method returns a reference to the list of inputs, not a copy. Modifying the list then calling :meth:`DSSRecipeSettings.save()` commits the changes. :return: a list of virtual inputs, each one a dict. The field **index** holds the index of the dataset of this virtual input in the recipe's list of inputs. Pre-filter, computed columns and column selection properties (if applicable) are defined in each virtual input. :rtype: list[dict] """ return self.obj_payload["virtualInputs"] @property def raw_joins(self): """ Get raw list of joins. This method returns a reference to the list of joins, not a copy. Modifying the list then calling :meth:`DSSRecipeSettings.save()` commits the changes. :return: list of the join definitions, each as a dict. The **table1** and **table2** fields give the indices of the virtual inputs on the left side and right side respectively. :rtype: list[dict] """ return self.obj_payload["joins"] def add_virtual_input(self, input_dataset_index): """ Add a virtual input pointing to the specified input dataset of the recipe. :param int input_dataset_index: index of the dataset in the list of input_dataset_index """ self.raw_virtual_inputs.append({"index": input_dataset_index}) def add_pre_join_computed_column(self, virtual_input_index, computed_column): """ Add a computed column to a virtual input. You can use :meth:`dataikuapi.dss.utils.DSSComputedColumn.formula()` to build the computed_column object. :param int input_dataset_index: index of the dataset in the list of input_dataset_index :param dict computed_column: a computed column definition, as a dict of: * **mode** : type of expression used to define the computations. One of GREL or SQL. * **name** : name of the column generated * **type** : name of a DSS type for the computed column * **expr** : if **mode** is CUSTOM, a formula in DSS `formula language `_ . If **mode** is SQL, a SQL expression. """ self.raw_virtual_inputs[virtual_input_index]["computedColumns"].append(computed_column) def add_join(self, join_type="LEFT", input1=0, input2=1): """ Add a join between two virtual inputs. The join is initialized with no condition. Use :meth:`add_condition_to_join()` on the return value to add a join condition (for example column equality) to the join. :return: the newly added join as a dict (see :meth:`raw_joins()`) :rtype: dict """ jp = self.obj_payload if not "joins" in jp: jp["joins"] = [] join = { "conditionsMode": "AND", "on": [], "table1": input1, "table2": input2, "type": join_type } jp["joins"].append(join) return join @staticmethod def add_condition_to_join(join, type="EQ", column1=None, column2=None): """ Add a condition to a join. :param dict join: definition of a join :param string type: type of join condition. Possible values are EQ, LTE, LT, GTE, GT, NE, WITHIN_RANGE, K_NEAREST, K_NEAREST_INFERIOR, CONTAINS, STARTS_WITH :param string column1: name of left-side column :param string column2: name of right-side column """ cond = { "type": type, "column1": {"name": column1, "table": join["table1"]}, "column2": {"name": column2, "table": join["table2"]}, } join["on"].append(cond) return cond def add_post_join_computed_column(self, computed_column): """ Add a post-join computed column. Use :class:`dataikuapi.dss.utils.DSSComputedColumn` to build the computed_column object. .. note:: The columns accessible to the expression of the computed column are those selected in the different joins, in their "output" form. For example if a virtual inputs 0 and 1 are joined, and column "bar" of the first input is selected with a prefix of "foo", then the computed column can use "foobar" but not "bar". :param dict computed_column: a computed column definition, as a dict of: * **mode** : type of expression used to define the computations. One of GREL or SQL. * **name** : name of the column generated * **type** : name of a DSS type for the computed column * **expr** : if **mode** is CUSTOM, a formula in DSS `formula language `_ . If **mode** is SQL, a SQL expression. """ self.obj_payload["computedColumns"].append(computed_column) def set_post_filter(self, postfilter): """ Add a post filter on the join. Use the methods on :class:`dataikuapi.dss.utils.DSSFilter` to build filter definition. :param dict postfilter: definition of a filter, as a dict of: * **distinct** : whether the records in the output should be deduplicated * **enabled** : whether filtering is enabled * **uiData** : settings of the filter, if **enabled** is True, as a dict of: * **mode** : type of filter. Possible values: CUSTOM, SQL, '&&' (boolean AND of conditions) and '||' (boolean OR of conditions) * **conditions** : if mode is '&&' or '||', then a list of the actual filter conditions, each one a dict * **expression** : if **uiData.mode** is CUSTOM, a formula in DSS `formula language `_ . If **uiData.mode** is SQL, a SQL expression. """ self.obj_payload["postFilter"] = postfilter def set_unmatched_output(self, ref, side='right', append_mode=False): """ Adds an unmatched join output :param str ref: name of the dataset :param str side: side of the unmatched output, 'right' or 'left'. :param bool append_mode: whether the recipe should append or overwrite the output when running """ if side not in ['right', 'left']: raise ValueError("Unknown side : " + side) if len(self.obj_payload["joins"]) != 1: raise DataikuException("Unmatched output can only be set on recipe that have a single join") join_type = self.obj_payload["joins"][0]["type"] if join_type not in ["RIGHT", "LEFT", "INNER"] or (side == 'left' and join_type == "LEFT") or (side == 'right' and join_type == "RIGHT"): raise DataikuException("No unmatched rows for " + side + " side with join type " + join_type) self._get_or_create_output_role("unmatchedRight" if side == 'right' else "unmatchedLeft")["items"] = [{"ref": ref, "appendMode": append_mode}] class JoinRecipeCreator(VirtualInputsSingleOutputRecipeCreator): """ Create a Join recipe. The recipe is created with default joins guessed by matching column names in the inputs. .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.new_recipe()` instead. """ def __init__(self, name, project): VirtualInputsSingleOutputRecipeCreator.__init__(self, 'join', name, project) class FuzzyJoinRecipeCreator(VirtualInputsSingleOutputRecipeCreator): """ Create a FuzzyJoin recipe The recipe is created with default joins guessed by matching column names in the inputs. .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.new_recipe()` instead. """ def __init__(self, name, project): VirtualInputsSingleOutputRecipeCreator.__init__(self, 'fuzzyjoin', name, project) class GeoJoinRecipeCreator(VirtualInputsSingleOutputRecipeCreator): """ Create a GeoJoin recipe The recipe is created with default joins guessed by matching column names in the inputs. .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.new_recipe()` instead. """ def __init__(self, name, project): VirtualInputsSingleOutputRecipeCreator.__init__(self, 'geojoin', name, project) class StackRecipeSettings(DSSRecipeSettings): """ Settings of a stack recipe. .. important:: Do not instantiate directly, use :meth:`DSSRecipe.get_settings()` """ pass # TODO: Write helpers for stack class StackRecipeCreator(VirtualInputsSingleOutputRecipeCreator): """ Create a Stack recipe .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.new_recipe()` instead. """ def __init__(self, name, project): VirtualInputsSingleOutputRecipeCreator.__init__(self, 'vstack', name, project) class SamplingRecipeSettings(DSSRecipeSettings): """ Settings of a sampling recipe. .. important:: Do not instantiate directly, use :meth:`DSSRecipe.get_settings()` """ pass # TODO: Write helpers for sampling class SamplingRecipeCreator(SingleOutputRecipeCreator): """ Create a Sample/Filter recipe .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.new_recipe()` instead. """ def __init__(self, name, project): SingleOutputRecipeCreator.__init__(self, 'sampling', name, project) class SplitRecipeSettings(DSSRecipeSettings): """ Settings of a split recipe. .. important:: Do not instantiate directly, use :meth:`DSSRecipe.get_settings()` """ pass # TODO: Write helpers for split class SplitRecipeCreator(DSSRecipeCreator): """ Create a Split recipe .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.new_recipe()` instead. """ def __init__(self, name, project): DSSRecipeCreator.__init__(self, "split", name, project) def _finish_creation_settings(self): pass class DownloadRecipeSettings(DSSRecipeSettings): """ Settings of a download recipe. .. important:: Do not instantiate directly, use :meth:`DSSRecipe.get_settings()` """ pass # TODO: Write helpers for download class DownloadRecipeCreator(SingleOutputRecipeCreator): """ Create a Download recipe .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.new_recipe()` instead. """ def __init__(self, name, project): SingleOutputRecipeCreator.__init__(self, 'download', name, project) ##################################################### # Per-recipe-type classes: Code recipes ##################################################### class CodeRecipeSettings(DSSRecipeSettings): """ Settings of a code recipe. .. important:: Do not instantiate directly, use :meth:`DSSRecipe.get_settings()` """ def get_code(self): """ Get the code of the recipe. :rtype: string """ self._payload_to_str() return self._str_payload def set_code(self, code): """ Update the code of the recipe. :param string code: the new code """ self.set_payload(code) def get_code_env_settings(self): """ Get the code env settings for this recipe. :return: settings to select the code env used by the recipe, as a dict of: * **envMode** : one of USE_BUILTIN_MODE, INHERIT (inherit from project settings and/or instance settings), EXPLICIT_ENV * **envName** : if **envMode** is EXPLICIT_ENV, the name of the code env to use :rtype: dict """ rp = self.get_recipe_params() if not "envSelection" in rp: raise ValueError("This recipe kind does not seem to take a code env selection") return rp["envSelection"] def set_code_env(self, code_env=None, inherit=False, use_builtin=False): """ Set which code env this recipe uses. Exactly one of `code_env`, `inherit` or `use_builtin` must be passed. :param string code_env: name of a code env :param boolean inherit: if True, use the project's default code env :param boolean use_builtin: if true, use the builtin code env """ rp = self.get_recipe_params() if not "envSelection" in rp: raise ValueError("This recipe kind does not seem to take a code env selection") if code_env is not None: rp["envSelection"] = {"envMode": "EXPLICIT_ENV", "envName": code_env} elif inherit: rp["envSelection"] = {"envMode": "INHERIT"} elif use_builtin: rp["envSelection"] = {"envMode": "USE_BUILTIN_MODE"} else: raise ValueError("No env setting selected") class CodeRecipeCreator(DSSRecipeCreator): """ Create a recipe running a script. .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.new_recipe()` """ def __init__(self, name, type, project): DSSRecipeCreator.__init__(self, type, name, project) self.script = None def with_script(self, script): """ Set the code of the recipe-to-be-created. :param string script: code of the recipe """ self.script = script return self def with_new_output_dataset(self, name, connection, type=None, format=None, copy_partitioning_from="FIRST_INPUT", append=False, overwrite=False, **kwargs): """ Create a new managed dataset as output to the recipe-to-be-created. The dataset is created immediately. :param string name: name of the dataset :param string connection: name of the connection to create the dataset on :param string type: sub-type of dataset or managed folder, for connections where the type could be ambiguous. Typically applies to SSH connections, where sub-types can be SCP or SFTP :param string format: name of a format preset relevant for the dataset type. Possible values are: CSV_ESCAPING_NOGZIP_FORHIVE, CSV_UNIX_GZIP, CSV_EXCEL_GZIP, CSV_EXCEL_GZIP_BIGQUERY, CSV_NOQUOTING_NOGZIP_FORPIG, PARQUET_HIVE, AVRO, ORC :param string partitioning_option_id: to copy the partitioning schema of an existing dataset 'foo', pass a value of 'copy:dataset:foo'. If unset, then the output will be non-partitioned :param boolean append: whether the recipe should append or overwrite the output when running (note: not available for all dataset types) :param boolean overwrite: If the dataset being created already exists, overwrite it (and delete data) """ for k in kwargs: # for backward compat if k == "format_option_id": format = kwargs.get("format_option_id") elif k == "typeOptionId": type = kwargs.get("typeOptionId") elif k == "type_option_id": type = kwargs.get("type_option_id") else: raise Exception("Unknown argument '{}'".format(k)) ch = self.project.new_managed_dataset_creation_helper(name) ch.with_store_into(connection, type_option_id=type, format_option_id=format) # FIXME: can't manage input folder if copy_partitioning_from == "FIRST_INPUT": inputs = self._get_input_refs() if len(inputs) == 0: logging.warn("No input declared yet, can't copy partitioning from first input") else: self.creation_settings["partitioningOptionId"] = "copy:dataset:%s" % (inputs[0]) elif copy_partitioning_from is not None: self.creation_settings["partitioningOptionId"] = "copy:%s" % copy_partitioning_from ch.create(overwrite=overwrite) self.with_output(name, append=append) return self def with_new_output_streaming_endpoint(self, name, connection, format=None, overwrite=False, **kwargs): """ Create a new managed streaming endpoint as output to the recipe-to-be-created. The streaming endpoint is created immediately. :param str name: name of the streaming endpoint to create :param str connection: name of the connection to create the streaming endpoint on :param str format: name of a format preset relevant for the streaming endpoint type. Possible values are: json, avro, single (kafka endpoints) or json, string (SQS endpoints). If None, uses the default :param overwrite: If the streaming endpoint being created already exists, overwrite it """ for k in kwargs: # for backward compat if k == "format_option_id": format = kwargs.get("format_option_id") else: raise Exception("Unknown argument '{}'".format(k)) ch = self.project.new_managed_streaming_endpoint(name) ch.with_store_into(connection, format_option_id=format) ch.create(overwrite=overwrite) self.with_output(name, append=False) return self def _finish_creation_settings(self): super(CodeRecipeCreator, self)._finish_creation_settings() self.creation_settings['script'] = self.script class PythonRecipeCreator(CodeRecipeCreator): """ Create a Python recipe. .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.new_recipe()` instead. A Python recipe can be defined either by its complete code, like a normal Python recipe, or by a function signature. """ def __init__(self, name, project): DSSRecipeCreator.__init__(self, "python", name, project) _DEFAULT_RECIPE_CODE_TMPL = """ # This code is autogenerated by PythonRecipeCreator function mode import dataiku, dataiku.recipe, json from {module_name} import {fname} input_datasets = dataiku.recipe.get_inputs_as_datasets() output_datasets = dataiku.recipe.get_outputs_as_datasets() params = json.loads('{params_json}') logging.info("Reading %d input datasets as dataframes" % len(input_datasets)) input_dataframes = [ds.get_dataframe() for ds in input_datasets] logging.info("Calling user function {fname}") function_input = input_dataframes if len(input_dataframes) > 1 else input_dataframes[0] output_dataframes = {fname}(function_input, **params) if not isinstance(output_dataframes, list): output_dataframes = [output_dataframes] if not len(output_dataframes) == len(output_datasets): raise Exception("Code function {fname}() returned %d dataframes but recipe expects %d output datasets", \\ (len(output_dataframes), len(output_datasets))) output = list(zip(output_datasets, output_dataframes)) for ds, df in output: logging.info("Writing function result to dataset %s" % ds.name) ds.write_with_schema(df) """ def with_function_name(self, module_name, function_name, custom_template=None, **function_args): """ Define this recipe as being a functional recipe calling a function. With the default template, the function must take as arguments: * A list of dataframes corresponding to the dataframes of the input datasets. If there is only one input, then a single dataframe * Optional named arguments corresponding to arguments passed to the creator as kwargs The function should then return a list of dataframes, one per recipe output. If there is a single output, it is possible to return a single dataframe rather than a list. :param string module_name: name of the module where the function is defined :param string function_name: name of the function :param kwargs function_args: additional parameters to the function. :param string custom_template: template to use to create the code of the recipe. The template is formatted with '{fname}' (function name), '{module_name}' (module name) and '{params_json}' (JSON representation of **function_args**) """ script_tmpl = PythonRecipeCreator._DEFAULT_RECIPE_CODE_TMPL if custom_template is None else custom_template if function_args is None: function_args = {} code = script_tmpl.format(module_name=module_name, fname=function_name, params_json = json.dumps(function_args)) self.with_script(code) return self def with_function(self, fn, custom_template=None, **function_args): """ Define this recipe as being a functional recipe calling a function. With the default template, the function must take as arguments: * A list of dataframes corresponding to the dataframes of the input datasets. If there is only one input, then a single dataframe * Optional named arguments corresponding to arguments passed to the creator as kwargs The function should then return a list of dataframes, one per recipe output. If there is a single output, it is possible to return a single dataframe rather than a list. :param string fn: function to call :param kwargs function_args: additional parameters to the function. :param string custom_template: template to use to create the code of the recipe. The template is formatted with '{fname}' (function name), '{module_name}' (module name) and '{params_json}' (JSON representation of **function_args**) """ import inspect #TODO: add in documentation that relative imports wont work module_name = inspect.getmodule(fn).__name__ fname = fn.__name__ return self.with_function_name(module_name, fname, custom_template, **function_args) class SQLQueryRecipeCreator(SingleOutputRecipeCreator): """ Create a SQL query recipe. .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.new_recipe()` instead. """ def __init__(self, name, project): SingleOutputRecipeCreator.__init__(self, 'sql_query', name, project) ##################################################### # Per-recipe-type classes: ML recipes ##################################################### class PredictionScoringRecipeCreator(SingleOutputRecipeCreator): """ Create a new Prediction scoring recipe. .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.new_recipe()` instead. Usage example: .. code-block:: python # Create a new prediction scoring recipe outputing to a new dataset project = client.get_project("MYPROJECT") builder = project.new_recipe("prediction_scoring", "my_scoring_recipe") builder.with_input_model("saved_model_id") builder.with_input("dataset_to_score") builder.with_new_output("my_output_dataset", "myconnection") # Or for a filesystem output connection # builder.with_new_output("my_output_dataset, "filesystem_managed", format="CSV_EXCEL_GZIP") new_recipe = builder.build() """ def __init__(self, name, project): SingleOutputRecipeCreator.__init__(self, 'prediction_scoring', name, project) def with_input_model(self, model_id): """ Set the input model. :param string model_id: identifier of a saved model """ return self._with_input(model_id, self.project.project_key, "model") class EvaluationRecipeCreator(DSSRecipeCreator): """ Create a new Evaluate recipe. .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.new_recipe()` instead. Usage example: .. code-block:: python # Create a new evaluation recipe outputing to a new dataset, to a metrics dataset and/or to a model evaluation store project = client.get_project("MYPROJECT") builder = project.new_recipe("evaluation") builder.with_input_model(saved_model_id) builder.with_input("dataset_to_evaluate") builder.with_output("output_scored") builder.with_output_metrics("output_metrics") builder.with_output_evaluation_store(evaluation_store_id) new_recipe = builder.build() # Access the settings er_settings = new_recipe.get_settings() payload = er_settings.obj_payload # Change the settings payload['dontComputePerformance'] = True payload['outputProbabilities'] = False payload['metrics'] = ["precision", "recall", "auc", "f1", "costMatrixGain"] # Manage evaluation labels payload['labels'] = [dict(key="label_1", value="value_1"), dict(key="label_2", value="value_2")] # Save the settings and run the recipe er_settings.save() new_recipe.run() Outputs must exist. They can be created using the following: .. code-block:: python builder = project.new_managed_dataset("output_scored") builder.with_store_into(connection) dataset = builder.create() builder = project.new_managed_dataset("output_scored") builder.with_store_into(connection) dataset = builder.create() evaluation_store_id = project.create_model_evaluation_store("output_model_evaluation").mes_id """ def __init__(self, name, project): DSSRecipeCreator.__init__(self, 'evaluation', name, project) def with_input_model(self, model_id): """ Set the input model. :param string model_id: identifier of a saved model """ return self._with_input(model_id, self.project.project_key, "model") def with_output(self, output_id): """ Set the output dataset containing the scored input. :param string output_id: name of the dataset, or identifier of the managed folder or identifier of the saved model """ return self._with_output(output_id, role="main") def with_output_metrics(self, name): """ Set the output dataset containing the metrics. :param string name: name of an existing dataset """ return self._with_output(name, role="metrics") def with_output_evaluation_store(self, mes_id): """ Set the output model evaluation store. :param string mes_id: identifier of a model evaluation store """ return self._with_output(mes_id, role="evaluationStore") class LLMEvaluationRecipeCreator(DSSRecipeCreator): """ Create a new LLM Evaluate recipe. .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.new_recipe()` instead. Usage example: .. code-block:: python # Create a new evaluation recipe outputing to a new dataset, to a metrics dataset and/or to a model evaluation store project = client.get_project("MYPROJECT") builder = project.new_recipe("nlp_llm_evaluation", "myrecipe") builder.with_input("dataset_to_evaluate") builder.with_output("output_scored") builder.with_output_metrics("output_metrics") builder.with_output_evaluation_store(evaluation_store_id) new_recipe = builder.build() # Access the settings recipe_settings = new_recipe.get_settings() payload = recipe_settings.obj_payload # Change the settings payload["taskType"] = "QUESTION_ANSWERING" payload["inputColumnName"] = "question" payload["outputColumnName"] = "answer" payload["groundTruthColumnName"] = "ground_truth" payload["contextColumnName"] = "context" payload["completionLMMId"] = payload["embeddingLLMId"] = payload["envSelection"] = {"envName": , "envMode": "EXPLICIT_ENV"} payload["metrics"] = ["answerRelevancy", "faithfulness"] # Manage evaluation labels payload["labels"] = [dict(key="label_1", value="value_1"), dict(key="label_2", value="value_2")] # Save the settings, update the schema and run the recipe recipe_settings.save() new_recipe.compute_schema_updates().apply() new_recipe.run() Outputs must exist. They can be created using the following: .. code-block:: python builder = project.new_managed_dataset("output") builder.with_store_into(connection) dataset = builder.create() builder = project.new_managed_dataset("metrics") builder.with_store_into(connection) dataset = builder.create() evaluation_store_id = project.create_model_evaluation_store("mes").mes_id """ def __init__(self, name, project): DSSRecipeCreator.__init__(self, 'nlp_llm_evaluation', name, project) def with_output(self, output_id): """ Set the output dataset containing the evaluation dataset scored row by row :param string output_id: name of the dataset """ return self._with_output(output_id, role="main") def with_output_metrics(self, name): """ Set the output dataset containing the metrics :param string name: name of an existing dataset """ return self._with_output(name, role="metrics") def with_output_evaluation_store(self, mes_id): """ Set the output evaluation store :param string mes_id: identifier of a model evaluation store """ return self._with_output(mes_id, role="evaluationStore") class StandaloneEvaluationRecipeCreator(DSSRecipeCreator): """ Create a new Standalone Evaluate recipe. .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.new_recipe()` instead. Usage example: .. code-block:: python # Create a new standalone evaluation of a scored dataset project = client.get_project("MYPROJECT") builder = project.new_recipe("standalone_evaluation") builder.with_input("scored_dataset_to_evaluate") builder.with_output_evaluation_store(evaluation_store_id) # Add a reference dataset (optional) to compute data drift builder.with_reference_dataset("reference_dataset") # Finish creation of the recipe new_recipe = builder.create() # Modify the model parameters in the SER settings ser_settings = new_recipe.get_settings() payload = ser_settings.obj_payload payload['predictionType'] = "BINARY_CLASSIFICATION" payload['targetVariable'] = "Survived" payload['predictionVariable'] = "prediction" payload['isProbaAware'] = True payload['dontComputePerformance'] = False # For a classification model with probabilities, the 'probas' section can be filled with the mapping of the class and the probability column # e.g. for a binary classification model with 2 columns: proba_0 and proba_1 class_0 = dict(key=0, value="proba_0") class_1 = dict(key=1, value="proba_1") payload['probas'] = [class_0, class_1] # Change the 'features' settings for this standalone evaluation # e.g. reject the features that you do not want to use in the evaluation feature_passengerid = dict(name="Passenger_Id", role="REJECT", type="TEXT") feature_ticket = dict(name="Ticket", role="REJECT", type="TEXT") feature_cabin = dict(name="Cabin", role="REJECT", type="TEXT") payload['features'] = [feature_passengerid, feature_ticket, feature_cabin] # To set the cost matrix properly, access the 'metricParams' section of the payload and set the cost matrix weights: payload['metricParams'] = dict(costMatrixWeights=dict(tpGain=0.4, fpGain=-1.0, tnGain=0.2, fnGain=-0.5)) # Save the recipe and run the recipe # Note that with this method, all the settings that were not explicitly set are instead set to their default value. ser_settings.save() new_recipe.run() Output model evaluation store must exist. It can be created using the following: .. code-block:: python evaluation_store_id = project.create_model_evaluation_store("output_model_evaluation").mes_id """ def __init__(self, name, project): DSSRecipeCreator.__init__(self, 'standalone_evaluation', name, project) def with_output_evaluation_store(self, mes_id): """ Set the output model evaluation store. :param string mes_id: identifier of a model evaluation store """ return self._with_output(mes_id, role="main") def with_reference_dataset(self, dataset_name): """ Set the dataset to use as a reference in data drift computation. :param string dataset_name: name of a dataset """ return self._with_input(dataset_name, self.project.project_key, role="reference") class ClusteringScoringRecipeCreator(SingleOutputRecipeCreator): """ Create a new Clustering scoring recipe,. .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.new_recipe()` instead. Usage example: .. code-block:: python # Create a new prediction scoring recipe outputing to a new dataset project = client.get_project("MYPROJECT") builder = project.new_recipe("clustering_scoring", "my_scoring_recipe") builder.with_input_model("saved_model_id") builder.with_input("dataset_to_score") builder.with_new_output("my_output_dataset", "myconnection") # Or for a filesystem output connection # builder.with_new_output("my_output_dataset, "filesystem_managed", format="CSV_EXCEL_GZIP") new_recipe = builder.build() """ def __init__(self, name, project): SingleOutputRecipeCreator.__init__(self, 'clustering_scoring', name, project) def with_input_model(self, model_id): """ Set the input model. :param string model_id: identifier of a saved model """ return self._with_input(model_id, self.project.project_key, "model") dataiku-api-client-python-13.2.2/dataikuapi/dss/savedmodel.py000066400000000000000000001402121471211575100241710ustar00rootroot00000000000000import os import shutil import tempfile from dataikuapi.dss.utils import DSSDatasetSelectionBuilder from .discussion import DSSObjectDiscussions from .managedfolder import DSSManagedFolder from .metrics import ComputedMetrics from .ml import DSSMLTask from .ml import DSSTrainedClusteringModelDetails from .ml import DSSTrainedPredictionModelDetails from ..utils import _make_zipfile, dku_basestring_type try: basestring except NameError: basestring = str class DatabricksRepositoryContextManager(object): def __init__(self, connection_info, use_unity_catalog): self.connection_info = connection_info self.use_unity_catalog = use_unity_catalog def __enter__(self): self._setup_mlflow_for_databricks_registry() def __exit__(self, exc_type, exc_val, exc_tb): self._restore_previous_mlflow_setup() def _setup_mlflow_for_databricks_registry(self): params = self.connection_info.get("params") auth_type = params.get("authType") if auth_type == "OAUTH2_APP": token = self.connection_info.get("resolvedOAuth2Credential").get("accessToken") elif auth_type == "PERSONAL_ACCESS_TOKEN": token = params.get("personalAccessToken") else: raise Exception("Unhandled auth method: " + auth_type) databricks_host = "https://" if params.get("ssl") else "http://" host = params.get("host") port = params.get("port") if not host: raise Exception("Host undefined for Databricks connection") if not port: raise Exception("Port undefined for Databricks connection") databricks_host += host + ":" + str(port) http_path = params.get("httpPath") if http_path: if http_path.endswith("/"): http_path = http_path[:-1] if not http_path.startswith("/"): http_path = "/" + http_path databricks_host += http_path self.previous_tracking_uri = os.environ.get("MLFLOW_TRACKING_URI", None) self.previous_databricks_host = os.environ.get("DATABRICKS_HOST", None) self.previous_databricks_token = os.environ.get("DATABRICKS_TOKEN", None) proxy_as_string = self.connection_info.get("proxySettingsAsString") if proxy_as_string: self.previous_http_proxy = os.environ.get("HTTP_PROXY", None) self.previous_https_proxy = os.environ.get("HTTPS_PROXY", None) os.environ["http_proxy"] = "http://" + proxy_as_string os.environ["https_proxy"] = "http://" + proxy_as_string else: self.previous_http_proxy = None self.previous_https_proxy = None import mlflow self.previous_registry_uri = mlflow.get_registry_uri() if self.use_unity_catalog: mlflow.set_registry_uri("databricks-uc") else: mlflow.set_registry_uri(None) os.environ["MLFLOW_TRACKING_URI"] = "databricks" os.environ["DATABRICKS_HOST"] = databricks_host os.environ["DATABRICKS_TOKEN"] = token def _restore_previous_mlflow_setup(self): if self.previous_tracking_uri: os.environ["MLFLOW_TRACKING_URI"] = self.previous_tracking_uri else: os.environ.pop("MLFLOW_TRACKING_URI") if self.previous_databricks_host: os.environ["DATABRICKS_HOST"] = self.previous_databricks_host else: os.environ.pop("DATABRICKS_HOST") if self.previous_databricks_token: os.environ["DATABRICKS_TOKEN"] = self.previous_databricks_token else: os.environ.pop("DATABRICKS_TOKEN") import mlflow mlflow.set_registry_uri(self.previous_registry_uri) if self.previous_http_proxy: os.environ["http_proxy"] = self.previous_http_proxy if self.previous_https_proxy: os.environ["https_proxy"] = self.previous_https_proxy class DSSSavedModel(object): """ Handle to interact with a saved model on the DSS instance. .. important:: Do not create this class directly, instead use :meth:`dataikuapi.dss.DSSProject.get_saved_model` :param client: an api client to connect to the DSS backend :type client: :class:`dataikuapi.dssclient.DSSClient` :param str project_key: identifier of the project containing the model :param str sm_id: identifier of the saved model """ def __init__(self, client, project_key, sm_id): self.client = client self.project = client.get_project(project_key) self.project_key = project_key self.sm_id = sm_id @property def id(self): """ Returns the identifier of the saved model :rtype: str """ return self.sm_id def get_settings(self): """ Returns the settings of this saved model. :return: settings of this saved model :rtype: :class:`dataikuapi.dss.savedmodel.DSSSavedModelSettings` """ data = self.client._perform_json( "GET", "/projects/%s/savedmodels/%s" % (self.project_key, self.sm_id)) return DSSSavedModelSettings(self, data) ######################################################## # Versions ######################################################## def list_versions(self): """ Gets the versions of this saved model. This returns each version as a dict of object. Each object contains at least an "id" parameter, which can be passed to :meth:`get_metric_values`, :meth:`get_version_details` and :meth:`set_active_version`. :return: The list of the versions :rtype: list[dict] """ return self.client._perform_json( "GET", "/projects/%s/savedmodels/%s/versions" % (self.project_key, self.sm_id)) def get_active_version(self): """ Gets the active version of this saved model. The returned dict contains at least an "id" parameter, which can be passed to :meth:`get_metric_values`, :meth:`get_version_details` and :meth:`set_active_version`. :return: A dict representing the active version or None if no version is active. :rtype: Union[dict, None] """ filtered = [x for x in self.list_versions() if x["active"]] if len(filtered) == 0: return None else: return filtered[0] def get_version_details(self, version_id): """ Gets details for a version of a saved model :param str version_id: identifier of the version, as returned by :meth:`list_versions` :return: details of this trained model :rtype: :class:`dataikuapi.dss.ml.DSSTrainedPredictionModelDetails` """ details = self.client._perform_json( "GET", "/projects/%s/savedmodels/%s/versions/%s/details" % (self.project_key, self.sm_id, version_id)) snippet = self.client._perform_json( "GET", "/projects/%s/savedmodels/%s/versions/%s/snippet" % (self.project_key, self.sm_id, version_id)) if "facts" in details: return DSSTrainedClusteringModelDetails(details, snippet, saved_model=self, saved_model_version=version_id) else: return DSSTrainedPredictionModelDetails(details, snippet, saved_model=self, saved_model_version=version_id) def set_active_version(self, version_id): """ Sets a particular version of the saved model as the active one. :param str version_id: Identifier of the version, as returned by :meth:`list_versions` """ self.client._perform_empty( "POST", "/projects/%s/savedmodels/%s/versions/%s/actions/setActive" % (self.project_key, self.sm_id, version_id)) def delete_versions(self, versions, remove_intermediate=True): """ Deletes version(s) of the saved model. :param list[str] versions: list of versions to delete :param bool remove_intermediate: If True, also removes intermediate versions. In the case of a partitioned model, an intermediate version is created every time a partition has finished training. (defaults to **True**) """ if not isinstance(versions, list): versions = [versions] body = { "versions": versions, "removeIntermediate": remove_intermediate } self.client._perform_empty( "POST", "/projects/%s/savedmodels/%s/actions/delete-versions" % (self.project_key, self.sm_id), body=body) def get_origin_ml_task(self): """ Fetches the last ML task that has been exported to this saved model. :return: origin ML task or None if the saved model does not have an origin ml task :rtype: Union[:class:`dataikuapi.dss.ml.DSSMLTask`, None] """ fmi = self.get_settings().get_raw().get("lastExportedFrom") if fmi is not None: return DSSMLTask.from_full_model_id(self.client, fmi, project_key=self.project_key) def import_mlflow_version_from_path(self, version_id, path, code_env_name="INHERIT", container_exec_config_name="NONE", set_active=True, binary_classification_threshold=0.5): """ Creates a new version for this saved model from a path containing a MLFlow model. .. important:: Requires the saved model to have been created using :meth:`dataikuapi.dss.project.DSSProject.create_mlflow_pyfunc_model`. :param str version_id: identifier of the version, as returned by :meth:`list_versions` :param str path: absolute path on the local filesystem - must be a folder, and must contain a MLFlow model :param str code_env_name: Name of the code env to use for this model version. The code env must contain at least mlflow and the package(s) corresponding to the used MLFlow-compatible frameworks. * If value is "INHERIT", the default active code env of the project will be used. (defaults to **INHERIT**) :param str container_exec_config_name: Name of the containerized execution configuration to use for reading the metadata of the model * If value is "INHERIT", the container execution configuration of the project will be used. * If value is "NONE", local execution will be used (no container) (defaults to **INHERIT**) :param bool set_active: sets this new version as the active version of the saved model (defaults to **True**) :param float binary_classification_threshold: for binary classification, defines the actual threshold for the imported version (defaults to **0.5**) :return: external model version handler in order to interact with the new MLFlow model version :rtype: :class:`dataikuapi.dss.savedmodel.ExternalModelVersionHandler` """ # TODO: Add a check that it's indeed a MLFlow model folder import shutil import os archive_temp_dir = tempfile.mkdtemp() try: archive_filename = _make_zipfile(os.path.join(archive_temp_dir, "tmpmodel.zip"), path) with open(archive_filename, "rb") as fp: self.client._perform_empty( "POST", "/projects/{project_id}/savedmodels/{saved_model_id}/versions/{version_id}".format( project_id=self.project_key, saved_model_id=self.sm_id, version_id=version_id ), params={"codeEnvName": code_env_name, "containerExecConfigName": container_exec_config_name, "setActive": set_active, "binaryClassificationThreshold": binary_classification_threshold}, files={"file": (archive_filename, fp)}) return self.get_external_model_version_handler(version_id) finally: shutil.rmtree(archive_temp_dir) def import_mlflow_version_from_managed_folder(self, version_id, managed_folder, path, code_env_name="INHERIT", container_exec_config_name="INHERIT", set_active=True, binary_classification_threshold=0.5): """ Creates a new version for this saved model from a managed folder. .. important:: Requires the saved model to have been created using :meth:`dataikuapi.dss.project.DSSProject.create_mlflow_pyfunc_model`. :param str version_id: identifier of the version, as returned by :meth:`list_versions` :param managed_folder: managed folder, or identifier of the managed folder :type managed_folder: :class:`dataikuapi.dss.managedfolder.DSSManagedFolder` or str :param str path: path of the MLflow folder in the managed folder :param str code_env_name: Name of the code env to use for this model version. The code env must contain at least mlflow and the package(s) corresponding to the used MLFlow-compatible frameworks. * If value is "INHERIT", the default active code env of the project will be used. (defaults to **INHERIT**) :param str container_exec_config_name: Name of the containerized execution configuration to use for reading the metadata of the model * If value is "INHERIT", the container execution configuration of the project will be used. * If value is "NONE", local execution will be used (no container) (defaults to **INHERIT**) :param bool set_active: sets this new version as the active version of the saved model (defaults to **True**) :param float binary_classification_threshold: for binary classification, defines the actual threshold for the imported version (defaults to **0.5**) :return: external model version handler in order to interact with the new MLFlow model version :rtype: :class:`dataikuapi.dss.savedmodel.ExternalModelVersionHandler` """ # TODO: Add a check that it's indeed a MLflow model folder folder_ref = None if isinstance(managed_folder, DSSManagedFolder): folder_ref = "{}.{}".format(managed_folder.project_key, managed_folder.id) elif isinstance(managed_folder, dku_basestring_type): folder_ref = managed_folder else: raise Exception("managed_folder should either be a string representing the identifier of the managed folder" " or an instance of dataikuapi.dss.managedfolder.DSSManagedFolder") self.client._perform_empty( "POST", "/projects/{project_id}/savedmodels/{saved_model_id}/versions/{version_id}".format( project_id=self.project_key, saved_model_id=self.sm_id, version_id=version_id ), params={ "folderRef": folder_ref, "path": path, "codeEnvName": code_env_name, "containerExecConfigName": container_exec_config_name, "setActive": set_active, "binaryClassificationThreshold": binary_classification_threshold }, files={"file": (None, None)} # required for backend-mandated multipart request ) return self.get_external_model_version_handler(version_id) def import_mlflow_version_from_databricks(self, version_id, connection_name, use_unity_catalog, model_name, model_version, code_env_name="INHERIT", container_exec_config_name="INHERIT", set_active=True, binary_classification_threshold=0.5): connection = self.client.get_connection(connection_name) connection_info = connection.get_info(self.project_key) if connection_info is None or connection_info["type"] != "DatabricksModelDeployment": raise ValueError("The connection " + connection_name + " is not a Databricks connection") if "params" not in connection_info: raise Exception("Permission to view details of the connection " + connection_name + "required") temp_folder = tempfile.mktemp() os.mkdir(temp_folder) try: with DatabricksRepositoryContextManager(connection_info, use_unity_catalog): self._download_from_databricks_registry(model_name, model_version, temp_folder) return self.import_mlflow_version_from_path(version_id, os.path.join(temp_folder), code_env_name=code_env_name, container_exec_config_name=container_exec_config_name, set_active=set_active, binary_classification_threshold=binary_classification_threshold) finally: if os.path.exists(temp_folder): shutil.rmtree(temp_folder) @staticmethod def _download_from_databricks_registry(model_name, model_version, target_directory): from mlflow.store.artifact.models_artifact_repo import ModelsArtifactRepository mar = ModelsArtifactRepository("models:/{model_name}/{model_version}".format(model_name=model_name, model_version=model_version)) mar.download_artifacts("", dst_path=target_directory) def create_external_model_version(self, version_id, configuration, target_column_name=None, class_labels=None, set_active=True, binary_classification_threshold=0.5, input_dataset=None, selection=None, use_optimal_threshold=True, skip_expensive_reports=True, features_list=None, container_exec_config_name="NONE", input_format="GUESS", output_format="GUESS", evaluate=True): """ Creates a new version of an external model. .. important:: Requires the saved model to have been created using :meth:`dataikuapi.dss.project.DSSProject.create_external_model`. :param version_id: Identifier of the version, as returned by :meth:`list_versions` :type version_id: str :param configuration: A dictionary containing the desired saved model version configuration. * For SageMaker, syntax is: .. code-block:: python configuration = { "protocol": "sagemaker", "endpoint_name": "" } * For AzureML, syntax is: .. code-block:: python configuration = { "protocol": "azure-ml", "endpoint_name": "" } * For Vertex AI, syntax is: .. code-block:: python configuration = { "protocol": "vertex-ai", "endpoint_id": "" } * For Databricks, syntax is: .. code-block:: python configuration = { "protocol": "databricks", "endpointName": "" } :type configuration: dict :param target_column_name: Name of the target column. Mandatory if model performance will be evaluated :type target_column_name: str :param class_labels: List of strings, ordered class labels. Mandatory for evaluation of classification models :type class_labels: list or None :param set_active: (optional) Sets this new version as the active version of the saved model (defaults to **True**) :type set_active: bool :param binary_classification_threshold: (optional) For binary classification, defines the actual threshold for the imported version (defaults to **0.5**). Overwritten during evaluation if an evaluation dataset is specified and `use_optimal_threshold` is True :type binary_classification_threshold: float :param input_dataset: (mandatory if either evaluate=True, input_format=GUESS, output_format=GUESS, features_list is None) Dataset to use to infer the features names and types (if features_list is not set), evaluate the model, populate interpretation tabs, and guess input/output formats (if input_format=GUESS or output_format=GUESS). :type input_dataset: str or :class:`dataikuapi.dss.dataset.DSSDataset` or :class:`dataiku.Dataset` :param selection: (optional) Sampling parameter for input_dataset during evaluation. * Example 1: head 100 lines ``DSSDatasetSelectionBuilder().with_head_sampling(100)`` * Example 2: random 500 lines ``DSSDatasetSelectionBuilder().with_random_fixed_nb_sampling(500)`` * Example 3: head 100 lines ``{"samplingMethod": "HEAD_SEQUENTIAL", "maxRecords": 100}`` Defaults to head 100 lines :type selection: dict or :class:`DSSDatasetSelectionBuilder` or None :param use_optimal_threshold: (optional) Set as threshold for this model version the threshold that has been computed during evaluation according to the metric set on the saved model setting (i.e. ``prediction_metrics_settings['thresholdOptimizationMetric']``) :type use_optimal_threshold: bool :param skip_expensive_reports: (optional) Skip computation of expensive/slow reports (e.g. feature importance). :type skip_expensive_reports: bool :param features_list: (optional) List of features, in JSON. Used if input_dataset is not defined :type features_list: list of ``{"name": "feature_name", "type": "feature_type"}`` or None :param container_exec_config_name: (optional) name of the containerized execution configuration to use for running the evaluation process. * If value is "INHERIT", the container execution configuration of the project will be used. * If value is "NONE", local execution will be used (no container) :type container_exec_config_name: str :param input_format: (optional) Input format to use when querying the underlying endpoint. For the 'azure-ml' and 'sagemaker' protocols, this option must be set if input_dataset is not set. Supported values are: * For all protocols: - GUESS (default): Guess the input format by cycling through supported input formats and making requests using data from input_dataset. * For Amazon SageMaker: - INPUT_SAGEMAKER_CSV - INPUT_SAGEMAKER_JSON - INPUT_SAGEMAKER_JSON_EXTENDED - INPUT_SAGEMAKER_JSONLINES - INPUT_DEPLOY_ANYWHERE_ROW_ORIENTED_JSON * For Vertex AI: - INPUT_VERTEX_DEFAULT * For Azure Machine Learning: - INPUT_AZUREML_JSON_INPUTDATA - INPUT_AZUREML_JSON_WRITER - INPUT_AZUREML_JSON_INPUTDATA_DATA - INPUT_DEPLOY_ANYWHERE_ROW_ORIENTED_JSON * For Databricks: - INPUT_RECORD_ORIENTED_JSON - INPUT_SPLIT_ORIENTED_JSON - INPUT_TF_INPUTS_JSON - INPUT_TF_INSTANCES_JSON - INPUT_DATABRICKS_CSV :type input_format: str :param output_format: (optional) Output format to use to parse the underlying endpoint's response. For the 'azure-ml' and 'sagemaker' protocols, this option must be set if input_dataset is not set. Supported values are: * For all protocols: - GUESS (default): Guess the output format by cycling through supported output formats and making requests using data from input_dataset. * For Amazon SageMaker: - OUTPUT_SAGEMAKER_CSV - OUTPUT_SAGEMAKER_ARRAY_AS_STRING - OUTPUT_SAGEMAKER_JSON - OUTPUT_DEPLOY_ANYWHERE_JSON * For Vertex AI: - OUTPUT_VERTEX_DEFAULT * For Azure Machine Learning: - OUTPUT_AZUREML_JSON_OBJECT - OUTPUT_AZUREML_JSON_ARRAY - OUTPUT_DEPLOY_ANYWHERE_JSON * For Databricks: - OUTPUT_DATABRICKS_JSON :type output_format: str :param evaluate: (optional) True (default) if this model should be evaluated using input_dataset, False to disable evaluation. :type evaluate: bool * Example: create a SageMaker Saved Model and add an endpoint as a version, evaluated on a dataset: .. code-block:: python import dataiku client = dataiku.api_client() project = client.get_default_project() # create a SageMaker saved model, whose endpoints are hosted in region eu-west-1 sm = project.create_external_model("SaveMaker External Model", "BINARY_CLASSIFICATION", {"protocol": "sagemaker", "region": "eu-west-1"}) # configuration to add endpoint configuration = { "protocol": "sagemaker", "endpoint_name": "titanic-survived-endpoint" } smv = sm.create_external_model_version("v0", configuration, target_column_name="Survived", class_labels=["0", "1"], input_dataset="evaluation_dataset") A dataset named "evaluation_dataset" must exist in the current project. Its schema and content should match the endpoint expectations. Depending on the way the model deployed on the endpoint was created, it may require a certain schema and not accept extra columns, it may not deal with missing features, etc. * Example: create a Vertex AI Saved Model and add an endpoint as a version, without evaluating it: .. code-block:: python import dataiku client = dataiku.api_client() project = client.get_default_project() # create a VertexAI saved model, whose endpoints are hosted in region europe-west-1 sm = project.create_external_model("Vertex AI Proxy Model", "BINARY_CLASSIFICATION", {"protocol":"vertex-ai", "region":"europe-west1"}) configuration = { "protocol":"vertex-ai", "project_id": "my-project", "endpoint_id": "123456789012345678" } smv = sm.create_external_model_version("v1", configuration, target_column_name="Survived", class_labels=["0", "1"], input_dataset="titanic") A dataset named "my_dataset" must exist in the current project. It will be used to infer the schema of the data to submit to the endpoint. As there is no evaluation dataset specified, the interpretation tabs of this model version will be for the most empty. But the model still can be used to score datasets. It can also be evaluated on a dataset by an Evaluation Recipe. * Example: create an AzureML Saved Model .. code-block:: python import dataiku client = dataiku.api_client() project = client.get_default_project() # create an Azure ML saved model. No region specified, as this notion does not exist for Azure ML sm = project.create_external_model("Azure ML Proxy Model", "BINARY_CLASSIFICATION", {"protocol": "azure-ml"}) configuration = { "protocol": "azure-ml", "subscription_id": ">", "resource_group": "", "workspace": "", "endpoint_name": "" } features_list = [{'name': 'Pclass', 'type': 'bigint'}, {'name': 'Age', 'type': 'double'}, {'name': 'SibSp', 'type': 'bigint'}, {'name': 'Parch', 'type': 'bigint'}, {'name': 'Fare', 'type': 'double'}] smv = sm.create_external_model_version("20230324-in-prod", configuration, target_column_name="Survived", class_labels=["0", "1"], features_list=features_list) * Example: minimalistic creation of a VertexAI model binary classification model .. code-block:: python import dataiku client = dataiku.api_client() project = client.get_default_project() sm = project.create_external_model("Raw Vertex AI Proxy Model", "BINARY_CLASSIFICATION", {"protocol": "vertex-ai", "region": "europe-west1"}) configuration = { "protocol": "vertex-ai", "project_id": "my-project", "endpoint_id": "123456789012345678" } smv = sm.create_external_model_version("legacy-model", configuration, class_labels=["0", "1"]) This model will have empty interpretation tabs and can not be evaluated later by an Evaluation Recipe, as its target is not defined, but it can be scored. * Example: create a Databricks Saved Model .. code-block:: python import dataiku client = dataiku.api_client() project = client.get_default_project() sm = project.create_external_model("Databricks External Model", "BINARY_CLASSIFICATION", {"protocol": "databricks","connection": "db"}) smv = sm.create_external_model_version("vX", {"protocol": "databricks", "endpointName": ""}, target_column_name="Survived", class_labels=["0", "1"], input_dataset="train_titanic_prepared") """ model_version = self._create_external_model_version(version_id, configuration, set_active, binary_classification_threshold) model_version._set_core_external_metadata(target_column_name, class_labels, features_list, input_dataset, container_exec_config_name=container_exec_config_name, input_format=input_format, output_format=output_format) if evaluate: if input_dataset is None: raise ValueError("You must provide an input_dataset to evaluate this model, or set evaluate to False") model_version.evaluate(input_dataset, container_exec_config_name, selection, use_optimal_threshold, skip_expensive_reports) def _create_external_model_version(self, version_id, configuration, set_active=True, binary_classification_threshold=0.5): """ .. important:: Requires the saved model to have been created using :meth:`dataikuapi.dss.project.DSSProject.create_external_model`. :param str version_id: identifier of the version, as returned by :meth:`list_versions` :param dict configuration: A dictionary containing the required params for the selected protocol :param bool set_active: sets this new version as the active version of the saved model (defaults to **True**) :param float binary_classification_threshold: for binary classification, defines the actual threshold for the imported version (defaults to **0.5**) :return: external model version handler in order to interact with the new Proxy model version :rtype: :class:`dataikuapi.dss.savedmodel.ExternalModelVersionHandler` """ import json # Set the type in configuration assert isinstance(configuration, dict), "configuration should be a dictionary containing the required params for the selected protocol" self.client._perform_empty( "POST", "/projects/{project_id}/savedmodels/{saved_model_id}/versions/{version_id}".format( project_id=self.project_key, saved_model_id=self.sm_id, version_id=version_id ), params={ "setActive": set_active, "proxyModelVersionConfiguration": json.dumps(configuration), "binaryClassificationThreshold": binary_classification_threshold }, files={"file": (None, None)} # required for backend-mandated multipart request ) return self.get_external_model_version_handler(version_id) def get_external_model_version_handler(self, version_id): """ Returns a handler to interact with an external model version (MLflow or Proxy model) :param str version_id: identifier of the version, as returned by :meth:`list_versions` :return: external model version handler :rtype: :class:`dataikuapi.dss.savedmodel.ExternalModelVersionHandler` """ return ExternalModelVersionHandler(self, version_id) ######################################################## # Metrics ######################################################## def get_metric_values(self, version_id): """ Gets the values of the metrics on the specified version of this saved model :param str version_id: identifier of the version, as returned by :meth:`list_versions` :return: a list of metric objects and their value :rtype: list """ return ComputedMetrics(self.client._perform_json( "GET", "/projects/%s/savedmodels/%s/metrics/%s" % (self.project_key, self.sm_id, version_id))) ######################################################## # Misc ######################################################## def get_zone(self): """ Gets the flow zone of this saved model :return: the saved model's flow zone :rtype: :class:`dataikuapi.dss.flow.DSSFlowZone` """ return self.project.get_flow().get_zone_of_object(self) def move_to_zone(self, zone): """ Moves this object to a flow zone :param object zone: flow zone where the object should be moved :type zone: :class:`dataikuapi.dss.flow.DSSFlowZone` """ if isinstance(zone, basestring): zone = self.project.get_flow().get_zone(zone) zone.add_item(self) def share_to_zone(self, zone): """ Shares this object to a flow zone :param object zone: flow zone where the object should be shared :type zone: :class:`dataikuapi.dss.flow.DSSFlowZone` """ if isinstance(zone, basestring): zone = self.project.get_flow().get_zone(zone) zone.add_shared(self) def unshare_from_zone(self, zone): """ Unshares this object from a flow zone :param object zone: flow zone from which the object shouldn't be shared :type zone: :class:`dataikuapi.dss.flow.DSSFlowZone` """ if isinstance(zone, basestring): zone = self.project.get_flow().get_zone(zone) zone.remove_shared(self) def get_usages(self): """ Gets the recipes referencing this model :return: a list of usages :rtype: list """ return self.client._perform_json("GET", "/projects/%s/savedmodels/%s/usages" % (self.project_key, self.sm_id)) def get_object_discussions(self): """ Gets a handle to manage discussions on the saved model :return: the handle to manage discussions :rtype: :class:`dataikuapi.dss.discussion.DSSObjectDiscussions` """ return DSSObjectDiscussions(self.client, self.project_key, "SAVED_MODEL", self.sm_id) ######################################################## # Deletion ######################################################## def delete(self): """Deletes the saved model""" return self.client._perform_empty("DELETE", "/projects/%s/savedmodels/%s" % (self.project_key, self.sm_id)) class MLFlowVersionSettings: """ Handle for the settings of an imported MLFlow model version. .. important:: Do not create this class directly, instead use :meth:`dataikuapi.dss.savedmodel.ExternalModelVersionHandler.get_settings` :param version_handler: handler to interact with an external model version :type version_handler: :class:`dataikuapi.dss.savedmodel.ExternalModelVersionHandler` :param dict data: raw settings of the imported MLFlow model version """ def __init__(self, version_handler, data): self.version_handler = version_handler self.data = data @property def raw(self): """ :return: The raw settings of the imported MLFlow model version :rtype: dict """ return self.data def save(self): """Saves the settings of this MLFlow model version""" self.version_handler.saved_model.client._perform_empty("PUT", "/projects/%s/savedmodels/%s/versions/%s/external-ml/metadata" % (self.version_handler.saved_model.project_key, self.version_handler.saved_model.sm_id, self.version_handler.version_id), body=self.data) class ExternalModelVersionHandler: """ Handler to interact with an External model version (MLflow import of Proxy model). .. important:: Do not create this class directly, instead use :meth:`dataikuapi.dss.savedmodel.DSSSavedModel.get_external_model_version_handler` :param saved_model: the saved model object :type saved_model: :class:`dataikuapi.dss.savedmodel.DSSSavedModel` :param str version_id: identifier of the version, as returned by :meth:`dataikuapi.dss.savedmodel.DSSSavedModel.list_versions` """ def __init__(self, saved_model, version_id): self.saved_model = saved_model self.version_id = version_id def get_settings(self): """ Returns the settings of the MLFlow model version :return: settings of the MLFlow model version :rtype: :class:`dataikuapi.dss.savedmodel.MLFlowVersionSettings` """ metadata = self.saved_model.client._perform_json("GET", "/projects/%s/savedmodels/%s/versions/%s/external-ml/metadata" % (self.saved_model.project_key, self.saved_model.sm_id, self.version_id)) return MLFlowVersionSettings(self, metadata) def _init_model_version_info(self, target_column_name, class_labels=None, get_features_from_dataset=None, features_list=None): if features_list is not None and get_features_from_dataset is not None: raise Exception("The information of the features should come either from the features_list or get_features_from_dataset, but not both.") metadata = self.saved_model.client._perform_json("GET", "/projects/%s/savedmodels/%s/versions/%s/external-ml/metadata" % (self.saved_model.project_key, self.saved_model.sm_id, self.version_id)) if target_column_name is not None: metadata["targetColumnName"] = target_column_name if class_labels is not None: metadata["classLabels"] = [{"label": label} for label in class_labels] if get_features_from_dataset is not None: metadata["gatherFeaturesFromDataset"] = get_features_from_dataset # TODO: add support for get_features_from_signature=False, # if get_features_from_signature: # raise Exception("Get features from signature is not yet implemented") if features_list is not None: for feature in features_list: if not ("name" in feature and "type" in feature): raise Exception("The features_list should be a list of {'name': 'feature_name', 'type': 'feature_type'}") metadata["features"] = features_list return metadata def _set_core_external_metadata(self, target_column_name, class_labels=None, features_list=None, input_dataset=None, container_exec_config_name="NONE", input_format="GUESS", output_format="GUESS"): """ Sets core metadata of external models, see :meth:`DSSSavedModel.create_external_model_version` for details about parameters. """ if features_list is not None: model_version_info = self._init_model_version_info(target_column_name, class_labels, None, features_list) else: model_version_info = self._init_model_version_info(target_column_name, class_labels, input_dataset, None) protocol = model_version_info["proxyModelVersionConfiguration"]["protocol"] if protocol == "vertex-ai": model_version_info["inputFormat"] = "INPUT_VERTEX_DEFAULT" model_version_info["outputFormat"] = "OUTPUT_VERTEX_DEFAULT" elif protocol in ["azure-ml", "sagemaker", "databricks"]: model_version_info["inputFormat"] = input_format model_version_info["outputFormat"] = output_format if input_dataset is not None: model_version_info["signatureAndFormatsGuessingDataset"] = input_dataset self.saved_model.client._perform_empty( "PUT", "/projects/{project_key}/savedmodels/{sm_id}/versions/{version_id}/external-ml/metadata?containerExecConfigName={container_exec_config_name}".format( project_key=self.saved_model.project_key, sm_id=self.saved_model.sm_id, version_id=self.version_id, container_exec_config_name=container_exec_config_name), body=model_version_info) def set_core_metadata(self, target_column_name, class_labels=None, get_features_from_dataset=None, features_list=None, container_exec_config_name="NONE"): """ Sets metadata for this MLFlow model version In addition to ``target_column_name``, one of ``get_features_from_dataset`` or ``features_list`` must be passed in order to be able to evaluate performance :param str target_column_name: name of the target column. Mandatory in order to be able to evaluate performance :param class_labels: List of strings, ordered class labels. Mandatory in order to be able to evaluate performance on classification models :type class_labels: list or None :param get_features_from_dataset: name of a dataset to get feature names from :type get_features_from_dataset: str or None :param features_list: list of ``{"name": "feature_name", "type": "feature_type"}`` :type features_list: list or None :param str container_exec_config_name: name of the containerized execution configuration to use for running the evaluation process. * If value is "INHERIT", the container execution configuration of the project will be used. * If value is "NONE", local execution will be used (no container) (defaults to **None**) """ model_version_info = self._init_model_version_info(target_column_name, class_labels, get_features_from_dataset, features_list) self.saved_model.client._perform_empty( "PUT", "/projects/{project_key}/savedmodels/{sm_id}/versions/{version_id}/external-ml/metadata?containerExecConfigName={container_exec_config_name}".format( project_key=self.saved_model.project_key, sm_id=self.saved_model.sm_id, version_id=self.version_id, container_exec_config_name=container_exec_config_name ), body=model_version_info ) def evaluate(self, dataset_ref, container_exec_config_name="INHERIT", selection=None, use_optimal_threshold=True, skip_expensive_reports=True): """ Evaluates the performance of this model version on a particular dataset. After calling this, the "result screens" of the MLFlow model version will be available (confusion matrix, error distribution, performance metrics, ...) and more information will be available when calling: :meth:`dataikuapi.dss.savedmodel.DSSSavedModel.get_version_details` Evaluation is available only for models having BINARY_CLASSIFICATION, MULTICLASS or REGRESSION as prediction type. See :meth:`DSSProject.create_mlflow_pyfunc_model`. .. important:: :meth:`set_core_metadata` must be called before you can evaluate a dataset :param dataset_ref: Evaluation dataset to use :type dataset_ref: str or :class:`dataikuapi.dss.dataset.DSSDataset` or :class:`dataiku.Dataset` :param str container_exec_config_name: Name of the containerized execution configuration to use for running the evaluation process. * If value is "INHERIT", the container execution configuration of the project will be used. * If value is "NONE", local execution will be used (no container) (defaults to **INHERIT**) :param selection: Sampling parameter for the evaluation. * Example 1: ``DSSDatasetSelectionBuilder().with_head_sampling(100)`` * Example 2: ``{"samplingMethod": "HEAD_SEQUENTIAL", "maxRecords": 100}`` (defaults to **None**) :type selection: dict or :class:`DSSDatasetSelectionBuilder` or None :param bool use_optimal_threshold: Choose between optimized or actual threshold. Optimized threshold has been computed according to the metric set on the saved model setting (i.e. ``prediction_metrics_settings['thresholdOptimizationMetric']``) (defaults to **True**) :param boolean skip_expensive_reports: Skip expensive/slow reports (e.g. feature importance). """ sampling_param = selection.build() if isinstance( selection, DSSDatasetSelectionBuilder) else selection if hasattr(dataset_ref, 'name'): dataset_ref = dataset_ref.name req = { "datasetRef": dataset_ref, "containerExecConfigName": container_exec_config_name, "samplingParam": sampling_param } post = ("/projects/{project_id}/savedmodels/{saved_model_id}/versions/{version_id}/external-ml/actions/evaluate" "?useOptimalThreshold={use_optimal_threshold}&skipExpensiveReports={skip_expensive_reports}").format( project_id=self.saved_model.project_key, saved_model_id=self.saved_model.sm_id, version_id=self.version_id, use_optimal_threshold=use_optimal_threshold, skip_expensive_reports=skip_expensive_reports) self.saved_model.client._perform_empty("POST", post, body=req) class DSSSavedModelSettings: """ Handle on the settings of a saved model. .. important:: Do not create this class directly, instead use :meth:`dataikuapi.dss.DSSSavedModel.get_settings` :param saved_model: the saved model object :type saved_model: :class:`dataikuapi.dss.savedmodel.DSSSavedModel` :param dict settings: the settings of the saved model """ def __init__(self, saved_model, settings): self.saved_model = saved_model self.settings = settings def get_raw(self): """ Returns the raw settings of the saved model :return: the raw settings of the saved model :rtype: dict """ return self.settings @property def prediction_metrics_settings(self): """ Returns the metrics-related settings :rtype: dict """ return self.settings["miniTask"]["modeling"]["metrics"] def save(self): """Saves the settings of this saved model""" self.saved_model.client._perform_empty("PUT", "/projects/%s/savedmodels/%s" % (self.saved_model.project_key, self.saved_model.sm_id), body=self.settings) dataiku-api-client-python-13.2.2/dataikuapi/dss/scenario.py000066400000000000000000001211161471211575100236530ustar00rootroot00000000000000from datetime import datetime import time, warnings from ..utils import DataikuException, _timestamp_ms_to_zoned_datetime, _local_timezone from .discussion import DSSObjectDiscussions from .utils import DSSTaggableObjectListItem from dateutil.tz import tzlocal class DSSScenario(object): """ A handle to interact with a scenario on the DSS instance. .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.get_scenario` """ def __init__(self, client, project_key, id): self.client = client self.id = id self.project_key = project_key def abort(self): """ Abort the scenario. This method does nothing if the scenario is not currently running. """ return self.client._perform_json( "POST", "/projects/%s/scenarios/%s/abort" % (self.project_key, self.id)) def run_and_wait(self, params=None, no_fail=False): """ Request a run of the scenario and wait the end of the run to complete. The method requests a new run, which will start after a few seconds. :param dict params: additional parameters that will be passed to the scenario through trigger params (defaults to `{}`) :param boolean no_fail: if False, raises if the run doesn't end with a SUCCESS outcome :return: a handle on the run :rtype: :class:`DSSScenarioRun` """ if params is None: params = {} trigger_fire = self.run(params) scenario_run = trigger_fire.wait_for_scenario_run(no_fail) waiter = DSSScenarioRunWaiter(scenario_run, trigger_fire) return waiter.wait(no_fail) def run(self, params=None): """ Request a run of the scenario. The method requests a new run, which will start after a few seconds. .. note:: This method returns a trigger fire, NOT a Scenario run object. The trigger fire may ultimately result in a run or not. Usage example: .. code-block:: python scenario = project.get_scenario("myscenario") trigger_fire = scenario.run() # When you call `run` a scenario, the scenario is not immediately # started. Instead a "manual trigger" fires. # # This trigger fire can be cancelled if the scenario was already running, # or if another trigger fires. Thus, the scenario run is not available # immediately, and we must "wait" for it scenario_run = trigger_fire.wait_for_scenario_run() # Now the scenario is running. We can wait for it synchronously with # scenario_run.wait_for_completion(), but if we want to do other stuff # at the same time, we can use refresh while True: scenario_run.refresh() if scenario_run.running: print("Scenario is still running ...") else: print("Scenario is not running anymore") break time.sleep(5) :params dict params: additional parameters that will be passed to the scenario through trigger params (defaults to `{}`) :return: a request for a run, as a trigger fire object :rtype: :class:`DSSTriggerFire` """ if params is None: params = {} trigger_fire = self.client._perform_json( "POST", "/projects/%s/scenarios/%s/run" % (self.project_key, self.id), body=params) return DSSTriggerFire(self, trigger_fire) def get_last_runs(self, limit=10, only_finished_runs=False): """ Get the list of the last runs of the scenario. :param int limit: maximum number of last runs to retrieve :param boolean only_finished_runs: if True, currently running runs are not returned. :return: a list of :class:`DSSScenarioRun` :rtype: list """ runs = self.client._perform_json( "GET", "/projects/%s/scenarios/%s/get-last-runs" % (self.project_key, self.id), params={ 'limit' : limit, 'onlyFinishedRuns' : only_finished_runs }) return [DSSScenarioRun(self.client, run) for run in runs] def get_runs_by_date(self, from_date, to_date=datetime.now()): """ Get the list of the runs of the scenario in a given date range. :param datetime from_date: start of the date range to retrieve runs for, inclusive :param datetime to_date: end of the date range to retrieve runs for, exclusive :return: a list of :class:`DSSScenarioRun` :rtype: list """ def as_date(d): if isinstance(d, datetime): return d.strftime("%Y-%m-%d") else: return d runs = self.client._perform_json( "GET", "/projects/%s/scenarios/%s/get-runs-by-date" % (self.project_key, self.id), params={ 'fromDate': as_date(from_date), 'toDate': as_date(to_date) }) return [DSSScenarioRun(self.client, run) for run in runs] def get_last_finished_run(self): """ Get the last run that completed. The run may be successful or failed, or even aborted. :rtype: :class:`DSSScenarioRun` """ lr = [sr for sr in self.get_last_runs() if not sr.running] if len(lr) == 0: raise ValueError("No scenario run completed") return lr[0] def get_last_successful_run(self): """ Get the last run that completed successfully. :rtype: :class:`DSSScenarioRun` """ lr = self.get_last_runs(only_finished_runs=True) if len(lr) == 0: raise ValueError("No scenario run completed successfully") return lr[0] def get_current_run(self): """ Get the current run of the scenario. If the scenario is not running at the moment, returns None. :rtype: :class:`DSSScenarioRun` """ last_run = self.get_last_runs(1) if len(last_run) == 0: return None last_run = last_run[0] if 'result' in last_run.run: return None # has a result means it's done else: return last_run def get_run(self, run_id): """ Get a handle to a run of the scenario. :param string run_id: identifier of the run. :rtype: :class:`DSSScenarioRun` """ run_details = self.client._perform_json( "GET", "/projects/%s/scenarios/%s/%s/" % (self.project_key, self.id, run_id)) return DSSScenarioRun(self.client, run_details["scenarioRun"]) def get_status(self): """ Get the status of this scenario. :rtype: :class:`DSSScenarioStatus` """ data = self.client._perform_json("GET", "/projects/%s/scenarios/%s/light" % (self.project_key, self.id)) return DSSScenarioStatus(self, data) def get_settings(self): """ Get the settings of this scenario. :return: a :class:`StepBasedScenarioSettings` for step-based scenarios, or a :class:`PythonScriptBasedScenarioSettings` for scenarios defined by a python script. :rtype: :class:`DSSScenarioSettings` """ data = self.client._perform_json("GET", "/projects/%s/scenarios/%s" % (self.project_key, self.id)) if data["type"] == "step_based": return StepBasedScenarioSettings(self.client, self, data) else: payload = self.client._perform_json( "GET", "/projects/%s/scenarios/%s/payload" % (self.project_key, self.id)).get('script', '') return PythonScriptBasedScenarioSettings(self.client, self, data, payload) def get_average_duration(self, limit=3): """ Get the average duration of the last runs of this scenario. The duration is computed on successful runs only, that is, on runs that ended with SUCCESS or WARNING satus. If there are not enough runs to perform the average, returns None :param int limit: number of last runs to average on :return: the average duration of the last runs, in seconds :rtype: float """ last_runs = self.get_last_runs(limit=limit, only_finished_runs=True) if len(last_runs) < limit: return None return sum([run.get_duration() for run in last_runs]) / len(last_runs) def delete(self): """ Delete this scenario. """ return self.client._perform_json( "DELETE", "/projects/%s/scenarios/%s" % (self.project_key, self.id)) def get_object_discussions(self): """ Get a handle to manage discussions on the scenario. :returns: the handle to manage discussions :rtype: :class:`dataikuapi.dss.discussion.DSSObjectDiscussions` """ return DSSObjectDiscussions(self.client, self.project_key, "SCENARIO", self.id) ######################################################## # Advanced ######################################################## def get_trigger_fire(self, trigger_id, trigger_run_id): """ Get a trigger fire object. .. caution:: Advanced usages only (see :meth:`run()`) :param string trigger_id: identifier of the trigger, in the scenario's settings :param string trigger_run_id: identifier of the run of the trigger :rtype: :class:`DSSTriggerFire` """ trigger_fire = self.client._perform_json( "GET", "/projects/%s/scenarios/trigger/%s/%s" % (self.project_key, self.id, trigger_id), params={ 'triggerRunId' : trigger_run_id }) return DSSTriggerFire(self, trigger_fire) ######################################################## # Deprecated ######################################################## def get_definition(self, with_status=True): """ Get the definition of the scenario. .. attention:: Deprecated, use :meth:`get_settings` and :meth:`get_status` :param bool with_status: if True, get only the run status of the scenario. If False, get the raw definition of the scenario. :return: if **with_status** is False, the scenario's definition as returned by :meth:`DSSScenarioSettings.get_raw()`. If **with_status** is True, a summary of the scenario as returned by :meth:`DSSScenarioStatus.get_raw()` :rtype: dict """ warnings.warn("DSSScenario.get_definition is deprecated, please use get_settings", DeprecationWarning) suffix = '/light' if with_status else '' return self.client._perform_json( "GET", "/projects/%s/scenarios/%s%s" % (self.project_key, self.id, suffix)) def set_definition(self, definition, with_status=True): """ Update the definition of this scenario. .. attention:: Deprecated, use :meth:`get_settings` and :meth:`DSSScenarioSettings.save` :param dict definition: a scenario definition obtained by calling :meth:`get_definition()`, then modified :param bool with_status: should be the same as the value passed to :meth:`get_definition()`. If True, the only fields that can be modified are active, checklists, description, shortDesc and tags """ warnings.warn("DSSScenario.set_definition is deprecated, please use get_settings", DeprecationWarning) suffix = '/light' if with_status else '' return self.client._perform_json( "PUT", "/projects/%s/scenarios/%s%s" % (self.project_key, self.id, suffix), body = definition) def get_payload(self, extension='py'): """ Get the payload of the scenario. .. attention:: Deprecated, use :meth:`get_settings` and :meth:`get_status` :param string extension: the type of script. Default is 'py' for python """ warnings.warn("DSSScenario.get_payload is deprecated, please use get_settings", DeprecationWarning) return self.client._perform_json( "GET", "/projects/%s/scenarios/%s/payload" % (self.project_key, self.id), body = {'extension' : extension}).get('script', '') # used to have a with_status parameter, that was unused and had a default value, so was likely never used by callers def set_payload(self, script, extension='py'): """ Update the payload of this scenario. .. attention:: Deprecated, use :meth:`get_settings` and :meth:`DSSScenarioSettings.save` :param string script: the new value of the script :param string extension: the type of script. Default is 'py' for python """ warnings.warn("DSSScenario.set_payload is deprecated, please use get_settings", DeprecationWarning) return self.client._perform_json( "PUT", "/projects/%s/scenarios/%s/payload" % (self.project_key, self.id), body = {'script' : script, 'extension' : extension}) class DSSScenarioStatus(object): """ Status of a scenario. .. important:: Do not instantiate directly, use :meth:`DSSScenario.get_status()` """ def __init__(self, scenario, data): self.scenario = scenario self.data = data def get_raw(self): """ Get the raw status data. :return: the status, as a dict. Notable fields are: * **active** : whether the scenario runs its automatic triggers * **running** : whether the scenario is currently running * **start** : if the scenario is running, the timestamp of the beginning of the run :rtype: dict """ return self.data @property def running(self): """ Whether the scenario is currently running :rtype: boolean """ return self.data["running"] @property def next_run(self): """ Time at which the scenario is expected to run next. This expected time is computed based on the only triggers for which forecasts are possible, that is, the active time-based triggers. May be None if there is no such trigger. This is an approximate indication as scenario run may be delayed, especially in the case of multiple triggers or high load. :rtype: :class:`datetime.datetime` """ if not "nextRun" in self.data or self.data["nextRun"] == 0: return None return _timestamp_ms_to_zoned_datetime(self.data["nextRun"]) class DSSScenarioSettings(object): """ Settings of a scenario. .. important:: Do not instantiate directly, use :meth:`DSSScenario.get_settings()` """ def __init__(self, client, scenario, data): self.client = client self.scenario = scenario self.data = data def get_raw(self): """ Get the raw definition of the scenario. This method returns a reference to the settings, not a copy. Modifying the settings then calling :meth:`save()` saves the changes made. :return: the scenario, as a dict. The type-specific parameters of the scenario are in a **params** sub-dict. For step-based scenarios, the **params** will contain the definitions of the steps as a **steps** list of dict. :rtype: dict """ return self.data @property def active(self): """ Whether this scenario is currently active, i.e. its auto-triggers are executing. :rtype: boolean """ return self.data["active"] @active.setter def active(self, active): self.data["active"] = active @property def run_as(self): """ Get the login of the user the scenario runs as. None means that the scenario runs as the last user who modified the scenario. Only administrators may set a non-None value. :rtype: string """ return self.data.get("runAsUser", None) @run_as.setter def run_as(self, run_as): self.data["runAsUser"] = run_as @property def effective_run_as(self): """ Get the effective 'run as' of the scenario. If the value returned by :meth:`run_as()` is not None, then that value. Otherwise, this will be the login of the last user who modified the scenario. .. note:: If this method returns None, it means that it was not possible to identify who this scenario should run as. This scenario is probably not currently functioning. :rtype: string """ # Note: this logic must match the one in ScenarioBaseService if "runAsUser" in self.data: return self.data["runAsUser"] elif "versionTag" in self.data: return self.data["versionTag"]["lastModifiedBy"]["login"] elif "creationTag" in self.data: return self.data["creationTag"]["lastModifiedBy"]["login"] else: return None @property def raw_triggers(self): """ Get the list of automatic triggers. This method returns a reference to the settings, not a copy. Modifying the settings then calling :meth:`save()` saves the changes made. :return: list of the automatic triggers, each one a dict. An **active** boolean field indicates whether the trigger is running automatically. :rtype: list[dict] """ return self.data["triggers"] @property def raw_reporters(self): """ Get the list of reporters. This method returns a reference to the settings, not a copy. Modifying the settings then calling :meth:`save()` saves the changes made. :return: list of reporters on the scenario, each one a dict. :rtype: list[dict] """ return self.data["reporters"] def add_periodic_trigger(self, every_minutes=5): """ Add a trigger that runs the scenario every X minutes. :param int every_minutes: interval between activations of the trigger, in minutes """ trigger = {"active": True, "type": "temporal", "params": { "frequency": "Minutely", "repeatFrequency": every_minutes }} self.raw_triggers.append(trigger) def add_hourly_trigger(self, minute_of_hour=0, year=None, month=None, day=None, starting_hour=0, repeat_every=1, timezone="SERVER"): """ Add a trigger that runs the scenario every X hours. :param int repeat_every: interval between activations of the trigger, in hours :param int minute_of_hour: minute in the hour when the trigger should run :param int year: year part of the date/time before which the trigger won't run :param int month: month part of the date/time before which the trigger won't run :param int day: day part of the date/time before which the trigger won't run :param int starting_hour: hour part of the date/time before which the trigger won't run :param string timezone: timezone in which the start date/time is expressed. Can be a time zone name like "Europe/Paris" or "SERVER" for the time zone of the DSS server """ starting_date =self.__get_starting_date__(year=year, month=month, day=day) trigger = {"active": True, "type": "temporal", "params": { "frequency": "Hourly", "hour": starting_hour, "minute": minute_of_hour, "startingFrom": starting_date, "repeatFrequency": repeat_every, "timezone": timezone }} self.raw_triggers.append(trigger) def add_daily_trigger(self, hour=2, minute=0, days=None, year=None, month=None, day=None, repeat_every=1, timezone="SERVER"): """ Add a trigger that runs the scenario every X days. :param int repeat_every: interval between activations of the trigger, in days :param list days: if None, the trigger runs every **repeat_every** other day. If set to a list of day names, the trigger runs every **repeat_every** other week, on the designated days. The day names are Monday, Tuesday, Wednesday, Thursday, Friday, Saturday, Sunday :param int hour: hour in the day when the trigger should run :param int minute: minute in the hour when the trigger should run :param int year: year part of the date/time before which the trigger won't run :param int month: month part of the date/time before which the trigger won't run :param int day: day part of the date/time before which the trigger won't run :param string timezone: timezone in which the start date/time is expressed. Can be a time zone name like "Europe/Paris" or "SERVER" for the time zone of the DSS server """ starting_date =self.__get_starting_date__(year=year, month=month, day=day) if days is None: trigger = {"active": True, "type": "temporal", "params": { "frequency": "Daily", "hour": hour, "minute": minute, "startingFrom": starting_date, "repeatFrequency": repeat_every, "timezone": timezone }} else: trigger = {"active": True, "type": "temporal", "params": { "frequency": "Weekly", "hour": hour, "minute": minute, "startingFrom": starting_date, "daysOfWeek": days, "repeatFrequency": repeat_every, "timezone": timezone }} self.raw_triggers.append(trigger) def add_monthly_trigger(self, day=1, hour=2, minute=0, year=None, month=None, run_on="ON_THE_DAY", repeat_every=1, timezone="SERVER"): """ Add a trigger that runs the scenario every X months. :param int repeat_every: interval between activations of the trigger, in months :parma string run_on: when in the month the trigger should run. Possible values are ON_THE_DAY, LAST_DAY_OF_THE_MONTH, FIRST_WEEK, SECOND_WEEK, THIRD_WEEK, FOURTH_WEEK, LAST_WEEK. :param int day: day in the day when the trigger should run, when **run_on** is ON_THE_DAY or None :param int hour: hour in the day when the trigger should run, when **run_on** is ON_THE_DAY or None :param int minute: minute in the hour when the trigger should run, when **run_on** is ON_THE_DAY or None :param int minute_of_hour: position in the hour of the firing of the trigger :param int year: year part of the date/time before which the trigger won't run :param int month: month part of the date/time before which the trigger won't run :param string timezone: timezone in which the start date/time is expressed. Can be a time zone name like "Europe/Paris" or "SERVER" for the time zone of the DSS server """ starting_date = self.__get_starting_date__(year=year, month=month, day=day, default_is_today=False) trigger = {"active": True, "type": "temporal", "params": { "frequency": "Monthly", "startingFrom": starting_date, "hour": hour, "minute": minute, "monthlyRunOn": run_on, "repeatFrequency": repeat_every, "timezone": timezone }} self.raw_triggers.append(trigger) def __get_starting_date__(self, year, month, day, default_is_today=True): today = datetime.now(tzlocal()) if day is None or not isinstance(day, int) or day < 1 or day > 31: if default_is_today: day = today.day else: # Will be use to avoid Monthly trigger on the 31 of February if the day is not set day = 1 if month is None or not isinstance(month, int) or month < 1 or month > 12: if default_is_today: month = today.month else: # Will be use to avoid Monthly trigger on the 31 of February if the month is not set month = 1 if year is None or not isinstance(year, int): year = today.year start_date = today.replace(year=year, month=month, day=day, hour=0, minute=0, second=0, microsecond=0) return start_date.strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3] + start_date.strftime('%z') def save(self): """ Saves the settings to the scenario """ self.client._perform_json("PUT", "/projects/%s/scenarios/%s" % (self.scenario.project_key, self.scenario.id), body = self.data) class StepBasedScenarioSettings(DSSScenarioSettings): """ Settings of a step-based scenario. .. important:: Do not instantiate directly, use :meth:`DSSScenario.get_settings()`. """ @property def raw_steps(self): """ Returns raw definition of steps. This method returns a reference to the settings, not a copy. Modifying the settings then calling :meth:`save()` saves the changes made. :return: a list of scenario steps, each one a dict. Notable fields are: * **id** : identifier of the step (unique in the scenario) * **name** : label of the step * **type** : type of the step. There are many types, commonly used ones are build_flowitem, custom_python or exec_sql * **params** : type-specific parameters for the step, as a dict :rtype: list[dict] """ return self.data["params"]["steps"] class PythonScriptBasedScenarioSettings(DSSScenarioSettings): """ Settings of a scenario defined by a Python script. .. important:: Do not instantiate directly, use :meth:`DSSScenario.get_settings()`. """ def __init__(self, client, scenario, data, script): super(PythonScriptBasedScenarioSettings, self).__init__(client, scenario, data) self.script = script @property def code(self): """ Get the Python script of the scenario :rtype: string """ return self.script @code.setter def code(self, script): self.script = script def save(self): """ Saves the settings to the scenario. """ super(PythonScriptBasedScenarioSettings, self).save() self.client._perform_json("PUT", "/projects/%s/scenarios/%s/payload" % (self.scenario.project_key, self.scenario.id), body = {'script' : self.script}) class DSSScenarioRun(object): """ A handle containing basic info about a past run of a scenario. .. important:: Do not instantiate directly, use :meth:`DSSScenario.get_run()`, :meth:`DSSScenario.get_current_run()` or :meth:`DSSScenario.get_last_runs()` This handle can also be used to fetch additional information about the run. """ def __init__(self, client, run): self.client = client self.run = run @property def id(self): """ Get the identifier of this run. :rtype: string """ return self.run["runId"] def refresh(self): """ Refresh the details of the run. For ongoing scenario runs, this updates the set of outcomes of the steps and their results. .. note:: This method performs another API call """ updated_run_details = self.client._perform_json("GET", "/projects/%s/scenarios/%s/%s/" % \ (self.run["scenario"]["projectKey"], self.run["scenario"]["id"], self.run["runId"])) self.run = updated_run_details["scenarioRun"] def wait_for_completion(self, no_fail=False): """ Wait for the scenario run to complete. If the scenario run is already finished, this method returns immediately. :param boolean no_fail: if False, raises an exception if scenario fails """ while self.running: self.refresh() time.sleep(5) if self.outcome != 'SUCCESS' and no_fail == False: raise DataikuException("Scenario run returned status %s" % outcome) @property def running(self): """ Whether this scenario run is currently running. :rtype: boolean """ return not "result" in self.run @property def outcome(self): """ The outcome of this scenario run, if available. :return: one of SUCCESS, WARNING, FAILED, or ABORTED :rtype: string """ if not "result" in self.run: raise ValueError("outcome not available for this scenario run. Maybe still running?") return self.run["result"]["outcome"] @property def trigger(self): """ Get the trigger that triggered this scenario run. :return: the definition of a trigger, as in the :meth:`DSSScenarioSettings.raw_triggers()` list :rtype: dict """ return self.run["trigger"]["trigger"] def get_info(self): """ Get the raw information of the scenario run. :return: the scenario run, as a dict. The identifier of the run is a **runId** field. If the scenario run is finished, the detailed outcome of the run is a **result** sub-dict, with notably an **outcome** field (SUCCESS, WARNING, FAILED, or ABORTED) :rtype: dict """ return self.run def get_details(self): """ Get the full details of the scenario run. This includes notably the individual step runs inside the scenario run. .. note:: This method performs another API call :return: full details on a scenario run, as a dict with fields: * **scenarioRun** : the run definition and base status, as a dict (see :meth:`get_info()`) * **stepRuns** : details about each step that has executed so far, as a list of dicts (see :meth:`DSSScenarioRunDetails.steps()`) :rtype: dict """ raw_data = self.client._perform_json( "GET", "/projects/%s/scenarios/%s/%s/" % (self.run['scenario']['projectKey'], self.run['scenario']['id'], self.run['runId'])) details = DSSScenarioRunDetails(raw_data) if "stepRuns" in details: structured_steps = [] for step in details["stepRuns"]: structured_steps.append(DSSStepRunDetails(step)) details["stepRuns"] = structured_steps return details def get_start_time(self): """ Get the start time of the scenario run. :rtype: :class:`datetime.datetime` """ return _timestamp_ms_to_zoned_datetime(self.run['start']) start_time = property(get_start_time) def get_end_time(self): """ Get the end time of the scenario run, if it completed, else raises. :rtype: :class:`datetime.datetime` """ if "end" in self.run and self.run["end"] > 0: return _timestamp_ms_to_zoned_datetime(self.run['end']) else: raise ValueError("Scenario run has not completed") end_time = property(get_end_time) def get_duration(self): """ Get the duration of this run (in fractional seconds). If the run is still running, get the duration since it started. :rtype: float """ end_time = datetime.now(_local_timezone) if self.run['end'] > 0: end_time = _timestamp_ms_to_zoned_datetime(self.run['end']) duration = (end_time - self.get_start_time()).total_seconds() return duration duration = property(get_duration) class DSSScenarioRunDetails(dict): """ Details of a scenario run, notably the outcome of its steps. .. important:: Do not instantiate directly, see :meth:`DSSScenarioRun.get_details()` """ def __init__(self, data): super(DSSScenarioRunDetails, self).__init__(data) @property def steps(self): """ Get the list of runs of the steps. Only completed or ongoing steps are included. .. note:: When the instance of :class:`DSSScenarioRunDetails` was obtained via :meth:`DSSScenarioRun.get_details()`, then the returned list is made of instances of :class:`DSSStepRunDetails`. :return: a list of step runs, each as a dict. The **runId** in the dict is the identifier of the step run, not of the overall scenario run. A **result** sub-dict contains the outcome of the step. :rtype: list[dict] """ return self["stepRuns"] @property def last_step(self): """ Get the last step run. :return: a step run, as a dict. See :meth:`steps()`. :rtype: dict """ return self["stepRuns"][len(self["stepRuns"]) - 1] @property def first_error_details(self): """ Get the details of the first error if this run failed. This will not always be able to find the error details (it returns None in that case) :return: a serialized exception, as in :meth:`DSSStepRunDetails.first_error_details()` :rtype: dict """ for step in self.steps: step_error = step.first_error_details if step_error is not None: return step_error class DSSStepRunDetails(dict): """ Details of a run of a step in a scenario run. .. important:: Do not instantiate directly, see :meth:`DSSScenarioRunDetails.steps()` on an instance of :class:`DSSScenarioRunDetails` obtained via :meth:`DSSScenarioRun.get_details()` """ def __init__(self, data): super(DSSStepRunDetails, self).__init__(data) @property def outcome(self): """ Get the outcome of the step run/ :return: one of SUCCESS, WARNING, FAILED, or ABORTED :rtype: string """ return self["result"]["outcome"] @property def job_ids(self): """ Get the list of DSS job ids that were run as part of this step. :return: a list of job ids, each one a string :rtype: list[string] """ return [ri["jobId"] for ri in self["additionalReportItems"] if ri["type"] == "JOB_EXECUTED"] @property def first_error_details(self): """ Try to get the details of the first error if this step failed. This will not always be able to find the error details (it returns None in that case) :return: a serialized exception, as a with fields: * **clazz** : class name of the exception * **title** : short message of the exception * **message** : message of the exception * **stack** : stacktrace of the exception, as a single string * **code** : well-known error code, as listed in `the doc `_ * **fixability** : type of action to take in order to remediate the issue, if possible. For example USER_CONFIG_DATASET, ADMIN_SETTINGS_SECURITY, ... :rtype: dict """ if self.outcome == 'FAILED': step_thrown = self.get('result').get('thrown', None) if step_thrown is not None: return step_thrown for item in self['additionalReportItems']: if item.get("outcome", None) == 'FAILED': item_thrown = item.get('thrown', None) if item_thrown is not None: return item_thrown class DSSScenarioRunWaiter(object): """ Helper to wait for a scenario to run to complete. """ def __init__(self, scenario_run, trigger_fire): self.trigger_fire = trigger_fire self.scenario_run = scenario_run def wait(self, no_fail=False): """ Wait for the scenario run completion. :param boolean no_fail: if False, raises if the run doesn't end with a SUCCESS outcome :return: the final state of the scenario run (see :meth:`DSSScenarioRun.get_info()`) :rtype: dict """ while not self.scenario_run.run.get('result', False): self.scenario_run = self.trigger_fire.get_scenario_run() time.sleep(5) outcome = self.scenario_run.run.get('result', None).get('outcome', 'UNKNOWN') if outcome == 'SUCCESS' or no_fail: return self.scenario_run else: raise DataikuException("Scenario run returned status %s" % outcome) class DSSTriggerFire(object): """ A handle representing the firing of a trigger on a scenario. .. important:: Do not instantiate directly, use :meth:`DSSScenario.run` """ def __init__(self, scenario, trigger_fire): self.client = scenario.client self.project_key = scenario.project_key self.scenario_id = scenario.id self.trigger_id = trigger_fire['trigger']['id'] self.run_id = trigger_fire['runId'] self.trigger_fire = trigger_fire def get_raw(self): """ Get the definition of the trigger fire event. :return: the trigger fire, as a dict. The **runId** field in the dict is not an identifier of a scenario run, but of a run of the trigger. :rtype: dict """ def wait_for_scenario_run(self, no_fail=False): """ Poll for the run of the scenario that the trigger fire should initiate. This methos waits for the run of the sceanrio that this trigger activation launched to be available, or for the trigger fire to be cancelled (possibly cancelled by another trigger firing). :param boolean no_fail: if True, return None if the trigger fire is cancelled, else raise :return: a handle on a scenario run, or None :rtype: :class:`DSSScenarioRun` """ scenario_run = None refresh_trigger_counter = 0 while scenario_run is None: refresh_trigger_counter += 1 if refresh_trigger_counter == 10: refresh_trigger_counter = 0 if self.is_cancelled(refresh=refresh_trigger_counter == 0): if no_fail: return None else: raise DataikuException("Scenario run has been cancelled") scenario_run = self.get_scenario_run() time.sleep(5) return scenario_run def get_scenario_run(self): """ Get the run of the scenario that this trigger fire launched. May return None if the scenario run started from this trigger has not yet been created. :return: a handle on a scenario run, or None :rtype: :class:`DSSScenarioRun` """ run = self.client._perform_json( "GET", "/projects/%s/scenarios/%s/get-run-for-trigger" % (self.project_key, self.scenario_id), params= { 'triggerId' : self.trigger_id, 'triggerRunId' : self.run_id }) if 'scenarioRun' not in run: return None else: return DSSScenarioRun(self.client, run['scenarioRun']) def is_cancelled(self, refresh=False): """ Whether the trigger fire has been cancelled :param refresh: get the state of the trigger from the backend """ if refresh == True: self.trigger_fire = self.client._perform_json( "GET", "/projects/%s/scenarios/trigger/%s/%s" % (self.project_key, self.scenario_id, self.trigger_id), params={ 'triggerRunId' : self.run_id }) return self.trigger_fire["cancelled"] class DSSScenarioListItem(DSSTaggableObjectListItem): """ An item in a list of scenarios. .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.list_scenarios()` """ def __init__(self, client, data): super(DSSScenarioListItem, self).__init__(data) self.client = client def to_scenario(self): """ Get a handle corresponding to this scenario. :rtype: :class:`DSSScenario` """ return DSSScenario(self.client, self._data["projectKey"], self._data["id"]) @property def id(self): """ Get the identifier of the scenario. :rtype: string """ return self._data["name"] @property def running(self): """ Whether the scenario is currently running. :rtype: boolean """ return self._data["running"] @property def start_time(self): """ Get the start time of the scenario run. :return: timestap of the scenario run start, or None if it's not running at the moment. :rtype: :class:`datetime.datetime` """ return _timestamp_ms_to_zoned_datetime(self._data['start']) dataiku-api-client-python-13.2.2/dataikuapi/dss/sqlnotebook.py000066400000000000000000000225401471211575100244110ustar00rootroot00000000000000from .discussion import DSSObjectDiscussions from .utils import DSSTaggableObjectListItem class DSSSQLNotebookListItem(DSSTaggableObjectListItem): """ An item in a list of SQL notebooks .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.list_sql_notebooks` """ def __init__(self, client, data): super(DSSSQLNotebookListItem, self).__init__(data) self.client = client def to_notebook(self): """ Get a handle on the corresponding notebook :rtype: :class:`DSSSQLNotebook` """ return DSSSQLNotebook(self.client, self._data["projectKey"], self._data["id"]) @property def id(self): """ Get the id of the notebook Used as identifier """ return self._data["id"] @property def language(self): """ Get the language of the notebook Possible values are 'SQL', 'HIVE', 'IMPALA', 'SPARKSQL' :rtype: string """ return self._data["language"] @property def connection(self): """ Get the connection of the notebook :rtype: string """ return self._data["connection"] class DSSSQLNotebook(object): """ A handle on a SQL notebook .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.get_sql_notebook()` or :meth:`dataikuapi.dss.project.DSSProject.create_sql_notebook()` """ def __init__(self, client, project_key, notebook_id): self.client = client self.project_key = project_key self.notebook_id = notebook_id def get_content(self): """ Get the full content of this SQL notebook The content comprises cells Usage example: .. code-block:: python # collect all the SQL source code of a notebook lines = [] for cell in notebook.get_content().cells: if cell["type"] != "QUERY": continue lines = lines + cell["code"] print('\\n'.join(lines)) :rtype: :class:`dataikuapi.dss.sqlnotebook.DSSNotebookContent` """ raw_content = self.client._perform_json("GET", "/projects/%s/sql-notebooks/%s" % (self.project_key, self.notebook_id)) return DSSNotebookContent(self.client, self.project_key, self.notebook_id, raw_content) def get_history(self): """ Get the history of this SQL notebook :rtype: :class:`dataikuapi.dss.sqlnotebook.DSSNotebookHistory` """ raw_history = self.client._perform_json("GET", "/projects/%s/sql-notebooks/%s/history" % (self.project_key, self.notebook_id)) return DSSNotebookHistory(self.client, self.project_key, self.notebook_id, raw_history) def clear_history(self, cell_id=None, num_runs_to_retain=0): """ Clear the history of this SQL notebook :param string cell_id: The cell id for which to clear the history. Leave unspecified to clear the history for all cells of this notebook :param int num_runs_to_retain: The number of the most recent runs to retain for each cell (defaults to **0**) """ self.client._perform_empty("POST", "/projects/%s/sql-notebooks/%s/history/clear" % (self.project_key, self.notebook_id), body={"cellId": cell_id, "numRunsToRetain": num_runs_to_retain}) def delete(self): """ Delete this SQL notebook """ return self.client._perform_json("DELETE", "/projects/%s/sql-notebooks/%s" % (self.project_key, self.notebook_id)) ######################################################## # Discussions ######################################################## def get_object_discussions(self): """ Get a handle to manage discussions on the notebook :return: The handle to manage discussions :rtype: :class:`dataikuapi.dss.discussion.DSSObjectDiscussions` """ return DSSObjectDiscussions(self.client, self.project_key, "SQL_NOTEBOOK", self.notebook_id) class DSSNotebookContent(object): """ The content of a SQL notebook .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.sqlnotebook.DSSSQLNotebook.get_content` """ def __init__(self, client, project_key, notebook_id, content): self.client = client self.project_key = project_key self.notebook_id = notebook_id self.content = content def get_raw(self): """ Get the raw content of this SQL notebook The content comprises cells :return: A dict containing the full content of a notebook. Notable fields are: * **cells**: list of cells, each one is a dict (see :attr:`cells`) :rtype: dict """ return self.content @property def connection(self): """ Get the connection of this SQL notebook :rtype: string """ return self.content["connection"] @connection.setter def connection(self, connection): """ Set the connection of this SQL notebook :param string connection: The new connection """ self.content["connection"] = connection @property def cells(self): """ Get the cells of this SQL notebook :return: A list of cells. Each cell is a dict, with notable fields: * **id**: id of the cell * **type**: type of the cell, for example "QUERY" or "MARKDOWN" * **name**: name of the cell * **code**: content of the cell :rtype: list[dict] """ return self.content["cells"] @cells.setter def cells(self, cells): """ Set the cells of this SQL notebook :param list[dict] cells: The new cells """ self.content["cells"] = cells def save(self): """ Save the content of this SQL notebook """ return self.client._perform_json("PUT", "/projects/%s/sql-notebooks/%s" % (self.project_key, self.notebook_id), body=self.content) class DSSNotebookHistory(object): """ The history of a SQL notebook .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.sqlnotebook.DSSSQLNotebook.get_history` """ def __init__(self, client, project_key, notebook_id, history): self.client = client self.project_key = project_key self.notebook_id = notebook_id self.history = {cell_id: {qr["id"]: qr for qr in query_runs} for cell_id, query_runs in history.items()} def get_query_runs(self, cell_id, as_type="listitems"): """ Get the query runs of a cell in this SQL notebook Usage example: .. code-block:: python # delete query runs which failed for query_run in notebook_history.get_query_runs(cell_id): if query_run.get_raw()["state"] == "FAILED": query_run.delete() notebook_history.save() :param string cell_id: The cell id for which to retrieve the query runs :param string as_type: How to return the list. Currently the only supported (and default) value is "listitems" :return: The list of query runs :rtype: List of :class:`DSSNotebookQueryRunListItem` """ if as_type == "listitems": return [DSSNotebookQueryRunListItem(self.history[cell_id], qr) for qr in self.history[cell_id].values()] else: raise ValueError("Unknown as_type") def save(self): """ Save the history of this SQL notebook """ self.client._perform_empty("PUT", "/projects/%s/sql-notebooks/%s/history" % (self.project_key, self.notebook_id), body={cell_id: list(query_runs.values()) for cell_id, query_runs in self.history.items()}) class DSSNotebookQueryRunListItem(object): """ An item in a list of query runs of a SQL notebook .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.sqlnotebook.DSSNotebookHistory.get_query_result` """ def __init__(self, query_runs, query_run): self.query_runs = query_runs self.query_run = query_run def get_raw(self): """ Get the raw query run list item :returns: A dict containing the query run list item data. Notable fields are: * **runOn**: timestamp of the query run * **runBy**: user login of the query run * **state**: state of the query run, for example "DONE" or "FAILED" * **sql**: SQL code of the query run, with variables unexpanded * **expandedSql**: SQL code of the query run, with variables expanded :rtype: dict """ return self.query_run def delete(self): """ Delete the query run .. note:: The history must be then saved using :meth:`dataikuapi.dss.sqlnotebook.DSSNotebookHistory.save` """ del self.query_runs[self.query_run["id"]] dataiku-api-client-python-13.2.2/dataikuapi/dss/sqlquery.py000066400000000000000000000106251471211575100237370ustar00rootroot00000000000000from ..utils import DataikuException from ..utils import DataikuUTF8CSVReader from ..utils import DataikuStreamedHttpUTF8CSVReader import json class DSSSQLQuery(object): """ A connection to a database or database-like on which queries can be run through DSS. .. important:: Do not create this class directly, instead use :meth:`dataikuapi.DSSClient.sql_query` Usage example: .. code-block:: python # run some query on a connection query = client.sql_query('select * from "public"."SOME_TABLE"', connection='some_postgres_connection') n = 0 for row in query.iter_rows(): n += 1 if n < 10: print("row %s : %s" % (n, row)) query.verify() print("Returned %s rows" % n) """ def __init__(self, client, query, connection, database, dataset_full_name, pre_queries, post_queries, type, extra_conf, script_steps, script_input_schema, script_output_schema, script_report_location, read_timestamp_without_timezone_as_string, read_date_as_string, project_key): self.client = client self.streaming_session = self.client._perform_json( "POST", "/sql/queries/", body = { "query" : query, "preQueries" : pre_queries, "postQueries" : post_queries, "connection" : connection, "database" : database, "datasetFullName" : dataset_full_name, "projectKey" : project_key, "type" : type, "extraConf" : extra_conf, "scriptSteps" : script_steps, "scriptInputSchema" : script_input_schema, "scriptOutputSchema" : script_output_schema, "scriptReportLocation" : script_report_location, "readTimestampWithoutTimezoneAsString" : read_timestamp_without_timezone_as_string, "readDateAsString" : read_date_as_string }) self.queryId = self.streaming_session['queryId'] def get_schema(self): """ Get the query's result set's schema. The schema made of DSS column types, and built from mapping database types to DSS types. The actual type in the database can be found in the `originalType` field (`originalSQLType` in BigQuery) :return: a schema, as a dict with a `columns` array, in which each element is a column, itself as a dict of * **name** : the column name * **type** : the column type (smallint, int, bigint, float, double, boolean, date, string) * **length** : the string length * **comment** : the column name * **originalType** : type of the column in the database :rtype: dict """ return self.streaming_session['schema'] def iter_rows(self): """ Get an iterator on the query's results. :return: an iterator over the rows, each row being a tuple of values. The order of values in the tuples is the same as the order of columns in the schema returned by :meth:`~get_schema()`. The values are cast to python types according to the types in :meth:`~get_schema()` :rtype: iterator[list] """ csv_stream = self.client._perform_raw( "GET", "/sql/queries/%s/stream" % (self.queryId), params = { "format" : "tsv-excel-noheader" }) return DataikuStreamedHttpUTF8CSVReader(self.get_schema(), csv_stream).iter_rows() def verify(self): """ Verify that reading results completed successfully. When using the :meth:`~iter_rows()` method, and the iterator stops returning rows, there is no way to tell whether there are no more rows because the query didn't return more rows, or because an error in the query, or in the fetching of its results, happened. You should thus call :meth:`~verify()` after the iterator is done, because it will raise an Exception if an error happened. :raises: Exception """ resp = self.client._perform_empty( "GET", "/sql/queries/%s/finish-streaming" % (self.queryId)) # exception raising is done in _perform_empty()dataiku-api-client-python-13.2.2/dataikuapi/dss/statistics.py000066400000000000000000000246021471211575100242440ustar00rootroot00000000000000from .utils import DSSDatasetSelectionBuilder from .future import DSSFuture class DSSStatisticsWorksheet(object): """ A handle to interact with a worksheet. .. important:: Do not create this class directly, instead use :meth:`dataikuapi.dss.dataset.DSSDataset.get_statistics_worksheet` """ def __init__(self, client, project_key, dataset_name, worksheet_id): self.client = client self.project_key = project_key self.dataset_name = dataset_name self.worksheet_id = worksheet_id def delete(self): """ Deletes the worksheet. """ self.client._perform_empty( "DELETE", "/projects/%s/datasets/%s/statistics/worksheets/%s" % (self.project_key, self.dataset_name, self.worksheet_id)) def get_settings(self): """ Gets the worksheet settings. :returns: a handle for the worksheet settings :rtype: :class:`DSSStatisticsWorksheetSettings` """ worksheet_json = self.client._perform_json( "GET", "/projects/%s/datasets/%s/statistics/worksheets/%s" % ( self.project_key, self.dataset_name, self.worksheet_id) ) return DSSStatisticsWorksheetSettings(self.client, self.project_key, self.dataset_name, self.worksheet_id, worksheet_json) def run_worksheet(self, wait=True): """ Computes the result of the whole worksheet. When `wait` is `True`, the method waits for the computation to complete and returns the corresponding result, otherwise it returns immediately a handle for a :class:`~dataikuapi.dss.future.DSSFuture` result. :param bool wait: a flag to wait for the computation to complete (defaults to **True**) :returns: the corresponding card result or a handle for the future result :rtype: :class:`~DSSStatisticsCardResult` or :class:`~dataikuapi.dss.future.DSSFuture` """ root_card = self.get_settings().get_raw()['rootCard'] return self.run_card(root_card, wait=wait) def run_card(self, card, wait=True): """ Computes the result of a card in the context of the worksheet. When `wait` is `True`, the method waits for the computation to complete and returns the corresponding result, otherwise it returns immediately a handle for a :class:`~dataikuapi.dss.future.DSSFuture` result. .. note:: The card does not need to belong to the worksheet. :param card: the card to compute :type card: :class:`~DSSStatisticsCardSettings` or dict obtained from :meth:`DSSStatisticsCardSettings.get_raw` :param bool wait: a flag to wait for the computations to complete (defaults to **True**) :returns: the corresponding card result or a handle for the future result :rtype: :class:`DSSStatisticsCardResult` or :class:`~dataikuapi.dss.future.DSSFuture` """ card = DSSStatisticsCardSettings._from_card_or_dict(self.client, card) future_response = self.client._perform_json( "POST", "/projects/%s/datasets/%s/statistics/worksheets/%s/actions/run-card" % ( self.project_key, self.dataset_name, self.worksheet_id), body=card.get_raw() ) future = DSSFuture(self.client, future_response.get("jobId", None), future_response, result_wrapper=lambda raw_result: DSSStatisticsCardResult(raw_result)) return future.wait_for_result() if wait else future def run_computation(self, computation, wait=True): """ Runs a computation in the context of the worksheet. When `wait` is `True`, the method waits for the computation to complete and returns the corresponding result, otherwise it returns immediately a handle for a :class:`~dataikuapi.dss.future.DSSFuture` result. :param computation: the computation to perform :type computation: :class:`DSSStatisticsComputationSettings` or dict obtained from :meth:`DSSStatisticsComputationSettings.get_raw` :param bool wait: a flag to wait for the computations to complete (defaults to **True**) :returns: the corresponding computation result or a handle for the future result :returns: :class:`DSSStatisticsComputationResult` or :class:`~dataikuapi.dss.future.DSSFuture` """ computation = DSSStatisticsComputationSettings._from_computation_or_dict( computation) future_response = self.client._perform_json( "POST", "/projects/%s/datasets/%s/statistics/worksheets/%s/actions/run-computation" % ( self.project_key, self.dataset_name, self.worksheet_id), body=computation.get_raw() ) future = DSSFuture(self.client, future_response.get("jobId", None), future_response, result_wrapper=lambda raw_result: DSSStatisticsComputationResult(raw_result)) return future.wait_for_result() if wait else future class DSSStatisticsWorksheetSettings(object): """ A handle to interact with the worksheet settings. .. important:: Do not create this class directly, instead use :meth:`DSSStatisticsWorksheet.get_settings` """ def __init__(self, client, project_key, dataset_name, worksheet_id, worksheet_definition): self._worksheet_definition = worksheet_definition self.client = client self.project_key = project_key self.dataset_name = dataset_name self.worksheet_id = worksheet_id def add_card(self, card): """ Adds a new card to the worksheet. :param card: the card to add :type card: :class:`DSSStatisticsCardSettings` or dict obtained from :meth:`DSSStatisticsCardSettings.get_raw` """ card = DSSStatisticsCardSettings._from_card_or_dict(self.client, card) self._worksheet_definition['rootCard']['cards'].append(card.get_raw()) def list_cards(self): """ Lists the cards for the worksheet. :returns: a list of card handles :rtype: list of :class:`DSSStatisticsCardSettings` """ return [DSSStatisticsCardSettings(self.client, card_definition) for card_definition in self._worksheet_definition['rootCard']['cards']] def get_raw(self): """ Gets a reference to the raw representation of the worksheet settings. :returns: the worksheet settings :rtype: dict """ return self._worksheet_definition def set_sampling_settings(self, selection): """ Sets the worksheet sampling settings. :param selection: the sampling settings :type selection: :class:`~dataikuapi.dss.utils.DSSDatasetSelectionBuilder` or dict obtained from :meth:`get_raw_sampling_settings` """ raw_selection = selection.build() if isinstance( selection, DSSDatasetSelectionBuilder) else selection self._worksheet_definition['dataSpec']['datasetSelection'] = raw_selection def get_raw_sampling_settings(self): """ Gets a reference to the raw representation of the worksheet sampling settings. :returns: the sampling settings :rtype: dict """ return self._worksheet_definition['dataSpec']['datasetSelection'] def save(self): """ Saves the settings of the worksheet. """ self._worksheet_definition = self.client._perform_json( "PUT", "/projects/%s/datasets/%s/statistics/worksheets/%s" % ( self.project_key, self.dataset_name, self.worksheet_id), body=self._worksheet_definition ) class DSSStatisticsCardSettings(object): """ A handle to interact with the card settings. """ def __init__(self, client, card_definition): self.client = client self._card_definition = card_definition def get_raw(self): """ Gets a reference to the raw representation of the card settings. :returns: the card settings :rtype: dict """ return self._card_definition def compile(self): """ Gets the computation used to compute the card result. :returns: the computation settings :rtype: :class:`DSSStatisticsComputationSettings` """ computation_json = self.client._perform_json( "POST", "/statistics/cards/compile", body=self._card_definition ) return DSSStatisticsComputationSettings(computation_json) @staticmethod def _from_card_or_dict(client, card_or_dict): if isinstance(card_or_dict, DSSStatisticsCardSettings): card_or_dict = card_or_dict.get_raw() return DSSStatisticsCardSettings(client, card_or_dict) class DSSStatisticsCardResult(object): """ A handle to interact with the computed result of a :class:`DSSStatisticsCardSettings`. """ def __init__(self, card_result): self._card_result = card_result def get_raw(self): """ Gets a reference to the raw representation of the card result. :returns: the card result :rtype: dict """ return self._card_result class DSSStatisticsComputationSettings(object): """ A handle to interact with the computation settings. """ def __init__(self, computation_definition): self._computation_definition = computation_definition def get_raw(self): """ Gets a reference to the raw representation of the computation settings. :returns: the computation settings :rtype: dict """ return self._computation_definition @staticmethod def _from_computation_or_dict(computation_or_dict): if isinstance(computation_or_dict, DSSStatisticsComputationSettings): computation_or_dict = computation_or_dict.get_raw() return DSSStatisticsComputationSettings(computation_or_dict) class DSSStatisticsComputationResult(object): """ A handle to interact with the computed result of a :class:`DSSStatisticsComputationSettings`. """ def __init__(self, computation_result): self._computation_result = computation_result def get_raw(self): """ Gets a reference to the raw representation of the computation result. :returns: the computation result :rtype: dict """ return self._computation_result dataiku-api-client-python-13.2.2/dataikuapi/dss/streaming_endpoint.py000066400000000000000000000661231471211575100257470ustar00rootroot00000000000000from ..utils import DataikuException from .future import DSSFuture import json, warnings, logging from .utils import DSSTaggableObjectListItem from .future import DSSFuture from .discussion import DSSObjectDiscussions from . import recipe try: basestring except NameError: basestring = str class DSSStreamingEndpointListItem(DSSTaggableObjectListItem): """ An item in a list of streaming endpoints. .. important:: Do not instantiate this class, use :meth:`.DSSProject.list_streaming_endpoints()` instead """ def __init__(self, client, data): super(DSSStreamingEndpointListItem, self).__init__(data) self.client = client def __repr__(self): return u"<{} {}.{}>".format(self.__class__.__name__, self._data["projectKey"], self._data["id"]) def to_streaming_endpoint(self): """ Get a handle on the corresponding streaming endpoint. :rtype: :class:`DSSStreamingEndpoint` """ return DSSStreamingEndpoint(self.client, self._data["projectKey"], self._data["id"]) @property def name(self): """ Get the streaming endpoint name. :rtype: string """ return self._data["id"] @property def id(self): """ Get the streaming endpoint identifier. .. note:: For streaming endpoints, the identifier is equal to its name :rtype: string """ return self._data["id"] @property def type(self): """ Get the streaming endpoint type. :returns: the type is the same as the type of the connection that the streaming endpoint uses, ie `Kafka`, `SQS`, `KDBPlus` or `httpsse` :rtype: string """ return self._data["type"] @property def schema(self): """ Get the schema of the streaming endpoint. Usage example: .. code-block:: python # list all endpoints containing columns with a 'PII' in comment for endpoint in p.list_streaming_endpoints(): column_list = endpoint.schema['columns'] pii_columns = [c for c in column_list if 'PII' in c.get("comment", "")] if len(pii_columns) > 0: print("Streaming endpoint %s contains %s PII columns" % (endpoint.id, len(pii_columns))) :returns: a schema, as a dict with a `columns` array, in which each element is a column, itself as a dict of * **name** : the column name * **type** : the column type (smallint, int, bigint, float, double, boolean, date, string) * **length** : the string length * **comment** : user comment about the column :rtype: dict """ return self._data["schema"] @property def connection(self): """ Get the connection on which this streaming endpoint is attached. :returns: a connection name, or None for HTTP SSE endpoints :rtype: string """ if not "params" in self._data: return None return self._data["params"].get("connection", None) def get_column(self, column): """ Get a column in the schema, by its name. :param string column: name of column to find :returns: a dict of the column settings or None if column does not exist. Fields are: * **name** : the column name * **type** : the column type (smallint, int, bigint, float, double, boolean, date, string) * **length** : the string length * **comment** : user comment about the column :rtype: dict """ matched = [col for col in self.schema["columns"] if col["name"] == column] return None if len(matched) == 0 else matched[0] class DSSStreamingEndpoint(object): """ A streaming endpoint on the DSS instance. """ def __init__(self, client, project_key, streaming_endpoint_name): self.client = client self.project = client.get_project(project_key) self.project_key = project_key self.streaming_endpoint_name = streaming_endpoint_name @property def name(self): """ Get the streaming endpoint name. :rtype: string """ return self.streaming_endpoint_name @property def id(self): """ Get the streaming endpoint identifier. .. note:: For streaming endpoints, the identifier is equal to its name :rtype: string """ return self.streaming_endpoint_name ######################################################## # Streaming Endpoint deletion ######################################################## def delete(self): """ Delete the streaming endpoint from the flow, and objects using it (recipes or continuous recipes) .. attention:: This call doesn't delete the underlying streaming data. For example for a Kafka streaming endpoint the topic isn't deleted """ return self.client._perform_empty( "DELETE", "/projects/%s/streamingendpoints/%s" % (self.project_key, self.streaming_endpoint_name), params = { }) ######################################################## # Streaming Endpoint definition ######################################################## def get_settings(self): """ Get the settings of this streaming endpoint. Know subclasses of :class:`DSSStreamingEndpointSettings` include :class:`KafkaStreamingEndpointSettings` and :class:`HTTPSSEStreamingEndpointSettings` You must use :meth:`~DSSStreamingEndpointSettings.save()` on the returned object to make your changes effective on the streaming endpoint. .. code-block:: python # Example: changing the topic on a kafka streaming endpoint streaming_endpoint = project.get_streaming_endpoint("my_endpoint") settings = streaming_endpoint.get_settings() settings.set_connection_and_topic(None, "country") settings.save() :returns: an object containing the settings :rtype: :class:`DSSStreamingEndpointSettings` or a subclass """ data = self.client._perform_json("GET", "/projects/%s/streamingendpoints/%s" % (self.project_key, self.streaming_endpoint_name)) if data["type"] in ['kafka']: return KafkaStreamingEndpointSettings(self, data) elif data["type"] in ['httpsse']: return HTTPSSEStreamingEndpointSettings(self, data) else: return DSSStreamingEndpointSettings(self, data) def exists(self): """ Whether this streaming endpoint exists in DSS. :rtype: boolean """ try: self.get_schema() return True except Exception as e: return False ######################################################## # Streaming Endpoint metadata ######################################################## def get_schema(self): """ Get the schema of the streaming endpoint. :returns: a schema, as a dict with a `columns` array, in which each element is a column, itself as a dict of * **name** : the column name * **type** : the column type (smallint, int, bigint, float, double, boolean, date, string) * **comment** : user comment about the column :rtype: dict """ return self.client._perform_json( "GET", "/projects/%s/streamingendpoints/%s/schema" % (self.project_key, self.streaming_endpoint_name)) def set_schema(self, schema): """ Set the schema of the streaming endpoint. Usage example: .. code-block:: python # copy schema of the input of a continuous recipe to its output recipe = p.get_recipe('my_recipe_name') recipe_settings = recipe.get_settings() input_endpoint = p.get_streaming_endpoint(recipe_settings.get_flat_input_refs()[0]) output_endpoint = p.get_streaming_endpoint(recipe_settings.get_flat_output_refs()[0]) output_endpoint.set_schema(input_endpoint.get_schema()) :param dict schema: a schema, as a dict with a `columns` array, in which each element is a column, itself as a dict of * **name** : the column name * **type** : the column type (smallint, int, bigint, float, double, boolean, date, string) * **comment** : user comment about the column """ return self.client._perform_json( "PUT", "/projects/%s/streamingendpoints/%s/schema" % (self.project_key, self.streaming_endpoint_name), body=schema) ######################################################## # Misc ######################################################## def get_zone(self): """ Get the flow zone of this streaming endpoint. :returns: a flow zone :rtype: :class:`dataikuapi.dss.flow.DSSFlowZone` """ return self.project.get_flow().get_zone_of_object(self) def move_to_zone(self, zone): """ Move this object to a flow zone. :param object zone: a :class:`dataikuapi.dss.flow.DSSFlowZone` where to move the object, or its identifier """ if isinstance(zone, basestring): zone = self.project.get_flow().get_zone(zone) zone.add_item(self) def share_to_zone(self, zone): """ Share this object to a flow zone. :param object zone: a :class:`dataikuapi.dss.flow.DSSFlowZone` where to share the object, or its identifier """ if isinstance(zone, basestring): zone = self.project.get_flow().get_zone(zone) zone.add_shared(self) def unshare_from_zone(self, zone): """ Unshare this object from a flow zone. :param object zone: a :class:`dataikuapi.dss.flow.DSSFlowZone` from where to unshare the object, or its identifier """ if isinstance(zone, basestring): zone = self.project.get_flow().get_zone(zone) zone.remove_shared(self) def get_usages(self): """ Get the recipes referencing this streaming endpoint. Usage example: .. code-block:: python for usage in streaming_endpoint.get_usages(): if usage["type"] == 'RECIPE_INPUT': print("Used as input of %s" % usage["objectId"]) :returns: a list of usages, each one a dict of: * **type** : the type of usage, either "RECIPE_INPUT" or "RECIPE_OUTPUT" * **objectId** : name of the recipe or continuous recipe * **objectProjectKey** : project of the recipe or continuous recipe :rtype: list[dict] """ return self.client._perform_json("GET", "/projects/%s/streamingendpoints/%s/usages" % (self.project_key, self.streaming_endpoint_name)) ######################################################## # Discussions ######################################################## def get_object_discussions(self): """ Get a handle to manage discussions on the streaming endpoint. :returns: the handle to manage discussions :rtype: :class:`dataikuapi.dss.discussion.DSSObjectDiscussions` """ return DSSObjectDiscussions(self.client, self.project_key, "STREAMING_ENDPOINT", self.streaming_endpoint_name) ######################################################## # Test / Autofill ######################################################## def test_and_detect(self, infer_storage_types=False, limit=10, timeout=60): """ Used internally by :meth:`autodetect_settings`. It is not usually required to call this method .. attention:: Only Kafka and HTTP-SSE streaming endpoints are handled .. note:: Schema inferrence is done on the captured rows from the underlying stream. If no record is captured, for example because no message is posted to the stream while the capture is done, then no schema inferrence can take place. :returns: a future object on the result of the detection. The future's result is a dict with fields: * **table** : the captured rows * **schemaDetection** : the result of the schema inference. Notable sub-fields are * **detectedSchema** : the inferred schema * **detectedButNotInSchema** : list of column names found in the capture, but not yet in the endpoint's schema * **inSchemaButNotDetected** : list of column names present in the endpoint's schema, but not found in the capture :rtype: :class:`dataikuapi.dss.future.DSSFuture` """ settings = self.get_settings() if settings.type in ['kafka']: future_resp = self.client._perform_json("POST", "/projects/%s/streamingendpoints/%s/actions/testAndDetectSettings/kafka"% (self.project_key, self.streaming_endpoint_name), body = { "inferStorageTypes" : infer_storage_types , "limit" : limit, "timeout" : timeout }) return DSSFuture(self.client, future_resp.get('jobId', None), future_resp) elif settings.type in ['httpsse']: future_resp = self.client._perform_json("POST", "/projects/%s/streamingendpoints/%s/actions/testAndDetectSettings/httpsse"% (self.project_key, self.streaming_endpoint_name), body = { "inferStorageTypes" : infer_storage_types , "limit" : limit, "timeout" : timeout }) return DSSFuture(self.client, future_resp.get('jobId', None), future_resp) else: raise ValueError("don't know how to test/detect on streaming endpoint type:%s" % settings.type) def autodetect_settings(self, infer_storage_types=False, limit=10, timeout=60): """ Detect an appropriate schema for this streaming endpoint using Dataiku detection engine. The detection bases itself on a capture of the stream that this streaming endpoint represents. First a number of messages are captured from the stream, within the bounds passed as parameters, then a detection of the columns and their types is done on the captured messages. If no message is send on the stream during the capture, no column is inferred. .. attention:: Only Kafka and HTTP-SSE streaming endpoints are handled .. note:: Format-related settings are not automatically inferred. For example the method will not detect whether Kafka messages are JSON-encoded or Avro-encoded Usage example: .. code-block:: python # create a kafka endpoint on a json stream and detect its schema e = project.create_kafka_streaming_endpoint('test_endpoint', 'kafka_connection', 'kafka-topic') s = e.autodetect_settings(infer_storage_types=True) s.save() :param boolean infer_storage_types: if True, DSS will try to guess types of the columns. If False, all columns will be assumed to be strings (default: False) :param int limit: max number of rows to use for the autodetection (default: 10) :param int timeout: max duration in seconds of the stream capture to use for the autodetection (default: 60) :returns: streaming endpoint settings with an updated schema that you can :meth:`DSSStreamingEndpointSettings.save`. :rtype: :class:`DSSStreamingEndpointSettings` or a subclass """ settings = self.get_settings() if settings.type in ['kafka']: future = self.test_and_detect(infer_storage_types, limit, timeout) result = future.wait_for_result() if result.get("schemaDetection", {}).get("detectedSchema") is None: logging.warn("No schema detected, leaving existing schema") else: settings.get_raw()["schema"] = result["schemaDetection"]["detectedSchema"] return settings elif settings.type in ['httpsse']: future = self.test_and_detect(infer_storage_types, limit, timeout) result = future.wait_for_result() if result.get("schemaDetection", {}).get("detectedSchema") is None: logging.warn("No schema detected, leaving existing schema") else: settings.get_raw()["schema"] = result["schemaDetection"]["detectedSchema"] return settings else: raise ValueError("don't know how to test/detect on streaming endpoint type:%s" % settings.type) def get_as_core_streaming_endpoint(self): """ Get the streaming endpoint as a handle for use inside DSS. :returns: the streaming endpoint :rtype: :class:`dataiku.StreamingEndpoint` """ import dataiku return dataiku.StreamingEndpoint("%s.%s" % (self.project_key, self.streaming_endpoint_name)) ######################################################## # Creation of recipes ######################################################## def new_code_recipe(self, type, code=None, recipe_name=None): """ Start the creation of a new code recipe taking this streaming endpoint as input. Usage example: .. code-block:: python # create a continuous python recipe from an endpoint recipe_creator = endpoint.new_code_recipe('cpython', 'some code here', 'compute_something') recipe_creator.with_new_output_streaming_endpoint('something', 'my_kafka_connection', 'avro') recipe = recipe_creator.create() :param string type: type of the recipe. Can be 'cpython', 'streaming_spark_scala' or 'ksql'; the non-continuous type 'python' is also possible :param string code: (optional) The script of the recipe :param string recipe_name: (optional) base name for the new recipe. :returns: an object to create a new recipe :rtype: :class:`dataikuapi.dss.recipe.CodeRecipeCreator` """ builder = recipe.CodeRecipeCreator(recipe_name, type, self.project) builder.with_input(self.streaming_endpoint_name) if code is not None: builder.with_script(code) return builder def new_recipe(self, type, recipe_name=None): """ Start the creation of a new recipe taking this streaming endpoint as input. This method can create non-code recipes like Sync or continuous Sync recipes. For more details, please see :meth:`dataikuapi.dss.project.DSSProject.new_recipe`. Usage example: .. code-block:: python # create a continuous sync from an endpoint to a dataset recipe_creator = endpoint.new_recipe('csync', 'compute_my_dataset_name') recipe_creator.with_new_output('my_dataset_name', 'filesystem_managed') recipe = recipe_creator.create() :param string type: type of the recipe. Possible values are 'csync', 'cpython', 'python', 'ksql', 'streaming_spark_scala' :param string recipe_name: (optional) base name for the new recipe. :returns: A new DSS Recipe Creator handle :rtype: :class:`dataikuapi.dss.recipe.DSSRecipeCreator` or a subclass """ builder = self.project.new_recipe(type=type, name=recipe_name) builder.with_input(self.streaming_endpoint_name) return builder class DSSStreamingEndpointSettings(object): """ Base settings class for a DSS streaming endpoint. .. important:: Do not instantiate this class directly, use :meth:`DSSStreamingEndpoint.get_settings` Use :meth:`save` to save your changes """ def __init__(self, streaming_endpoint, settings): self.streaming_endpoint = streaming_endpoint self.settings = settings def get_raw(self): """ Get the streaming endpoint settings. :returns: the settings, as a dict. The type-specific parameters, that depend on the connection type, are a **params** sub-dict. :rtype: dict """ return self.settings def get_raw_params(self): """ Get the type-specific (Kafka/ HTTP-SSE/ Kdb+...) params as a dict. :returns: the type-specific params. Each type defines a set of fields; commonly found fields are : * **connection** : name of the connection used by the streaming endpoint * **topic** or **queueName** : name of the Kafka topic or SQS queue corresponding to this streaming endpoint :rtype: dict """ return self.settings["params"] @property def type(self): """ Get the type of streaming system that the streaming endpoint uses. :return: a type of streaming system. Possible values: 'kafka', 'SQS', 'kdbplustick', 'httpsse' :rtype: string """ return self.settings["type"] def add_raw_schema_column(self, column): """ Add a column in the schema :param dict column: a dict defining the column. It should contain a "name" field and a "type" field (with a DSS type, like bigint, double, string, date, boolean, ...), optionally a "length" field for string-typed columns and a "comment" field. """ self.settings["schema"]["columns"].append(column) def save(self): """ Save the changes to the settings on the streaming endpoint. """ self.streaming_endpoint.client._perform_empty( "PUT", "/projects/%s/streamingendpoints/%s" % (self.streaming_endpoint.project_key, self.streaming_endpoint.streaming_endpoint_name), body=self.settings) class KafkaStreamingEndpointSettings(DSSStreamingEndpointSettings): """ Settings for a Kafka streaming endpoint. This class inherits from :class:`DSSStreamingEndpointSettings`. .. important:: Do not instantiate this class directly, use :meth:`DSSStreamingEndpoint.get_settings` Use :meth:`save` to save your changes """ def __init__(self, streaming_endpoint, settings): super(KafkaStreamingEndpointSettings, self).__init__(streaming_endpoint, settings) def set_connection_and_topic(self, connection=None, topic=None): """ Change the connection and topic of an endpoint. :param string connection: (optional) name of a Kafka connection in DSS :param string topic: (optional) name of a Kafka topic. Can contain DSS variables """ if connection is not None: self.settings["params"]["connection"] = connection if topic is not None: self.settings["params"]["topic"] = topic class HTTPSSEStreamingEndpointSettings(DSSStreamingEndpointSettings): """ Settings for a HTTP-SSE streaming endpoint. This class inherits from :class:`DSSStreamingEndpointSettings`. .. important:: Do not instantiate this class directly, use :meth:`DSSStreamingEndpoint.get_settings` Use :meth:`save` to save your changes """ def __init__(self, streaming_endpoint, settings): super(HTTPSSEStreamingEndpointSettings, self).__init__(streaming_endpoint, settings) def set_url(self, url): """ Change the URL of the endpoint. :param string url: url to connect to """ self.settings["params"]["url"] = url class DSSManagedStreamingEndpointCreationHelper(object): """ Utility class to help create a new streaming endpoint. .. note:: Do not instantiate directly, use :meth:`.DSSProject.new_managed_streaming_endpoint()` instead .. important:: Only Kafka and SQS endpoints support managed streaming endpoints """ def __init__(self, project, streaming_endpoint_name, streaming_endpoint_type): self.project = project self.streaming_endpoint_name = streaming_endpoint_name self.streaming_endpoint_type = streaming_endpoint_type self.creation_settings = {} def get_creation_settings(self): """ Get the settings for the creation of the new managed streaming endpoint. .. note:: Modifying the values in the creation settings directly is discouraged, use :meth:`with_store_into()` instead. :returns: the settings as a dict :rtype: dict """ return self.creation_settings def with_store_into(self, connection, format_option_id = None): """ Set the DSS connection underlying the new streaming endpoint. :param string connection: name of the connection to store into :param string format_option_id: (optional) identifier of a serialization format option. For Kafka endpoints, possible values are : * **json** : messages are JSON strings * **single** : messages are handled as a single typed field * **avro** : messages are Kafka-Avro messages (ie an avro message padded with a field indicating the avro schema version in the kafka schema registry) For SQS endpoints, possible values are * **json** : messages are JSON strings * **string** : messages are raw strings :returns: self """ self.creation_settings["connectionId"] = connection if format_option_id is not None: self.creation_settings["formatOptionId"] = format_option_id return self def create(self, overwrite=False): """ Execute the creation of the streaming endpoint according to the selected options :param boolean overwrite: If the streaming endpoint being created already exists, delete it first :returns: an object corresponding to the newly created streaming endpoint :rtype: :class:`DSSStreamingEndpoint` """ if overwrite and self.already_exists(): self.project.get_streaming_endpoint(self.streaming_endpoint_name).delete() self.project.client._perform_json("POST", "/projects/%s/streamingendpoints/managed" % self.project.project_key, body = { "id": self.streaming_endpoint_name, "type": self.streaming_endpoint_type, "creationSettings": self.creation_settings }) return DSSStreamingEndpoint(self.project.client, self.project.project_key, self.streaming_endpoint_name) def already_exists(self): """ Whether the desired name for the new streaming endpoint is already used by another streaming endpoint :rtype: boolean """ streaming_endpoint = self.project.get_streaming_endpoint(self.streaming_endpoint_name) try: streaming_endpoint.get_schema() return True except Exception as e: return Falsedataiku-api-client-python-13.2.2/dataikuapi/dss/tools/000077500000000000000000000000001471211575100226345ustar00rootroot00000000000000dataiku-api-client-python-13.2.2/dataikuapi/dss/tools/__init__.py000066400000000000000000000000001471211575100247330ustar00rootroot00000000000000dataiku-api-client-python-13.2.2/dataikuapi/dss/tools/codegen.py000066400000000000000000000410521471211575100246140ustar00rootroot00000000000000import logging import re from ..dataset import DSSDataset from ..recipe import CodeRecipeSettings class _IndentContext(object): def __init__(self, flow_code_generator): self.flow_code_generator = flow_code_generator def __enter__(self): self.flow_code_generator.cur_indent += 1 def __exit__(self, b, c, d): self.flow_code_generator.cur_indent -= 1 def slugify(name): return re.sub(r"[^A-Za-z0-9_]", "_", name) class FlowCodeGenerator(object): def __init__(self): self.code = "" self.cur_indent = 0 self.remove_metrics_on_datasets = False self.remove_display_width_on_prepare = False def set_options(self, remove_metrics_on_datasets=False, remove_display_width_on_prepare=False): self.remove_metrics_on_datasets = remove_metrics_on_datasets self.remove_display_width_on_prepare = remove_display_width_on_prepare pass def generate_code_for_dataset(self, dataset): entrypoint_name = "create_dataset_%s" % dataset.dataset_name self._generate_code_for_dataset(dataset, entrypoint_name) return self.code def generate_code_for_recipe(self, recipe): entrypoint_name = "create_recipe_%s" % recipe.recipe_name self._generate_code_for_recipe(recipe, entrypoint_name) return self.code def generate_code_for_project(self, project, entrypoint_name = None): self.gen("import json") self.gen("") if entrypoint_name is None: entrypoint_name = "create_flow_for_project" self.gen("def %s(project):" % entrypoint_name) flow_graph = project.get_flow().get_graph() flow_items = flow_graph.get_items_in_traversal_order(as_type="object") entrypoints_to_call = [] with _IndentContext(self) as ic: for item in flow_items: if isinstance(item, DSSDataset): entrypoint_name = "create_dataset_%s" % slugify(item.dataset_name) self._generate_code_for_dataset(item, entrypoint_name) else: entrypoint_name = "create_recipe_%s" % slugify(item.recipe_name) self._generate_code_for_recipe(item, entrypoint_name) entrypoints_to_call.append(entrypoint_name) self.gen("") self.gen("# Actual creation of the Flow from the individual functions") for ep in entrypoints_to_call: self.gen("%s(project)" % ep) return self.code def _generate_code_for_dataset(self, dataset, entrypoint_name): self.gen("def %s(project):" % entrypoint_name) settings = dataset.get_settings() raw = settings.get_raw() templates = dataset.client._perform_json("GET", "/projects/X/datasets/templates") do_not_copy = [ "projectKey", "name", "type", "versionTag", "creationTag", "schema" ] self.gen(" # Base dataset params") if raw["type"] == "UploadedFiles": self.gen(" dataset = project.create_upload_dataset(\"%s\")" % dataset.dataset_name) self.gen(" settings = dataset.get_settings()") self.lf() self.codegen_object_fields_explicit(settings.get_raw(), templates["dataset"], ["params"], "settings.get_raw()") do_not_copy.append("params") elif raw["type"] in DSSDataset.FS_TYPES: srcp = raw["params"] self.gen(" dataset = project.create_fslike_dataset(\"%s\", \"%s\", \"%s\", \"%s\")" % \ (dataset.dataset_name, raw["type"], srcp["connection"], srcp.get("path", "/"))) self.gen(" settings = dataset.get_settings()") self.lf() self.codegen_object_fields(srcp, templates["abstractFSConfig"], ["connection", "path"], "settings.get_raw_params()") do_not_copy.append("params") elif raw["type"] in DSSDataset.SQL_TYPES: self.gen(" dataset = project.create_dataset(\"%s\", \"%s\")" % (dataset.dataset_name, raw["type"])) self.gen(" settings = dataset.get_settings()") self.lf() srcp = raw["params"] if srcp.get("mode", None) == "table": self.gen(" settings.set_table(%s, %s, %s)" % \ (self.objstr(srcp.get("connection")), self.objstr(srcp.get("schema")), self.objstr(srcp.get("table")))) self.codegen_object_fields(srcp, templates["abstractSQLConfig"], ["mode", "connection", "schema", "table"], "settings.get_raw_params()") do_not_copy.append("params") else: self.gen(" dataset = project.create_dataset(\"%s\", \"%s\")" % (dataset.dataset_name, raw["type"])) self.gen(" settings = dataset.get_settings()") self.codegen_object_fields_explicit(settings.get_raw(), templates["dataset"], ["params"], "settings.get_raw()") do_not_copy.append("params") # Copy of format params if "formatType" in raw: self.lf() self.gen(" # Format params") handled = False if raw["formatType"] == "csv": csv_format = raw["formatParams"] self.gen(" settings.set_csv_format(separator=\"%s\", style=\"%s\", skip_rows_before=%d, header_row=%s, skip_rows_after=%d)"%\ (csv_format.get("separator", None), csv_format["style"], csv_format["skipRowsBeforeHeader"],\ csv_format["parseHeaderRow"], csv_format["skipRowsAfterHeader"])) self.codegen_object_fields(csv_format, templates["csvFormat"], ["separator", "style", "skipRowsBeforeHeader", "parseHeaderRow", "skipRowsAfterHeader", "probableNumberOfRecords"], "settings.get_raw_format_params()") else: self.codegen_object_fields_explicit(settings.get_raw(), templates["dataset"], ["formatType", "formatParams"], "settings.get_raw()") do_not_copy.extend(["formatType", "formatParams"]) self.lf() self.gen(" # Schema") for column in settings.get_raw()["schema"]["columns"]: self.gen(" settings.add_raw_schema_column(%s)" % column) if not self.remove_metrics_on_datasets: self.lf() self.gen(" # Metrics") self.codegen_object_fields(settings.get_raw()["metrics"], templates["dataset"]["metrics"], [], "settings.get_raw()[\"metrics\"]") do_not_copy.append("metrics") self.lf() self.gen(" # Other dataset params") self.codegen_object_fields(settings.get_raw(), templates["dataset"], do_not_copy, "settings.get_raw()") self.lf() self.gen(" settings.save()") self.lf() def _generate_code_for_recipe(self, recipe, entrypoint_name): logging.info("Codegen for recipe %s" % entrypoint_name) self.gen("def %s(project):" % entrypoint_name) settings = recipe.get_settings() raw = settings.get_recipe_raw_definition() templates = recipe.client._perform_json("GET", "/projects/%s/recipes/templates" % recipe.project_key) template = {"tags":[], "optionalDependencies": False, "redispatchPartitioning": False, "maxRunningActivities": 0, "neverRecomputeExistingPartitions" : False, "customFields":{}, "customMeta": {"kv":{}}, "checklists" : {"checklists":[]}} do_not_copy = [ "projectKey", "name", "type", "versionTag", "creationTag", "inputs", "outputs" ] self.gen(" # Create the recipe as a blank recipe") self.gen(" builder = project.new_recipe(\"%s\", \"%s\")" % (raw["type"], recipe.recipe_name)) self.gen(" builder.set_raw_mode()") self.gen(" recipe = builder.create()") self.lf() self.gen(" # Setup the recipe") self.gen(" settings = recipe.get_settings()" ) self.lf() self.gen(" # Recipe inputs/outputs") for (input_role, input_item) in settings._get_flat_inputs(): if len(input_item["deps"]) > 0: self.gen(" settings.add_input(%s, %s, %s)" % (self.objstr(input_role), self.objstr(input_item["ref"]), self.objstr(input_item["deps"]))) else: self.gen(" settings.add_input(%s, %s)" % (self.objstr(input_role), self.objstr(input_item["ref"]))) for (output_role, output_item) in settings._get_flat_outputs(): self.gen(" settings.add_output(%s, %s, %s)" % (self.objstr(output_role), self.objstr(output_item["ref"]), self.objstr(output_item["appendMode"]))) self.lf() types_with_obj_payload = ["join", "grouping", "shaker"] # A bit of "classical cleanup" # Remove the dirty "map" in Spark read params if settings.type in types_with_obj_payload and "engineParams" in settings.obj_payload: rp = settings.obj_payload["engineParams"].get("spark", {}).get("readParams", {}) if rp.get("mode", "?") == "AUTO": rp["map"] = {} if raw is not None and "params" in raw and "engineParams" in raw["params"]: rp = raw["params"]["engineParams"].get("spark", {}).get("readParams", {}) if rp.get("mode", "?") == "AUTO": rp["map"] = {} # And some per-type cleanup def cleanup_grouping(): for grouping_key in settings.obj_payload.get("keys", []): for item in ["count", "last", "min", "max", "sum", "countDistinct", "stddev", "avg", "concat", "first"]: if item in grouping_key and grouping_key[item] == False: del grouping_key[item] for aggregation in settings.obj_payload.get("values", []): for item in ["count", "last", "min", "max", "sum", "countDistinct", "stddev", "avg", "concat", "first", "concatDistinct", "$idx", "sum2", "firstLastNotNull"]: if item in aggregation and aggregation.get(item, None) == False: del aggregation[item] def cleanup_join(): for vi in settings.raw_virtual_inputs: if not vi["preFilter"]["enabled"]: del vi["preFilter"] def cleanup_shaker(): if self.remove_display_width_on_prepare: if "columnWidthsByName" in settings.obj_payload: del settings.obj_payload["columnWidthsByName"] cleanup_by_type = { "grouping": cleanup_grouping, "join": cleanup_join, "shaker": cleanup_shaker } if settings.type in cleanup_by_type: cleanup_by_type[settings.type]() # Output payload, either globally for code if isinstance(settings, CodeRecipeSettings): code = settings.get_code() self.gen(" # Recipe code") self.gen(" settings.set_payload(%s)" % self.payloadstr(code)) # per-field for recipes with JSON payload elif settings.type in types_with_obj_payload: prefix_by_type = { "join": "Join details", "shaker": "Prepare script" } self.gen(" # %s" % (prefix_by_type.get(settings.type, "Recipe payload"))) payload = settings.obj_payload payload_template = templates["payloadsByType"][settings.type] self.gen(" settings.set_payload(\"{}\")") self.codegen_object_fields(payload, payload_template, [], "settings.obj_payload") # Or as string for others elif len(settings.get_payload()) > 0: self.gen(" # Recipe payload") self.gen(" settings.set_payload(%s)" % self.payloadstr(settings.get_payload())) # Then params if settings.type in templates["paramsByType"] and "params" in raw: self.lf() self.gen(" # Type-specific parameters") self.codegen_object_fields(raw["params"], templates["paramsByType"][settings.type], [], "settings.raw_params") do_not_copy.append("params") # And finally other recipe fields that are not params self.lf() self.gen(" # Other parameters") self.codegen_object_fields(raw, template, do_not_copy, "settings.get_recipe_raw_definition()") self.lf() self.gen(" settings.save()") self.lf() # Helpers def gen(self, code): self.code += "%s%s\n" % (" " * (4 * self.cur_indent), code) def lf(self): self.code += "\n" def payloadstr(self, payload): if payload.endswith("\n"): return "\"\"\"%s\"\"\"" % payload else: return "\"\"\"%s\n\"\"\"" % payload def objstr(self, obj): return ObjectFieldFormatter(self.cur_indent + 1).format(obj) def codegen_object_fields_explicit(self, object, template, copy, prefix): for key in copy: if not key in object: continue self.codegen_object_field(object, key, template, prefix) def codegen_object_fields(self, object, template, do_not_copy, prefix): for key in object.keys(): if key in do_not_copy: continue self.codegen_object_field(object, key, template, prefix) def codegen_object_field(self, object, key, template, prefix): value_for_key = object[key] default_value_for_key = template.get(key, None) if default_value_for_key is not None and value_for_key == default_value_for_key: #print("Skipping value equal to default: %s" % key) return else: #print("Not equal for %s" % key) #print("Template: %s" % default_value_for_key) #print("Real: %s" % value_for_key) self.gen(" %s[\"%s\"] = %s" % ( prefix, key, self.objstr(value_for_key))) class ObjectFieldFormatter(object): def __init__(self, base_indent): self.formatters = { dict: self.__class__.format_dict, list: self.__class__.format_list, } self.sp = " " self.base_indent = base_indent self.no_pretty_level = 0 def format(self, value, depth=0): base_repr = repr(value) if len(base_repr) < 25: # This entire value is very small, don't bother pretty-priting it return base_repr elif type(value) in self.formatters: return self.formatters[type(value)](self, value, depth) else: return base_repr def format_dict(self, value, depth): indent = self.base_indent + depth if depth > 2 or self.no_pretty_level > 0: return repr(value) else: items = [ "\n" + self.sp * (indent + 1) + repr(key) + ': ' + self.format(value[key], depth + 1) for key in value ] return '{%s}' % (','.join(items) + "\n" + self.sp * indent) def format_list(self, value, depth): indent = self.base_indent + depth if depth > 2 or self.no_pretty_level > 0 or len(value) == 0: return repr(value) else: # Big array, don't pretty-print inner if len(value) > 3: self.no_pretty_level += 1 items = [ "\n" + self.sp * (indent + 1) + self.format(item, depth + 1) for item in value ] if len(value) > 3: self.no_pretty_level -= 1 return '[%s]' % (','.join(items) + "\n" + self.sp * indent)dataiku-api-client-python-13.2.2/dataikuapi/dss/tools/langchain.py000066400000000000000000000052511471211575100251350ustar00rootroot00000000000000from threading import RLock class StopSequencesAwareStreamer: """ Helper class to enforce stop sequences when streaming """ def __init__(self, stop_sequences, stop_sequences_enforcer): self.stop_sequences = stop_sequences or [] self.stop_sequences_enforcer = stop_sequences_enforcer self.current_chunk = None self._current_chunk_lock = RLock() def append(self, chunk): """ Appends a chunk to the current one, or just sets it if None """ with self._current_chunk_lock: if self.should_stop(): return if not self.current_chunk: self.current_chunk = chunk else: self.current_chunk += chunk def should_stop(self): """ Checks if streaming should stop because a stop sequence was found :returns: True if a stop sequence is found in the current chunk """ with self._current_chunk_lock: if not self.stop_sequences: return False if not self.current_chunk: return False # if a stop sequence is found, remove it from the current chunk cutoff_text = self.stop_sequences_enforcer(self.current_chunk.text, self.stop_sequences) if cutoff_text != self.current_chunk.text: self.current_chunk.text = cutoff_text if hasattr(self.current_chunk, 'message'): self.current_chunk.message.content = cutoff_text return True return False def can_yield(self): """ Checks if a chunk is safe to yield :returns: True unless a partial stop sequence is found in the current chunk """ with self._current_chunk_lock: if not self.current_chunk: return False for sequence in self.stop_sequences: if self._has_sequence_started(sequence): return False # at this point we're sure that the current chunk contains none of the stop sequences return True def yield_(self, producer): """ Returns the current chunk and resets the current chunk :returns: the current chunk """ with self._current_chunk_lock: chunk = producer(self.current_chunk) self.current_chunk = None return chunk def _has_sequence_started(self, sequence): """ :returns: true if any of the stop sequences has started in the current chunk """ for i in range(len(sequence)): if self.current_chunk.text.endswith(sequence[:i + 1]): return True return False dataiku-api-client-python-13.2.2/dataikuapi/dss/unifiedmonitoring.py000066400000000000000000000346031471211575100256050ustar00rootroot00000000000000MANAGED_API_ENDPOINT_MONITORING_TYPE = "MANAGED_API_ENDPOINT" EXTERNAL_API_ENDPOINT_MONITORING_TYPE = "EXTERNAL_API_ENDPOINT" class DSSUnifiedMonitoring(object): """ Handle to interact with Unified Monitoring .. warning:: Do not create this class directly, use :meth:`dataikuapi.dssclient.DSSClient.get_unified_monitoring` """ def __init__(self, client): self.client = client def list_monitored_project_deployments(self): """ Lists the monitored project deployments :return: The list of monitored projects :rtype: list of :class:`dataikuapi.dss.unifiedmonitoring.MonitoredProjectDeployment` """ items = self.client._perform_json("GET", "/unified-monitoring/deployer/projects") return [MonitoredProjectDeployment(monitoring) for monitoring in items] def list_monitored_api_endpoints(self, remove_duplicated_external_endpoints=True): """ Lists the monitored API endpoints :param boolean remove_duplicated_external_endpoints: if True, an endpoint that is both in a Deploy Anywhere Infrastructure and in an External endpoint scope will be listed only once, under the Deploy Anywhere Infrastructure. Optional :return: The list of monitored API endpoints :rtype: list of Union[:class:`dataikuapi.dss.unifiedmonitoring.MonitoredManagedApiEndpoint`, :class:`dataikuapi.dss.unifiedmonitoring.MonitoredExternalApiEndpoint`] """ params = { 'removeDuplicatedExternalEndpoints': remove_duplicated_external_endpoints } items = self.client._perform_json("GET", "/unified-monitoring/deployer/api-endpoints", params=params) monitorings = [] for monitoring in items: if MANAGED_API_ENDPOINT_MONITORING_TYPE == monitoring.get("type"): monitorings.append(MonitoredManagedApiEndpoint(self.client, monitoring)) elif EXTERNAL_API_ENDPOINT_MONITORING_TYPE == monitoring.get("type"): monitorings.append(MonitoredExternalApiEndpoint(self.client, monitoring)) return monitorings def list_monitored_api_endpoint_with_activity_metrics(self, endpoints_to_filter_on=None, remove_duplicated_external_endpoints=True): """ Lists the monitored API endpoints with their activity metrics :param endpoints_to_filter_on: endpoints for which monitoring and activity metrics should be retrieved. If None or empty, all endpoints are considered :type endpoints_to_filter_on: list of Union[:class:`dataikuapi.dss.unifiedmonitoring.DSSApiEndpointMonitoring`, :class:`dataikuapi.dss.unifiedmonitoring.ExternalApiEndpointMonitoring`]. Optional :param boolean remove_duplicated_external_endpoints: if True, an endpoint that is both in a Deploy Anywhere Infrastructure and in an External Endpoint Scope will be listed only once, under the Deploy Anywhere Infrastructure. Optional :return: The list of API endpoint monitorings with their activity metrics :rtype: list of Union[:class:`dataikuapi.dss.unifiedmonitoring.DSSApiEndpointMonitoringWithActivityMetrics`, :class:`dataikuapi.dss.unifiedmonitoring.ExternalApiEndpointMonitoringWithActivityMetrics`] """ formatted_endpoint_params = [] if endpoints_to_filter_on: for endpoint_monitoring in endpoints_to_filter_on: if isinstance(endpoint_monitoring, MonitoredManagedApiEndpoint): formatted_endpoint_params.append({'endpointId': endpoint_monitoring.endpoint_id, 'deploymentId': endpoint_monitoring.deployment_id}) elif isinstance(endpoint_monitoring, MonitoredExternalApiEndpoint): formatted_endpoint_params.append({'endpointName': endpoint_monitoring.endpoint_name, 'externalEndpointsScopeName': endpoint_monitoring.external_endpoints_scope.get("name")}) else: raise TypeError("Unsupported type: " + endpoint_monitoring.__class__.__name__) params = { 'removeDuplicatedExternalEndpoints': remove_duplicated_external_endpoints, } items = self.client._perform_json("POST", "/unified-monitoring/deployer/api-endpoints-with-activity-metrics", params=params, body=formatted_endpoint_params) monitorings = [] for monitoring_with_activity_metrics in items: if monitoring_with_activity_metrics.get("type") == MANAGED_API_ENDPOINT_MONITORING_TYPE: monitorings.append(MonitoredManagedApiEndpointWithActivityMetrics(self.client, monitoring_with_activity_metrics)) elif monitoring_with_activity_metrics.get("type") == EXTERNAL_API_ENDPOINT_MONITORING_TYPE: monitorings.append(MonitoredExternalApiEndpointWithActivityMetrics(self.client, monitoring_with_activity_metrics)) return monitorings class AbstractMonitoredThing(object): def __init__(self, data): self.deployment_id = data.get("deploymentId") self.infrastructure_id = data.get("infrastructureId") self.stage = data.get("stage") self.snapshot_timestamp = data.get("snapshotTimestamp") self.data = data class MonitoredProjectDeployment(AbstractMonitoredThing): """ A handle on a monitored project deployment .. warning:: Do not create this class directly, instead use :meth:`dataikuapi.dss.unifiedmonitoring.DSSUnifiedMonitoring.list_monitored_project_deployments` """ def __init__(self, data): super(MonitoredProjectDeployment, self).__init__(data) self.deployment_id = data.get("deploymentId") """The deployment id""" self.infrastructure_id = data.get("infrastructureId") """The infrastructure id""" self.stage = data.get("stage") """The stage of the deployment""" self.snapshot_timestamp = data.get("snapshotTimestamp") """The timestamp of the snapshot""" self.bundle_name = data.get("bundleName") """The bundle name""" self.deployed_project_key = data.get("deployedProjectKey") """The deployed project key""" self.published_project_key = data.get("publishedProjectKey") """The published project key""" self.data = data def get_raw(self): """ Get the raw monitored project :rtype: dict """ return self.data class MonitoredManagedApiEndpoint(AbstractMonitoredThing): """ A handle on a monitored managed API endpoint .. warning:: Do not create this class directly, instead use :meth:`dataikuapi.dss.unifiedmonitoring.DSSUnifiedMonitoring.list_monitored_api_endpoints` """ def __init__(self, client, data): super(MonitoredManagedApiEndpoint, self).__init__(data) self.client = client self.deployment_id = data.get("deploymentId") """The deployment id""" self.infrastructure_id = data.get("infrastructureId") """The infrastructure id""" self.stage = data.get("stage") """The stage of the deployment""" self.snapshot_timestamp = data.get("snapshotTimestamp") """The timestamp of the snapshot""" self.endpoint_id = data.get("endpointId") """The endpoint id""" self.type = data.get("type") """'MANAGED_API_ENDPOINT'""" def get_activity_metrics(self): """ Get the activity metrics of the monitored managed API endpoint :return: The activity metrics :rtype: :class:`dataikuapi.dss.unifiedmonitoring.ManagedApiEndpointActivityMetrics` """ endpoint_identifier = { 'deploymentId': self.deployment_id, 'endpointId': self.endpoint_id } raw_activity_metrics = self.client._perform_json("POST", "/unified-monitoring/deployer/activity-metrics", body=endpoint_identifier) return ManagedApiEndpointActivityMetrics(raw_activity_metrics) if raw_activity_metrics else None def get_raw(self): """ Get the raw monitored managed API endpoint :rtype: dict """ return self.data class MonitoredExternalApiEndpoint(AbstractMonitoredThing): """ A handle on a monitored external API endpoint .. warning:: Do not create this class directly, instead use :meth:`dataikuapi.dss.unifiedmonitoring.DSSUnifiedMonitoring.list_monitored_api_endpoints` """ def __init__(self, client, data): super(MonitoredExternalApiEndpoint, self).__init__(data) self.client = client self.external_endpoints_scope = data.get("externalEndpointsScope") """The external endpoint scope configuration""" self.stage = data.get("stage") """The stage of the deployment""" self.snapshot_timestamp = data.get("snapshotTimestamp") """The timestamp of the snapshot""" self.endpoint_name = data.get("endpointName") """The endpoint name""" self.type = data.get("type") """'EXTERNAL_API_ENDPOINT'""" def get_activity_metrics(self): """ Get the activity metrics of the monitored external API endpoint :return: The activity metrics :rtype: :class:`dataikuapi.dss.unifiedmonitoring.ExternalApiEndpointActivityMetrics` """ endpoint_identifier = { 'externalEndpointsScopeName': self.external_endpoints_scope.get("name"), 'endpointName': self.endpoint_name } raw_activity_metrics = self.client._perform_json("POST", "/unified-monitoring/deployer/activity-metrics", body=endpoint_identifier) return ExternalApiEndpointActivityMetrics(raw_activity_metrics) if raw_activity_metrics else None def get_raw(self): """ Get the raw monitored external API endpoint :rtype: dict """ return self.data class ManagedApiEndpointActivityMetrics(object): """ A handle on the activity metrics of a managed API endpoint .. warning:: Do not create this class directly, instead use :meth:`dataikuapi.dss.unifiedmonitoring.DSSUnifiedMonitoring.list_monitored_api_endpoint_with_activity_metrics` """ def __init__(self, data): self.deployment_id = data.get("deploymentId") """The deployment id""" self.endpoint_id = data.get("endpointId") """The endpoint id""" self.period = data.get("period") """The time period spanned by this :class:`DSSApiEndpointActivityMetrics`""" self.period_all_requests_count = data.get("periodAllRequestsCount") """The count of all requests in the period""" self.period_error_rate = data.get("periodErrorRate") """The request error rate in the period""" self.period_response_time_ms = data.get("periodResponseTimeMs") """The average response time in the period""" self.counts = data.get("counts") """Detailed number of requests per timestamp""" self.data = data def get_raw(self): """ Get the raw activity metrics of a managed API endpoint :rtype: dict """ return self.data class ExternalApiEndpointActivityMetrics(object): """ A handle on the activity metrics of an external API endpoint .. warning:: Do not create this class directly, instead use :meth:`dataikuapi.dss.unifiedmonitoring.DSSUnifiedMonitoring.list_monitored_api_endpoint_with_activity_metrics` """ def __init__(self, data): self.external_endpoints_scope_name = data.get("externalEndpointsScopeName") """The name of the external endpoint scope""" self.endpoint_name = data.get("endpointName") """The endpoint name""" self.period = data.get("period") """The time period spanned by this :class:`ExternalApiEndpointActivityMetrics`""" self.period_all_requests_count = data.get("periodAllRequestsCount") """The count of all requests in the period""" self.period_error_rate = data.get("periodErrorRate") """The request error rate in the period""" self.period_response_time_ms = data.get("periodResponseTimeMs") """The average response time in the period""" self.counts = data.get("counts") """Detailed number of requests per timestamp""" self.data = data def get_raw(self): """ Get the raw activity metrics of an external API endpoint :rtype: dict """ return self.data class MonitoredManagedApiEndpointWithActivityMetrics(object): """ A handle on a monitored managed API endpoint and its activity metrics .. warning:: Do not create this class directly, instead use :meth:`dataikuapi.dss.unifiedmonitoring.DSSUnifiedMonitoring.list_monitored_api_endpoint_with_activity_metrics` """ def __init__(self, client, data): self.endpoint_monitoring = MonitoredManagedApiEndpoint(client, data.get("endpointMonitoring")) """The DSS API endpoint monitoring""" raw_activity_metrics = data.get("activityMetrics") self.activity_metrics = ManagedApiEndpointActivityMetrics(raw_activity_metrics) if raw_activity_metrics else None """The DSS API endpoint activity metrics""" self.type = data.get("type") """'MANAGED_API_ENDPOINT'""" self.data = data def get_raw(self): """ Get the raw monitored managed API endpoint with its activity metrics :rtype: dict """ return self.data class MonitoredExternalApiEndpointWithActivityMetrics(object): """ A handle on a monitored external API endpoint and its activity metrics .. warning:: Do not create this class directly, instead use :meth:`dataikuapi.dss.unifiedmonitoring.DSSUnifiedMonitoring.list_monitored_api_endpoint_with_activity_metrics` """ def __init__(self, client, data): self.endpoint_monitoring = MonitoredExternalApiEndpoint(client, data.get("endpointMonitoring")) """The external API endpoint monitoring""" raw_activity_metrics = data.get("activityMetrics") self.activity_metrics = ExternalApiEndpointActivityMetrics(raw_activity_metrics) if raw_activity_metrics else None """The external API endpoint activity metrics""" self.type = data.get("type") """'EXTERNAL_API_ENDPOINT'""" self.data = data def get_raw(self): """ Get the raw monitored external API endpoint with its activity metrics :rtype: dict """ return self.data dataiku-api-client-python-13.2.2/dataikuapi/dss/utils.py000066400000000000000000000416661471211575100232230ustar00rootroot00000000000000class DSSDatasetSelectionBuilder(object): """Builder for a "dataset selection". In DSS, a dataset selection is used to select a part of a dataset for processing. Depending on the location where it is used, a selection can include: * Sampling * Filtering by partitions (for partitioned datasets) * Filtering by an expression * Selection of columns * Ordering Please see the sampling documentation of DSS for a detailed explanation of the sampling methods. """ def __init__(self): self.selection = {} def build(self): """ :returns: the built selection dict :rtype: dict """ return self.selection def with_head_sampling(self, limit): """ Sets the sampling to 'first records' mode :param int limit: Maximum number of rows in the sample """ self.selection["samplingMethod"] = "HEAD_SEQUENTIAL" self.selection["maxRecords"] = limit return self def with_all_data_sampling(self): """Sets the sampling to 'no sampling, all data' mode""" self.selection["samplingMethod"] = "FULL" return self def with_random_fixed_nb_sampling(self, nb): """ Sets the sampling to 'Random sampling, fixed number of records' mode :param int nb: Maximum number of rows in the sample """ self.selection["samplingMethod"] = "RANDOM_FIXED_NB" self.selection["maxRecords"] = nb return self def with_selected_partitions(self, ids): """ Sets partition filtering on the given partition identifiers. .. warning:: The dataset to select must be partitioned. :param list ids: list of selected partitions """ self.selection["partitionSelectionMethod"] = "SELECTED" self.selection["selectedPartitions"] = ids return self class DSSComputedColumn(object): @staticmethod def formula(name, formula, type="double"): """ Create a computed column with a formula. :param string name: a name for the computed column :param string formula: formula to compute values, using the `GREL language `_ :param string type: name of a DSS type for the values of the column :return: a computed column as a dict :rtype: dict """ return {"expr": formula, "mode": "GREL", "name": name, "type": type} import sys if sys.version_info > (3,4): from enum import Enum else: class Enum(object): pass class DSSFilterOperator(Enum): EMPTY_ARRAY = "empty array" """ Test if an array is empty. """ NOT_EMPTY_ARRAY = "not empty array" """ Test if an array is not empty. """ CONTAINS_ARRAY = "array contains" """ Test if an array contains a value. """ NOT_EMPTY = "not empty" """ Test if a value is not empty and not null. """ EMPTY = "is empty" """ Test if a value is empty or null. """ NOT_EMPTY_STRING = "not empty string" """ Test if a string is not empty. """ EMPTY_STRING = "empty string" """ Test if a string is empty. """ IS_TRUE = "true" """ Test if a boolean is true. """ IS_FALSE = "false" """ Test if a boolean is false. """ EQUALS_STRING = "== [string]" """ Test if a string is equal to a given value. """ EQUALS_CASE_INSENSITIVE_STRING = "== [string]i" """ Test if a string is equal to a given value, ignoring case. """ NOT_EQUALS_STRING = "!= [string]" """ Test if a string is not equal to a given value. """ SAME = "== [NaNcolumn]" """ Test if two columns have the same value when formatted to string. """ DIFFERENT = "!= [NaNcolumn]" """ Test if two columns have different values when formatted to string. """ EQUALS_NUMBER = "== [number]" """ Test if a number is equal to a given value. """ NOT_EQUALS_NUMBER = "!= [number]" """ Test if a number is not equal to a given value. """ GREATER_NUMBER = "> [number]" """ Test if a number is greater than a given value. """ LESS_NUMBER = "< [number]" """ Test if a number is less than a given value. """ GREATER_OR_EQUAL_NUMBER = ">= [number]" """ Test if a number is greater or equal to a given value. """ LESS_OR_EQUAL_NUMBER = "<= [number]" """ Test if a number is less or equal to a given value. """ EQUALS_DATE = "== [date]" """ Test if a date/time is equal to a given date/time (rounded). """ GREATER_DATE = "> [date]" """ Test if a date/time is greater than a given date/time. """ GREATER_OR_EQUAL_DATE = ">= [date]" """ Test if a date/time is greater or equal than a given date/time. """ LESS_DATE = "< [date]" """ Test if a date/time is less than a given date/time. """ LESS_OR_EQUAL_DATE = "<= [date]" """ Test if a date/time is less or equal than a given date/time. """ BETWEEN_DATE = ">< [date]" """ Test if a date/time is between two given date/times. """ EQUALS_COL = "== [column]" """ Test if two columns have the same (typed) value. """ NOT_EQUALS_COL = "!= [column]" """ Test if two columns have different (typed) values. """ GREATER_COL = "> [column]" """ Test if one column is greater than another. """ LESS_COL = "< [column]" """ Test if one column is less than another. """ GREATER_OR_EQUAL_COL = ">= [column]" """ Test if one column is greater or equal than another. """ LESS_OR_EQUAL_COL = "<= [column]" """ Test if one column is less or equal than another. """ CONTAINS_STRING = "contains" """ Test if a column contains a given string. """ REGEX = "regex" """ Test if a column matches a regular expression. """ class DSSFilter(object): """ Helper class to build filter objects for use in visual recipes. """ @staticmethod def of_single_condition(column, operator, string=None, num=None, date=None, time=None, date2=None, time2=None, unit=None): """ Create a simple filter on a column. Which of the 'string', 'num', 'date', 'time', 'date2' and 'time2' parameter holds the literal to filter against depends on the filter operator. :param string column: name of a column to filter (left operand) :param string operator: type of filter applied to the column, one of the values in the :class:`DSSFilterOperator` enum :param string string: string literal for the right operand :param string num: numeric literal for the right operand :param string date: date part literal for the right operand :param string time: time part literal for the right operand :param string date2: date part literal for the right operand of BETWEEN_DATE :param string time2: time part literal for the right operand of BETWEEN_DATE :param string unit: date/time rounding for date operations. Possible values are YEAR, MONTH, WEEK, DAY, HOUR, MINUTE, SECOND """ return { "enabled": True, "uiData": { 'conditions': [DSSFilter.condition(column, operator, string, num, date, time, date2, time2, unit)], "mode": "&&" } } @staticmethod def of_and_conditions(conditions): """ Create a filter as an intersection of conditions. The resulting filter keeps rows that match all the conditions in the list. Conditions are for example the output of :meth:`condition()`. :param list conditions: a list of conditions :return: a filter, as a dict :rtype: dict """ return { "enabled": True, "uiData": { 'conditions': conditions, "mode": "&&" } } @staticmethod def of_or_conditions(conditions): """ Create a filter as an union of conditions. The resulting filter keeps rows that match any of the conditions in the list. Conditions are for example the output of :meth:`condition()`. :param list conditions: a list of conditions :return: a filter, as a dict :rtype: dict """ return { "enabled": True, "uiData": { 'conditions': conditions, "mode": "||" } } @staticmethod def of_formula(formula): """ Create a filter that applies a GREL formula. The resulting filter evaluates the formula and keeps rows for which the formula returns a True value. :param string formula: a `GREL formula `_ :return: a filter, as a dict :rtype: dict """ return { "enabled": True, "uiData": { "mode": "CUSTOM" }, "expression": formula } @staticmethod def of_sql_expression(sql_expression): """ Create a filter that applies a SQL expression. The resulting filter evaluates the sql expression and keeps rows for which the sql expression returns a True value. :param string sql_expression: a SQL expression :return: a filter, as a dict :rtype: dict """ return { "enabled": True, "uiData": { "mode": "SQL" }, "expression": sql_expression } @staticmethod def condition(column, operator, string=None, num=None, date=None, time=None, date2=None, time2=None, unit=None): """ Create a condition on a column for a filter. Which of the 'string', 'num', 'date', 'time', 'date2' and 'time2' parameter holds the literal to filter against depends on the filter operator. :param string column: name of a column to filter (left operand) :param string operator: type of filter applied to the column, one of the values in the :class:`DSSFilterOperator` enum :param string string: string literal for the right operand :param string num: numeric literal for the right operand :param string date: date part literal for the right operand :param string time: time part literal for the right operand :param string date2: date part literal for the right operand of BETWEEN_DATE :param string time2: time part literal for the right operand of BETWEEN_DATE :param string unit: date/time rounding for date operations. Possible values are YEAR, MONTH, WEEK, DAY, HOUR, MINUTE, SECOND """ if isinstance(operator, DSSFilterOperator): operator = operator.value cond = { "input": column, "operator": operator } if string is not None: cond["string"] = string if num is not None: cond["num"] = num if date is not None: cond["date"] = date if time is not None: cond["time"] = time if date2 is not None: cond["date2"] = date2 if time2 is not None: cond["time2"] = time2 if unit is not None: cond["unit"] = unit return cond class DSSFilterBuilder(object): """ Builder for a "filter". In DSS, a filter is used to define a subset of rows for processing. """ def __init__(self): self.filter = {"enabled": False, "distinct": False, "expression": None, "uiData": {"mode": "CUSTOM"}} def build(self): """ :returns: the built filter :rtype: dict """ return self.filter def with_distinct(self): """Sets the filter to deduplicate""" self.filter["distinct"] = True return self def with_formula(self, expression): """ Sets the formula (DSS formula) used to filter rows :param str expression: the DSS formula """ self.filter["enabled"] = True self.filter["expression"] = expression self.filter["uiData"]["mode"] = "CUSTOM" return self class AnyLoc(object): def __init__(self, project_key, object_id): self.project_key = project_key self.object_id = object_id def __eq__(self, obj): return isinstance(obj, AnyLoc) and obj.project_key == self.project_key and obj.object_id == self.object_id @staticmethod def from_ref(context_project_key, ref): if ref.find(".") >= 0: elts = ref.split(".") return AnyLoc(elts[0], elts[1]) else: return AnyLoc(context_project_key, ref) @staticmethod def from_full(ref): if ref.find(".") >= 0: elts = ref.split(".") return AnyLoc(elts[0], elts[1]) else: raise Exception("Cannot parse object id, it's not a full id") class DSSTaggableObjectListItem(dict): """An item in a list of taggable objects. Do not instantiate this class""" def __init__(self, data): super(DSSTaggableObjectListItem, self).__init__(data) self._data = data @property def tags(self): return self._data["tags"] class DSSTaggableObjectSettings(object): def __init__(self, taggable_object_data): self._tod = taggable_object_data @property def tags(self): """The tags of the object, as a list of strings""" return self._tod["tags"] @tags.setter def tags(self, tags): self._tod["tags"] = tags @property def description(self): """The description of the object as a string""" return self._tod.get("description", None) @description.setter def description(self, description): self._tod["description"] = description @property def short_description(self): """The short description of the object as a string""" return self._tod.get("shortDesc", None) @short_description.setter def short_description(self, short_description): self._tod["shortDesc"] = short_description @property def custom_fields(self): """The custom fields of the object as a dict. Returns None if there are no custom fields""" return self._tod.get("customFields", None) @custom_fields.setter def custom_fields(self, custom_fields): self._tod["customFields"] = custom_fields class DSSInfoMessages(object): """ Contains a list of :class:`dataikuapi.dss.utils.DSSInfoMessage`. .. important:: Do not instantiate this class. """ def __init__(self, data): self._data = data def __repr__(self): return "DSSInfoMessages(has_messages: {}, max_severity: {}, has_error: {}, has_warning: {}, has_success: {}, messages: {})".format( self.has_messages, self.max_severity, self.has_error, self.has_warning, self.has_success, "[...]" if self.has_messages else "[]" ) @property def messages(self): """The messages as a list of :class:`dataikuapi.dss.utils.DSSInfoMessage`""" return [DSSInfoMessage(message) for message in self._data.get("messages", [])] @property def has_messages(self): """`True` if there is any message""" return self._data.get("anyMessage", False) @property def has_error(self): """`True` if there is any error message""" return self._data.get("error", False) @property def max_severity(self): """The max severity of the messages""" return self._data.get("maxSeverity", None) @property def has_success(self): """`True` if there is any success message""" return self._data.get("success", False) @property def has_warning(self): """`True` if there is any warning message""" return self._data.get("warning", False) class DSSInfoMessage(object): """ A message with a code, a title, a severity and a content. .. important:: Do not instantiate this class. """ def __init__(self, data): self._data = data def __str__(self): return "{severity} - {title} - {details}".format( severity=self.severity, title=self.title, details=self.details ) @property def severity(self): """The severity of the message""" return self._data.get("severity", None) @property def code(self): """The code of the message""" return self._data.get("code", None) @property def details(self): """The details of the message""" return self._data.get("details", None) @property def title(self): """The title of the message""" return self._data.get("title", None) @property def message(self): """The full message""" return self._data.get("message", None) dataiku-api-client-python-13.2.2/dataikuapi/dss/webapp.py000066400000000000000000000141671471211575100233350ustar00rootroot00000000000000from .future import DSSFuture from .utils import DSSTaggableObjectListItem class DSSWebApp(object): """ A handle for the webapp. .. important:: Do not instantiate this class directly, instead use :meth:`dataikuapi.dss.project.DSSProject.list_webapps`. """ def __init__(self, client, project_key, webapp_id): self.client = client self.project_key = project_key self.webapp_id = webapp_id def get_state(self): """ :returns: A wrapper object holding the webapp backend state. :rtype: :class:`dataikuapi.dss.webapp.DSSWebAppBackendState` """ state = self.client._perform_json("GET", "/projects/%s/webapps/%s/backend/state" % (self.project_key, self.webapp_id)) return DSSWebAppBackendState(self.webapp_id, state) def stop_backend(self): """ Stop the webapp backend. """ self.client._perform_empty("PUT", "/projects/%s/webapps/%s/backend/actions/stop" % (self.project_key, self.webapp_id)) def start_or_restart_backend(self): """ Start or restart the webapp backend. :returns: A future tracking the restart progress. :rtype: :class:`dataikuapi.dss.future.DSSFuture` """ resp = self.client._perform_json("PUT", "/projects/%s/webapps/%s/backend/actions/restart" % (self.project_key, self.webapp_id)) return DSSFuture.from_resp(self.client, resp) def get_settings(self): """ :returns: A handle for the webapp settings. :rtype: :class:`dataikuapi.dss.webapp.DSSWebAppSettings` """ data = self.client._perform_json("GET", "/projects/%s/webapps/%s" % (self.project_key, self.webapp_id)) return DSSWebAppSettings(self.client, self, data) def get_backend_client(self): return DSSWebAppBackendClient(self.client, self) class DSSWebAppBackendClient(object): """ A client to interact by API with a standard webapp backend """ def __init__(self, client, webapp): self.client = client self.webapp = webapp # If authenticating by API key, we are already targeting the proper URL, do nothing # but if we are authenticating by ticket, we are targeting the backend URL and need to target the base URL instead if self.client.internal_ticket is not None: from dataiku.base import remoterun base_proto = remoterun.get_env_var("DKU_BASE_PROTOCOL", "http") base_port = remoterun.get_env_var("DKU_BASE_PORT", "???") dss_url = "%s://%s:%s" % (base_proto, remoterun.get_env_var("DKU_BACKEND_HOST", "127.0.0.1"), base_port) else: dss_url = self.client.host self._base_url = "%s/web-apps-backends/%s/%s/" % (dss_url, self.webapp.project_key, self.webapp.webapp_id) @property def base_url(self): return self._base_url @property def session(self): return self.client._session def url_for_path(self, path): while path.startswith('/'): path = path[1:] return self._base_url + path class DSSWebAppBackendState(object): """ A wrapper object holding the webapp backend state. .. important:: Do not instantiate this class directly, instead use :meth:`dataikuapi.dss.webapp.DSSWebApp.get_state`. """ def __init__(self, webapp_id, state): self.webapp_id = webapp_id self._state = state @property def state(self): """ :returns: The webapp backend state as a dict containing the keys: * **projectKey**: the related project key, * **webAppId**: the webapp id, * **futureInfo**: the status of the last webapp start or restart job if such job exists (prefer using :attr:`dataikuapi.dss.webapp.DSSWebAppBackendState.running`). :rtype: python dict """ return self._state @property def running(self): """ :returns: Is the backend of the webapp currently running. :rtype: bool """ return "futureInfo" in self._state and \ "alive" in self._state["futureInfo"] and \ self._state["futureInfo"]["alive"] class DSSWebAppSettings(object): """ A handle for the webapp settings. .. important:: Do not instantiate this class directly, instead use :meth:`dataikuapi.dss.webapp.DSSWebApp.get_settings`. """ def __init__(self, client, webapp, data): self.client = client self.webapp = webapp self.data = data def get_raw(self): """ :returns: The webapp settings as a dict containing among other keys: * **id**: the webapp id, * **name**: the webapp name, * **type**: the webapp type (e.g. "STANDARD"), * **projectKey**: the related project key, * **params**: a dict containing other information depending on the webapp type such as the source code. :rtype: python dict """ return self.data def save(self): """ Save the current webapp settings. """ self.client._perform_json("PUT", "/projects/%s/webapps/%s" % (self.webapp.project_key, self.webapp.webapp_id), body=self.data) class DSSWebAppListItem(DSSTaggableObjectListItem): """ An item in a list of webapps. .. important:: Do not instantiate this class directly, instead use :meth:`dataikuapi.dss.project.DSSProject.list_webapps`. """ def __init__(self, client, data): super(DSSWebAppListItem, self).__init__(data) self.client = client def to_webapp(self): """ Convert the current item. :returns: A handle for the webapp. :rtype: :class:`dataikuapi.dss.webapp.DSSWebApp` """ return DSSWebApp(self.client, self._data["projectKey"], self._data["id"]) @property def id(self): """ :returns: The id of the webapp. :rtype: string """ return self._data["id"] @property def name(self): """ :returns: The name of the webapp. :rtype: string """ return self._data["name"] dataiku-api-client-python-13.2.2/dataikuapi/dss/wiki.py000066400000000000000000000401441471211575100230140ustar00rootroot00000000000000from .discussion import DSSObjectDiscussions from ..utils import DataikuException import sys import copy import re if sys.version_info >= (3,0): import urllib.parse dku_quote_fn = urllib.parse.quote else: import urllib dku_quote_fn = urllib.quote class DSSWiki(object): """ A handle to manage the wiki of a project .. important:: Do not instantiate this class directly, instead use :meth:`dataikuapi.dss.project.DSSProject.get_wiki` """ def __init__(self, client, project_key): self.client = client self.project_key = project_key def get_settings(self): """ Get wiki settings :returns: a handle to manage the wiki settings (taxonomy, home article) :rtype: :class:`dataikuapi.dss.wiki.DSSWikiSettings` """ return DSSWikiSettings(self.client, self.project_key, self.client._perform_json("GET", "/projects/%s/wiki/" % (self.project_key))) def get_article(self, article_id_or_name): """ Get a wiki article :param str article_id_or_name: reference to the article, it can be its ID or its name :returns: a handle to manage the Article :rtype: :class:`dataikuapi.dss.wiki.DSSWikiArticle` """ return DSSWikiArticle(self.client, self.project_key, article_id_or_name) def __flatten_taxonomy__(self, taxonomy): """ Private recursive method to get the flatten list of article IDs from the taxonomy :param list taxonomy: :returns: list of articles :rtype: list of :class:`dataikuapi.dss.wiki.DSSWikiArticle` """ article_list = [] for article in taxonomy: article_list.append(self.get_article(article['id'])) article_list += self.__flatten_taxonomy__(article['children']) return article_list def list_articles(self): """ Get a list of all the articles in form of :class:`dataikuapi.dss.wiki.DSSWikiArticle` objects :returns: list of articles :rtype: list of :class:`dataikuapi.dss.wiki.DSSWikiArticle` """ return self.__flatten_taxonomy__(self.get_settings().get_taxonomy()) def create_article(self, article_name, parent_id=None, content=None): """ Create a wiki article and return a handle to interact with it. :param str article_name: the article name :param str parent_id: the parent article ID (or None if the article has to be at root level, defaults to **None**) :param str content: the article content (defaults to **None**) :returns: the created article :rtype: :class:`dataikuapi.dss.wiki.DSSWikiArticle` """ body = { "projectKey": self.project_key, "name": article_name, "parent": parent_id } result = self.client._perform_json("POST", "/projects/%s/wiki/" % (self.project_key), body=body) article = DSSWikiArticle(self.client, self.project_key, result['article']['id']) # set article content if given if content is not None: article_data = article.get_data() article_data.set_body(content) article_data.save() return article def get_export_stream(self, paper_size="A4", export_attachment=False): """ Download the whole wiki of the project in PDF format as a binary stream. .. warning:: You need to close the stream after download. Failure to do so will result in the DSSClient becoming unusable. :param str paper_size: the format of the exported page, can be one of 'A4', 'A3', 'US_LETTER' or 'LEDGER' (defaults to **A4**) :param bool export_attachment: export the attachments of the article(s) in addition to the pdf in a zip file (defaults to **False**) :returns: the exported pdf or zip file as a stream """ body = { "paperSize": paper_size, "exportAttachments": export_attachment } return self.client._perform_raw("POST", "/projects/%s/wiki/actions/export" % (self.project_key), body=body) def export_to_file(self, path, paper_size="A4", export_attachment=False): """ Download the whole wiki of the project in PDF format into the given output file. :param str path: the path of the file where the pdf or zip file will be downloaded :param str paper_size: the format of the exported page, can be one of 'A4', 'A3', 'US_LETTER' or 'LEDGER' (defaults to **A4**) :param bool export_attachment: export the attachments of the article(s) in addition to the pdf in a zip file (defaults to **False**) """ with self.get_export_stream(paper_size=paper_size, export_attachment=export_attachment) as stream: with open(path, 'wb') as f: for chunk in stream.iter_content(chunk_size=10000): if chunk: f.write(chunk) f.flush() class DSSWikiSettings(object): """ Global settings for the wiki, including taxonomy. Call save() to save """ def __init__(self, client, project_key, settings): """Do not instantiate this class directly, instead use :meth:`dataikuapi.dss.wiki.DSSWiki.get_settings`""" self.client = client self.project_key = project_key self.settings = settings def get_taxonomy(self): """ Get the taxonomy. The taxonomy is an array listing at top level the root article IDs and their children in a tree format. Every existing article of the wiki has to be in the taxonomy once and only once. For instance: .. code-block:: python [ { 'id': 'article1', 'children': [] }, { 'id': 'article2', 'children': [ { 'id': 'article3', 'children': [] } ] } ] .. note :: Note that this is a direct reference, not a copy, so modifications to the returned object will be reflected when saving :returns: The taxonomy :rtype: list """ return self.settings["taxonomy"] if "taxonomy" in self.settings else [] def __retrieve_article_in_taxonomy__(self, taxonomy, article_id, remove=False): """ Private recursive method that get the sub tree structure from the taxonomy for a specific article :param list taxonomy: the current level of taxonomy :param str article_id: the article to retrieve :param bool remove: either remove the sub tree structure or not (defaults to **False**) :returns: the sub tree structure at a specific article level :rtype: dict """ idx = 0 for tax_article in taxonomy: if tax_article["id"] == article_id: ret = taxonomy.pop(idx) if remove else taxonomy[idx] return ret children_ret = self.__retrieve_article_in_taxonomy__(tax_article["children"], article_id, remove) if children_ret is not None: return children_ret idx += 1 return None def move_article_in_taxonomy(self, article_id, parent_article_id=None): """ An helper to update the taxonomy by moving an article with its children as a child of another article :param str article_id: the main article ID :param str parent_article_id: the new parent article ID or None for root level (defaults to **None**) """ old_taxonomy = copy.deepcopy(self.settings["taxonomy"]) tax_article = self.__retrieve_article_in_taxonomy__(self.settings["taxonomy"], article_id, True) if tax_article is None: raise DataikuException("Article not found: %s" % (article_id)) if parent_article_id is None: self.settings["taxonomy"].append(tax_article) else: tax_parent_article = self.__retrieve_article_in_taxonomy__(self.settings["taxonomy"], parent_article_id, False) if tax_parent_article is None: self.settings["taxonomy"] = old_taxonomy raise DataikuException("Parent article not found (or is one of the article descendants): %s" % (parent_article_id)) tax_parent_article["children"].append(tax_article) def set_taxonomy(self, taxonomy): """ Set the taxonomy :param list taxonomy: the taxonomy """ self.settings["taxonomy"] = taxonomy def get_home_article_id(self): """ Get the home article ID :returns: The home article ID :rtype: str """ return self.settings["homeArticleId"] if "homeArticleId" in self.settings else None def set_home_article_id(self, home_article_id): """ Set the home article ID :param str home_article_id: the home article ID """ self.settings["homeArticleId"] = home_article_id def save(self): """ Save the current settings to the backend """ self.settings = self.client._perform_json("PUT", "/projects/%s/wiki/" % (self.project_key), body=self.settings) class DSSWikiArticle(object): """ A handle to manage an article """ def __init__(self, client, project_key, article_id_or_name): """Do not instantiate this class directly, instead use :meth:`dataikuapi.dss.wiki.DSSWiki.get_article`""" self.client = client self.project_key = project_key # Retrieve the real article id article_data = self.client._perform_json("GET", "/projects/%s/wiki/%s" % (project_key, article_id_or_name)) self.article_id = article_data["article"]['id'] # encode in UTF-8 if its python2 and unicode if sys.version_info < (3,0) and isinstance(self.article_id, unicode): self.article_id = self.article_id.encode('utf-8') def get_data(self): """ Get article data handle :returns: the article data handle :rtype: :class:`dataikuapi.dss.wiki.DSSWikiArticleData` """ article_data = self.client._perform_json("GET", "/projects/%s/wiki/%s" % (self.project_key, self.article_id)) return DSSWikiArticleData(self.client, self.project_key, self.article_id, article_data) def upload_attachement(self, fp, filename): """ Upload and attach a file to the article. .. note :: Note that the type of file will be determined by the filename extension :param file fp: A file-like object that represents the upload file :param str filename: The attachement filename """ clean_filename = re.sub(r'[^A-Za-z0-9 ._-]+', '', filename) self.client._perform_json("POST", "/projects/%s/wiki/%s/upload" % (self.project_key, dku_quote_fn(self.article_id)), files={"file":(clean_filename, fp)}) def get_uploaded_file(self, upload_id): """ Download an attachment of the article .. warning:: You need to close the stream after download. Failure to do so will result in the DSSClient becoming unusable. :param str upload_id: The attachement upload id :returns: The attachment file as a stream :rtype: :class:`requests.Response` """ return self.client._perform_raw("GET", "/projects/%s/wiki/%s/uploads/%s" % (self.project_key, self.article_id, upload_id)) def get_export_stream(self, paper_size="A4", export_children=False, export_attachment=False): """ Download the article in PDF format as a binary stream. .. warning:: You need to close the stream after download. Failure to do so will result in the DSSClient becoming unusable. :param str paper_size: the format of the exported page, can be one of 'A4', 'A3', 'US_LETTER' or 'LEDGER' (defaults to **A4**) :param bool export_children: export the children of the article in the pdf (defaults to **False**) :param bool export_attachment: export the attachments of the article(s) in addition to the pdf in a zip file (defaults to **False**) :returns: the exported pdf or zip file as a stream :rtype: :class:`requests.Response` """ body = { "paperSize": paper_size, "exportChildren": export_children, "exportAttachments": export_attachment } return self.client._perform_raw("POST", "/projects/%s/wiki/%s/actions/export" % (self.project_key, self.article_id), body=body) def export_to_file(self, path, paper_size="A4", export_children=False, export_attachment=False): """ Download the article in PDF format into the given output file. :param str path: the path of the file where the pdf or zip file will be downloaded :param str paper_size: the format of the exported page, can be one of 'A4', 'A3', 'US_LETTER' or 'LEDGER' (defaults to **A4**) :param bool export_children: export the children of the article in the pdf (defaults to **False**) :param bool export_attachment: export the attachments of the article(s) in addition to the pdf in a zip file (defaults to **False**) """ with self.get_export_stream(paper_size=paper_size, export_children=export_children, export_attachment=export_attachment) as stream: with open(path, 'wb') as f: for chunk in stream.iter_content(chunk_size=10000): if chunk: f.write(chunk) f.flush() def delete(self): """ Delete the article """ self.client._perform_empty("DELETE", "/projects/%s/wiki/%s" % (self.project_key, dku_quote_fn(self.article_id))) def get_object_discussions(self): """ Get a handle to manage discussions on the article :returns: the handle to manage discussions :rtype: :class:`dataikuapi.dss.wiki.DSSObjectDiscussions` """ return DSSObjectDiscussions(self.client, self.project_key, "ARTICLE", self.article_id) class DSSWikiArticleData(object): """ A handle to manage an article """ def __init__(self, client, project_key, article_id, article_data): """Do not instantiate this class directly, instead use :meth:`dataikuapi.dss.wiki.DSSWikiArticle.get_data`""" self.client = client self.project_key = project_key self.article_id = article_id # don't need to check unicode here (already done in DSSWikiArticle) self.article_data = article_data def get_body(self): """ Get the markdown body as string :returns: the article body :rtype: str """ return self.article_data["payload"] if "payload" in self.article_data else None def set_body(self, content): """ Set the markdown body :param str content: the article content """ self.article_data["payload"] = content def get_metadata(self): """ Get the article and attachment metadata .. note :: Note that this is a direct reference, not a copy, so modifications to the returned object will be reflected when saving :returns: the article metadata :rtype: dict """ return self.article_data["article"] if "article" in self.article_data else None def set_metadata(self, metadata): """ Set the article metadata :param dict metadata: the article metadata """ self.article_data["article"] = metadata def get_name(self): """ Get the article name :returns: the article name :rtype: str """ return self.article_data["article"]["name"] def set_name(self, name): """ Set the article name :param str name: the article name """ self.article_data["article"]["name"] = name def save(self): """ Save the current article data to the backend. """ self.article_data = self.client._perform_json("PUT", "/projects/%s/wiki/%s" % (self.project_key, dku_quote_fn(self.article_id)), body=self.article_data) dataiku-api-client-python-13.2.2/dataikuapi/dss/workspace.py000066400000000000000000000140711471211575100240470ustar00rootroot00000000000000from dataikuapi.dss.app import DSSApp from dataikuapi.dss.dataset import DSSDataset from dataikuapi.dss.wiki import DSSWikiArticle class DSSWorkspace: """ A handle to interact with a workspace on the DSS instance. Do not create this class directly, instead use :meth:`dataikuapi.DSSClient.get_workspace` """ def __init__(self, client, workspace_key): self.client = client self.workspace_key = workspace_key def get_settings(self): """ Gets the settings of this workspace. :returns: a handle to read, modify and save the settings :rtype: :class:`DSSWorkspaceSettings` """ return DSSWorkspaceSettings(self, self.client._perform_json("GET", "/workspaces/%s" % self.workspace_key)) def list_objects(self): """ List the objects in this workspace :returns: The list of the objects :rtype: list of :class:`.DSSWorkspaceObject` """ objects = self.client._perform_json("GET", "/workspaces/%s/objects" % self.workspace_key) return [DSSWorkspaceObject(self, object) for object in objects] def add_object(self, object): """ Add an object to this workspace. Object can be of different shapes (:class:`dataikuapi.dss.dataset.DSSDataset`, :class:`dataikuapi.dss.wiki.DSSWikiArticle`, :class:`dataikuapi.dss.app.DSSApp`, :class:`.DSSWorkspaceHtmlLinkObject` or a :class:`.dict` that contains the raw data) """ if isinstance(object, DSSDataset): data = {"reference": {"projectKey": object.project_key, "type": "DATASET", "id": object.dataset_name}} elif isinstance(object, DSSWikiArticle): data = {"reference": {"projectKey": object.project_key, "type": "ARTICLE", "id": object.article_id}} elif isinstance(object, DSSApp): data = {"appId": object.app_id} elif isinstance(object, DSSWorkspaceHtmlLinkObject): data = {"htmlLink": {"name": object.name, "url": object.url, "description": object.description}} elif isinstance(object, dict): data = object else: raise ValueError("Unsupported object type") self.client._perform_json("POST", "/workspaces/%s/objects" % self.workspace_key, body=data) def delete(self): """ Delete the workspace This call requires Administrator rights on the workspace. """ return self.client._perform_empty("DELETE", "/workspaces/%s" % self.workspace_key) class DSSWorkspaceHtmlLinkObject: def __init__(self, name, url, description): self.name = name self.url = url self.description = description class DSSWorkspaceObject: """ A handle on an object of a workspace Do not create this class directly, instead use :meth:`dataikuapi.dss.DSSWorkspace.list_objects` """ def __init__(self, workspace, data): self.workspace = workspace self.data = data def get_raw(self): return self.data def remove(self): """ Remove this object from the workspace This call requires Contributor rights on the workspace. """ self.workspace.client._perform_empty( "DELETE", "/workspaces/%s/objects/%s" % (self.workspace.workspace_key, self.data['id'])) class DSSWorkspaceSettings: """ A handle on the settings of a workspace Do not create this class directly, instead use :meth:`dataikuapi.dss.DSSWorkspace.get_settings` """ def __init__(self, workspace, settings): self.workspace = workspace self.settings = settings def get_raw(self): return self.settings @property def display_name(self): """ Get or set the name of the workspace :rtype: :class:`str` """ return self.settings['displayName'] @display_name.setter def display_name(self, value): self.settings['displayName'] = value @property def color(self): """ Get or set the background color of the workspace (using #xxxxxx syntax) :rtype: :class:`str` """ return self.settings['color'] @color.setter def color(self, value): self.settings['color'] = value @property def description(self): """ Get or set the description of the workspace :rtype: :class:`str` """ return self.settings['description'] @description.setter def description(self, value): self.settings['description'] = value @property def permissions(self): """ Get or set the permissions controlling who is a member, contributor or admin of the workspace If user is not workspace admin, the permissions field is redacted to None. :rtype: list of :class:`dict` or :class:`None` """ return self.settings['permissions'] if 'permissions' in self.settings else None @permissions.setter def permissions(self, value): self.settings['permissions'] = value def save(self): """ Save the changes made on the settings This call requires Administrator rights on the workspace. """ self.workspace.client._perform_empty( "PUT", "/workspaces/%s" % self.workspace.workspace_key, body=self.settings) class DSSWorkspacePermissionItem: @classmethod def admin_group(cls, group): return {"group": group, "admin": True, "write": True, "read": True} @classmethod def contributor_group(cls, group): return {"group": group, "admin": False, "write": True, "read": True} @classmethod def member_group(cls, group): return {"group": group, "admin": False, "write": False, "read": True} @classmethod def admin_user(cls, user): return {"user": user, "admin": True, "write": True, "read": True} @classmethod def contributor_user(cls, user): return {"user": user, "admin": False, "write": True, "read": True} @classmethod def member_user(cls, user): return {"user": user, "admin": False, "write": False, "read": True} dataiku-api-client-python-13.2.2/dataikuapi/dss_plugin_mlflow/000077500000000000000000000000001471211575100244325ustar00rootroot00000000000000dataiku-api-client-python-13.2.2/dataikuapi/dss_plugin_mlflow/__init__.py000066400000000000000000000000371471211575100265430ustar00rootroot00000000000000from .utils import MLflowHandledataiku-api-client-python-13.2.2/dataikuapi/dss_plugin_mlflow/artifact_repository.py000066400000000000000000000207311471211575100311030ustar00rootroot00000000000000import os import tempfile import urllib import re import sys from dataikuapi import DSSClient if sys.version_info > (3, 0): # MLflow only work for python3 (in > 1.18.0) from pathlib import PurePosixPath, Path def parse_dss_managed_folder_uri(uri): parsed = urllib.parse.urlparse(uri) if parsed.scheme != "dss-managed-folder": raise Exception("Not a DSS Managed Folder URI: %s" % uri) pattern = re.compile("^(\w+\.)?\w{8}") if not parsed.netloc or not pattern.match(parsed.netloc): raise Exception("Could not find a managed folder id in URI: %s" % uri) return parsed class PluginDSSManagedFolderArtifactRepository: def __init__(self, artifact_uri): self.client = DSSClient( os.environ.get("DSS_MLFLOW_HOST"), api_key=os.environ.get("DSS_MLFLOW_APIKEY"), internal_ticket=os.environ.get("DSS_MLFLOW_INTERNAL_TICKET"), insecure_tls= self._should_use_insecure_tls ) if 'DSS_MLFLOW_VERIFY_CERT' in os.environ: self.client._session.verify = os.environ.get('DSS_MLFLOW_VERIFY_CERT') self.project = self.client.get_project(os.environ.get("DSS_MLFLOW_PROJECTKEY")) parsed_uri = parse_dss_managed_folder_uri(artifact_uri) self.managed_folder = self.__get_managed_folder(parsed_uri.netloc) self.base_artifact_path = PurePosixPath(parsed_uri.path) @property def _should_use_insecure_tls(self): insecure_tls = os.environ.get("MLFLOW_TRACKING_INSECURE_TLS") # this env variable is documented in MLFlow, it's not ours, we must parse it similarly to how MLflow does it. # https://github.com/mlflow/mlflow/blob/dde5d79f57eada1820da1cafe4d58eeff476a022/mlflow/environment_variables.py#L70 if insecure_tls is not None: insecure_tls = insecure_tls.lower() return insecure_tls in ["true", "1"] else: return False def __get_managed_folder(self, managed_folder_smart_id): chunks = managed_folder_smart_id.split('.') if len(chunks) == 1: return self.project.get_managed_folder(chunks[0]) elif len(chunks) == 2: project = self.client.get_project(chunks[0]) return project.get_managed_folder(chunks[1]) else: raise Exception("Invalid managed folder id: %s" % managed_folder_smart_id) def log_artifact(self, local_file, artifact_path=None): """ Log a local file as an artifact, optionally taking an ``artifact_path`` to place it in within the run's artifacts. Run artifacts can be organized into directories, so you can place the artifact in a directory this way. :param local_file: Path to artifact to log :param artifact_path: Directory within the run's artifact directory in which to log the artifact. """ path = self.base_artifact_path if artifact_path is not None: path /= artifact_path self.managed_folder.put_file(str(path / os.path.basename(local_file)), open(local_file, "rb")) def log_artifacts(self, local_dir, artifact_path=None): """ Log the files in the specified local directory as artifacts, optionally taking an ``artifact_path`` to place them in within the run's artifacts. :param local_dir: Directory of local artifacts to log :param artifact_path: Directory within the run's artifact directory in which to log the artifacts """ path = self.base_artifact_path if artifact_path is not None: path /= artifact_path self.managed_folder.upload_folder(str(path), local_dir) def list_artifacts(self, path=""): """ Return all the artifacts for this run_id directly under path. If path is a file, returns an empty list. Will error if path is neither a file nor directory. :param path: Relative source path that contains desired artifacts :return: List of artifacts as FileInfo listed directly under path. """ from mlflow.entities import FileInfo param_path = path if path: if isinstance(path, FileInfo): path = self.base_artifact_path / path.path else: path = self.base_artifact_path / path else: path = self.base_artifact_path files = self.managed_folder.list_contents().get("items", []) ret = [] for file in files: if path == self.base_artifact_path / file["path"]: # Exact match, there can be only one return [FileInfo(param_path, False, file.get("size"))] candidate = PurePosixPath(file["path"]) if path in candidate.parents and self.base_artifact_path in candidate.parents: ret.append(FileInfo(candidate.relative_to(self.base_artifact_path).as_posix(), False, file.get("size"))) return ret def download_artifacts(self, artifact_path, dst_path=None): """ Download an artifact file or directory to a local directory if applicable, and return a local path for it. The caller is responsible for managing the lifecycle of the downloaded artifacts. :param artifact_path: Relative source path to the desired artifacts. :param dst_path: Absolute path of the local filesystem destination directory to which to download the specified artifacts. This directory must already exist. If unspecified, the artifacts will either be downloaded to a new uniquely-named directory on the local filesystem or will be returned directly in the case of the LocalArtifactRepository. :return: Absolute path of the local filesystem location containing the desired artifacts. """ from mlflow.exceptions import MlflowException from mlflow.protos.databricks_pb2 import INVALID_PARAMETER_VALUE, RESOURCE_DOES_NOT_EXIST if dst_path is None: dst_path = tempfile.mkdtemp() dst_path = Path(dst_path).absolute() if not dst_path.exists(): raise MlflowException( message=("The destination path for downloaded artifacts does not" " exist! Destination path: {dst_path}".format(dst_path=dst_path)), error_code=RESOURCE_DOES_NOT_EXIST, ) elif not dst_path.is_dir(): raise MlflowException( message=("The destination path for downloaded artifacts must be a directory!" " Destination path: {dst_path}".format(dst_path=dst_path)), error_code=INVALID_PARAMETER_VALUE, ) for path in self.list_artifacts(artifact_path): local_path = dst_path / path.path local_path.parent.mkdir(parents=True, exist_ok=True) self._download_file(path.path, local_path) return dst_path / artifact_path def _download_file(self, artifact_path, local_path): """ Download the file at the specified relative remote path and saves it at the specified local path. :param artifact_path: Source path to the remote file, relative to the root directory of the artifact repository. :param local_path: The path to which to save the downloaded file. """ full_path = self.base_artifact_path / artifact_path with self.managed_folder.get_file(str(full_path)) as remote_file: with open(local_path, "wb") as local_file: for line in remote_file: local_file.write(line) def delete_artifacts(self, artifact_path=None): """ Delete the artifacts at the specified location. Supports the deletion of a single file or of a directory. Deletion of a directory is recursive. :param artifact_path: Path of the artifact to delete """ path = ( self.base_artifact_path / artifact_path if artifact_path else self.base_artifact_path ) self.managed_folder.delete_file(str(path)) def verify_artifact_path(artifact_path): from mlflow.exceptions import MlflowException from mlflow.utils.validation import path_not_unique, bad_path_message if artifact_path and path_not_unique(artifact_path): raise MlflowException( "Invalid artifact path: '%s'. %s" % (artifact_path, bad_path_message(artifact_path)) ) dataiku-api-client-python-13.2.2/dataikuapi/dss_plugin_mlflow/header_provider.py000066400000000000000000000011211471211575100301410ustar00rootroot00000000000000import os class PluginDSSHeaderProvider: def in_context(self): env_variables = {"DSS_MLFLOW_HEADER", "DSS_MLFLOW_TOKEN", "DSS_MLFLOW_PROJECTKEY"} if len(env_variables.difference(set(os.environ))) == 0: return True def request_headers(self): headers = { os.environ.get("DSS_MLFLOW_HEADER"): os.environ.get("DSS_MLFLOW_TOKEN"), "x-dku-mlflow-project-key": os.environ.get("DSS_MLFLOW_PROJECTKEY"), "x-dku-mlflow-managed-folder-id": os.environ.get("DSS_MLFLOW_MANAGED_FOLDER_ID"), } return headers dataiku-api-client-python-13.2.2/dataikuapi/dss_plugin_mlflow/utils.py000066400000000000000000000166641471211575100261610ustar00rootroot00000000000000from ..utils import DataikuException from ..dss.managedfolder import DSSManagedFolder import logging import os import sys import tempfile import shutil from base64 import b64encode class MLflowHandle: def __init__(self, client, project, managed_folder, host=None): """ Add the MLflow-plugin parts of dataikuapi to MLflow local setup. This method deals with 1. importing dynamically the DSS MLflow plugin: MLflow uses entrypoints==0.3 to load entrypoints from plugins at import time. We add dss-mlflow-plugin entrypoints dynamically by adding them in sys.path at call time. 2. Setup the correct environment variables to setup the plugin to work with DSS backend as the tracking backend and enable using DSS managed folder as artifact location. """ self.project = project self.project_key = project.project_key self.client = client self.mlflow_env = {} self.env_save = {} # Load DSS as the plugin self.tempdir = tempfile.mkdtemp() plugin_dir = os.path.join(self.tempdir, "dss-plugin-mlflow.egg-info") if not os.path.isdir(plugin_dir): os.makedirs(plugin_dir) with open(os.path.join(plugin_dir, "entry_points.txt"), "w") as f: f.write( "[mlflow.request_header_provider]\n" "unused=dataikuapi.dss_plugin_mlflow.header_provider:PluginDSSHeaderProvider\n" "[mlflow.artifact_repository]\n" "dss-managed-folder=dataikuapi.dss_plugin_mlflow.artifact_repository:PluginDSSManagedFolderArtifactRepository\n" ) sys.path.insert(0, self.tempdir) if sys.version_info > (3, 4): from importlib import reload import mlflow import mlflow.store.artifact.artifact_repository_registry import mlflow.tracking.request_header.registry reload(mlflow.tracking.request_header.registry) # Reload the request_header registry in case MLflow was imported beforehand reload(mlflow.store.artifact.artifact_repository_registry) # Reload the artifact_repository_registry in case MLflow was imported beforehand mlflow.set_tracking_uri(None) # if user has changed tracking backend manually before mlflow.end_run() # if user already created a run with another tracking backend self.remove_dataiku_duplicates_in_request_header_registry() # Setup authentication if client._session.auth is not None: self.mlflow_env.update({ "DSS_MLFLOW_HEADER": "Authorization", "DSS_MLFLOW_TOKEN": "Basic {}".format( b64encode("{}:".format(self.client._session.auth.username).encode("utf-8")).decode("utf-8")), "DSS_MLFLOW_APIKEY": self.client.api_key }) elif client.internal_ticket: self.mlflow_env.update({ "DSS_MLFLOW_HEADER": "X-DKU-APITicket", "DSS_MLFLOW_TOKEN": self.client.internal_ticket, "DSS_MLFLOW_INTERNAL_TICKET": self.client.internal_ticket }) if client._session.verify == False: self.mlflow_env.update({"MLFLOW_TRACKING_INSECURE_TLS": "true"}) self.mlflow_env.update({"MLFLOW_TRACKING_SERVER_CERT_PATH": None}) else: self.mlflow_env.update({"MLFLOW_TRACKING_INSECURE_TLS": "false"}) if isinstance(client._session.verify, str): # when in a notebook with encrypted RPC, client._session.verify will have the RPC server cert # so we have to store it and restore it when creating again the api client (such as in the # artifact repository plugin) self.mlflow_env.update({"MLFLOW_TRACKING_SERVER_CERT_PATH": client._session.verify}) self.mlflow_env.update({"DSS_MLFLOW_VERIFY_CERT": client._session.verify}) mf_full_id = None if isinstance(managed_folder, DSSManagedFolder): mf_full_id = managed_folder.project.project_key + "." + managed_folder.id elif isinstance(managed_folder, str): mf_full_id = managed_folder else: try: from dataiku import Folder if isinstance(managed_folder, Folder): mf_full_id = managed_folder.name except ImportError: pass if not mf_full_id: raise TypeError('Type of managed_folder must be "str", "DSSManagedFolder" or "dataiku.Folder".') if "." not in mf_full_id: mf_full_id = self.project_key + "." + mf_full_id mf_project = mf_full_id.split(".")[0] mf_id = mf_full_id.split(".")[1] try: client.get_project(mf_project).get_managed_folder(mf_id).get_definition() except DataikuException as e: if "NotFoundException" in str(e): logging.error('The managed folder "%s" does not exist, please create it in your project flow before running this command.' % (mf_full_id)) raise # Set host, tracking URI, project key and managed_folder_id self.mlflow_env.update({ "DSS_MLFLOW_PROJECTKEY": self.project_key, "MLFLOW_TRACKING_URI": self.client.host + "/dip/publicapi" if host is None else host, "DSS_MLFLOW_HOST": self.client.host, "DSS_MLFLOW_MANAGED_FOLDER_ID": mf_full_id }) self.override_env(self.mlflow_env) def remove_dataiku_duplicates_in_request_header_registry(self): try: # Try to make sure we are not loading our MLflow plugin twice # see https://app.shortcut.com/dataiku/story/210232 from mlflow.tracking.request_header.registry import _request_header_provider_registry registry = _request_header_provider_registry._registry seen = set() i = 0 while i < len(registry): entrypoint = registry[i] entrypoint_name = "{}.{}".format(entrypoint.__class__.__module__, entrypoint.__class__.__name__) if entrypoint_name in seen and 'dataikuapi' in entrypoint_name: registry.pop(i) else: seen.add(entrypoint_name) i += 1 except Exception as e: logging.warning(str(e), exc_info=True) def clear(self): shutil.rmtree(self.tempdir) self.restore_env(self.mlflow_env) def override_env(self, overrides): assert len(self.env_save) == 0, "Environment variables already overriden, override_env called twice?" for key, value in overrides.items(): try: current_value = os.environ.pop(key) self.env_save[key] = current_value if value is None: # allow to unset env variables by using 'None' as a value in overrides del os.environ[key] except KeyError: # Means that overriden var does not exist, safe to ignore. pass if value is not None: os.environ[key] = value def restore_env(self, overrides): for variable in overrides: os.environ.pop(variable, None) os.environ.update(self.env_save) def import_mlflow(self): import mlflow return mlflow def __enter__(self): return self.import_mlflow() def __exit__(self, exc_type, exc_value, traceback): self.clear() dataiku-api-client-python-13.2.2/dataikuapi/dssclient.py000066400000000000000000002255641471211575100232630ustar00rootroot00000000000000import json, warnings import sys if sys.version_info >= (3,0): import urllib.parse dku_quote_fn = urllib.parse.quote else: import urllib dku_quote_fn = urllib.quote from requests import Session from requests import exceptions from requests.auth import HTTPBasicAuth from .iam.settings import DSSSSOSettings, DSSLDAPSettings, DSSAzureADSettings from .dss.data_collection import DSSDataCollection, DSSDataCollectionListItem from .dss.feature_store import DSSFeatureStore from .dss.notebook import DSSNotebook from .dss.future import DSSFuture from .dss.projectfolder import DSSProjectFolder from .dss.project import DSSProject from .dss.app import DSSApp, DSSAppListItem from .dss.plugin import DSSPlugin from .dss.admin import DSSGlobalApiKeyListItem, DSSPersonalApiKeyListItem, DSSUser, DSSUserActivity, DSSOwnUser, DSSGroup, DSSConnection, DSSConnectionListItem, DSSGeneralSettings, DSSCodeEnv, DSSGlobalApiKey, DSSCluster, DSSCodeStudioTemplate, DSSCodeStudioTemplateListItem, DSSGlobalUsageSummary, DSSInstanceVariables, DSSPersonalApiKey, DSSAuthorizationMatrix from .dss.messaging_channel import DSSMailMessagingChannel, DSSMessagingChannelListItem, DSSMessagingChannel from .dss.meaning import DSSMeaning from .dss.sqlquery import DSSSQLQuery from .dss.discussion import DSSObjectDiscussions from .dss.apideployer import DSSAPIDeployer from .dss.projectdeployer import DSSProjectDeployer from .dss.unifiedmonitoring import DSSUnifiedMonitoring from .dss.utils import DSSInfoMessages, Enum from .dss.workspace import DSSWorkspace import os.path as osp from .utils import DataikuException, dku_basestring_type class DSSClient(object): """Entry point for the DSS API client""" def __init__(self, host, api_key=None, internal_ticket=None, extra_headers=None, insecure_tls=False): """ Instantiate a new DSS API client on the given host with the given API key. API keys can be managed in DSS on the project page or in the global settings. The API key will define which operations are allowed for the client. """ self.api_key = api_key self.internal_ticket = internal_ticket self.host = host self._session = Session() if insecure_tls: self._session.verify = False if self.api_key is not None: self._session.auth = HTTPBasicAuth(self.api_key, "") elif self.internal_ticket is not None: self._session.headers.update({"X-DKU-APITicket" : self.internal_ticket}) else: raise ValueError("API Key is required") if extra_headers is not None: self._session.headers.update(extra_headers) ######################################################## # Futures ######################################################## def list_futures(self, as_objects=False, all_users=False): """ List the currently-running long tasks (a.k.a futures) :param boolean as_objects: if True, each returned item will be a :class:`dataikuapi.dss.future.DSSFuture` :param boolean all_users: if True, returns futures for all users (requires admin privileges). Else, only returns futures for the user associated with the current authentication context (if any) :return: list of futures. if as_objects is True, each future in the list is a :class:`dataikuapi.dss.future.DSSFuture`. Else, each future in the list is a dict. Each dict contains at least a 'jobId' field :rtype: list of :class:`dataikuapi.dss.future.DSSFuture` or list of dict """ list = self._perform_json("GET", "/futures/", params={"withScenarios":False, "withNotScenarios":True, 'allUsers' : all_users}) if as_objects: return [DSSFuture(self, state['jobId'], state) for state in list] else: return list def list_running_scenarios(self, all_users=False): """ List the running scenarios :param boolean all_users: if True, returns scenarios for all users (requires admin privileges). Else, only returns scenarios for the user associated with the current authentication context (if any) :return: list of running scenarios, each one as a dict containing at least a "jobId" field for the future hosting the scenario run, and a "payload" field with scenario identifiers :rtype: list of dicts """ return self._perform_json("GET", "/futures/", params={"withScenarios":True, "withNotScenarios":False, 'allUsers' : all_users}) def get_future(self, job_id): """ Get a handle to interact with a specific long task (a.k.a future). This notably allows aborting this future. :param str job_id: the identifier of the desired future (which can be returned by :py:meth:`list_futures` or :py:meth:`list_running_scenarios`) :returns: A handle to interact the future :rtype: :class:`dataikuapi.dss.future.DSSFuture` """ return DSSFuture(self, job_id) ######################################################## # Notebooks ######################################################## def list_running_notebooks(self, as_objects=True): """ List the currently-running Jupyter notebooks :param boolean as_objects: if True, each returned item will be a :class:`dataikuapi.dss.notebook.DSSNotebook` :return: list of notebooks. if as_objects is True, each entry in the list is a :class:`dataikuapi.dss.notebook.DSSNotebook`. Else, each item in the list is a dict which contains at least a "name" field. :rtype: list of :class:`dataikuapi.dss.notebook.DSSNotebook` or list of dict """ notebook_list = self._perform_json("GET", "/admin/notebooks/") if as_objects: return [DSSNotebook(self, notebook['projectKey'], notebook['name'], notebook) for notebook in notebook_list] else: return notebook_list ######################################################## # Project folders ######################################################## def get_root_project_folder(self): """ Get a handle to interact with the root project folder. :returns: A :class:`dataikuapi.dss.projectfolder.DSSProjectFolder` to interact with this project folder """ return self.get_project_folder("ROOT") def get_project_folder(self, project_folder_id): """ Get a handle to interact with a project folder. :param str project_folder_id: the project folder ID of the desired project folder :returns: A :class:`dataikuapi.dss.projectfolder.DSSProjectFolder` to interact with this project folder """ data = self._perform_json("GET", "/project-folders/%s" % project_folder_id) return DSSProjectFolder(self, data) ######################################################## # Projects ######################################################## def list_project_keys(self): """ List the project keys (=project identifiers). :returns: list of project keys identifiers, as strings :rtype: list of strings """ return [x["projectKey"] for x in self._perform_json("GET", "/projects/")] def list_projects(self): """ List the projects :returns: a list of projects, each as a dict. Each dictcontains at least a 'projectKey' field :rtype: list of dicts """ return self._perform_json("GET", "/projects/") def get_project(self, project_key): """ Get a handle to interact with a specific project. :param str project_key: the project key of the desired project :returns: A :class:`dataikuapi.dss.project.DSSProject` to interact with this project """ return DSSProject(self, project_key) def get_default_project(self): """ Get a handle to the current default project, if available (i.e. if dataiku.default_project_key() is valid) """ import dataiku return DSSProject(self, dataiku.default_project_key()) def create_project(self, project_key, name, owner, description=None, settings=None, project_folder_id=None): """ Creates a new project, and return a project handle to interact with it. Note: this call requires an API key with admin rights or the rights to create a project :param str project_key: the identifier to use for the project. Must be globally unique :param str name: the display name for the project. :param str owner: the login of the owner of the project. :param str description: a description for the project. :param dict settings: Initial settings for the project (can be modified later). The exact possible settings are not documented. :param str project_folder_id: the project folder ID in which the project will be created (root project folder if not specified) :returns: A :class:`dataikuapi.dss.project.DSSProject` project handle to interact with this project """ params = {} if project_folder_id is not None: params["projectFolderId"] = project_folder_id resp = self._perform_text( "POST", "/projects/", body={ "projectKey" : project_key, "name" : name, "owner" : owner, "settings" : settings, "description" : description }, params=params) return DSSProject(self, project_key) ######################################################## # Apps ######################################################## def list_apps(self, as_type="listitems"): """ List the apps. :param str as_type: How to return the list. Supported values are "listitems" and "objects" (defaults to **listitems**). :returns: The list of the apps. If "as_type" is "listitems", each one as a :class:`dataikuapi.dss.app.DSSAppListItem`. If "as_type" is "objects", each one as a :class:`dataikuapi.dss.app.DSSApp` :rtype: list """ items = self._perform_json("GET", "/apps/") if as_type == "listitems" or as_type == "listitem": return [DSSAppListItem(self, item) for item in items] elif as_type == "objects" or as_type == "object": return [DSSApp(self, item["appId"]) for item in items] else: raise ValueError("Unknown as_type") def get_app(self, app_id): """ Get a handle to interact with a specific app. .. note:: If a project XXXX is an app template, the identifier of the associated app is PROJECT_XXXX :param str app_id: the id of the desired app :returns: A :class:`dataikuapi.dss.app.DSSApp` to interact with this project """ return DSSApp(self, app_id) ######################################################## # Plugins ######################################################## def list_plugins(self): """ List the installed plugins :returns: list of dict. Each dict contains at least a 'id' field """ return self._perform_json("GET", "/plugins/") def download_plugin_stream(self, plugin_id): """ Download a development plugin, as a binary stream :param str plugin_id: identifier of the plugin to download :param plugin_id: :return: the binary stream """ return self._perform_raw("GET", "/plugins/%s/download" % plugin_id) def download_plugin_to_file(self, plugin_id, path): """ Download a development plugin to a file :param str plugin_id: identifier of the plugin to download :param str path: the path where to download the plugin :return: None """ stream = self.download_plugin_stream(plugin_id) with open(path, 'wb') as f: for chunk in stream.iter_content(chunk_size=10000): if chunk: f.write(chunk) f.flush() def install_plugin_from_archive(self, fp): """ Install a plugin from a plugin archive (as a file object) :param object fp: A file-like object pointing to a plugin archive zip """ files = {'file': fp } self._perform_json("POST", "/plugins/actions/installFromZip", files=files) def start_install_plugin_from_archive(self, fp): """ Install a plugin from a plugin archive (as a file object) Returns immediately with a future representing the process done asycnhronously :param object fp: A file-like object pointing to a plugin archive zip :return: A :class:`~dataikuapi.dss.future.DSSFuture` representing the install process :rtype: :class:`~dataikuapi.dss.future.DSSFuture` """ files = {'file': fp } f = self._perform_json("POST", "/plugins/actions/future/installFromZip", files=files) return DSSFuture.from_resp(self, f) def install_plugin_from_store(self, plugin_id): """ Install a plugin from the Dataiku plugin store :param str plugin_id: identifier of the plugin to install :return: A :class:`~dataikuapi.dss.future.DSSFuture` representing the install process """ f = self._perform_json("POST", "/plugins/actions/installFromStore", body={ "pluginId": plugin_id }) return DSSFuture.from_resp(self, f) def install_plugin_from_git(self, repository_url, checkout = "master", subpath=None): """ Install a plugin from a Git repository. DSS must be setup to allow access to the repository. :param str repository_url: URL of a Git remote :param str checkout: branch/tag/SHA1 to commit. For example "master" :param str subpath: Optional, path within the repository to use as plugin. Should contain a 'plugin.json' file :return: A :class:`~dataikuapi.dss.future.DSSFuture` representing the install process """ f = self._perform_json("POST", "/plugins/actions/installFromGit", body={ "gitRepositoryUrl": repository_url, "gitCheckout" : checkout, "gitSubpath": subpath }) return DSSFuture.from_resp(self, f) def get_plugin(self, plugin_id): """ Get a handle to interact with a specific plugin :param str plugin_id: the identifier of the desired plugin :returns: A :class:`dataikuapi.dss.project.DSSPlugin` """ return DSSPlugin(self, plugin_id) ######################################################## # SQL queries ######################################################## def sql_query(self, query, connection=None, database=None, dataset_full_name=None, pre_queries=None, post_queries=None, type='sql', extra_conf=None, script_steps=None, script_input_schema=None, script_output_schema=None, script_report_location=None, read_timestamp_without_timezone_as_string=True, read_date_as_string=False, project_key=None): """ Initiate a SQL, Hive or Impala query and get a handle to retrieve the results of the query. Internally, the query is run by DSS. The database to run the query on is specified either by passing a connection name, or by passing a database name, or by passing a dataset full name (whose connection is then used to retrieve the database) :param str query: the query to run :param str connection: the connection on which the query should be run (exclusive of database and dataset_full_name) :param str database: the database on which the query should be run (exclusive of connection and dataset_full_name) :param str dataset_full_name: the dataset on the connection of which the query should be run (exclusive of connection and database) :param list pre_queries: (optional) array of queries to run before the query :param list post_queries: (optional) array of queries to run after the query :param str type: the type of query : either 'sql', 'hive' or 'impala' (default: sql) :param str project_key: The project_key on which the query should be run (especially useful for user isolation/impersonation scenario) :return: a handle on the SQL query :rtype: :class:`dataikuapi.dss.sqlquery.DSSSQLQuery` """ if extra_conf is None: extra_conf = {} return DSSSQLQuery(self, query, connection, database, dataset_full_name, pre_queries, post_queries, type, extra_conf, script_steps, script_input_schema, script_output_schema, script_report_location, read_timestamp_without_timezone_as_string, read_date_as_string, project_key) ######################################################## # Users ######################################################## def list_users(self, as_objects=False): """ List all users setup on the DSS instance Note: this call requires an API key with admin rights :return: A list of users, as a list of :class:`dataikuapi.dss.admin.DSSUser` if as_objects is True, else as a list of dicts :rtype: list of :class:`dataikuapi.dss.admin.DSSUser` or list of dicts """ users = self._perform_json("GET", "/admin/users/") if as_objects: return [DSSUser(self, user["login"]) for user in users] else: return users def get_user(self, login): """ Get a handle to interact with a specific user :param str login: the login of the desired user :return: A :class:`dataikuapi.dss.admin.DSSUser` user handle """ return DSSUser(self, login) def create_user(self, login, password, display_name='', source_type='LOCAL', groups=None, profile='DATA_SCIENTIST'): """ Create a user, and return a handle to interact with it Note: this call requires an API key with admin rights :param str login: the login of the new user :param str password: the password of the new user :param str display_name: the displayed name for the new user :param str source_type: the type of new user. Admissible values are 'LOCAL' or 'LDAP' :param list groups: the names of the groups the new user belongs to (defaults to `[]`) :param str profile: The profile for the new user. Typical values (depend on your license): FULL_DESIGNER, DATA_DESIGNER, AI_CONSUMER, ... :return: A :class:`dataikuapi.dss.admin.DSSUser` user handle """ if groups is None: groups = [] resp = self._perform_text( "POST", "/admin/users/", body={ "login" : login, "password" : password, "displayName" : display_name, "sourceType" : source_type, "groups" : groups, "userProfile" : profile }) return DSSUser(self, login) def get_own_user(self): return DSSOwnUser(self) def list_users_activity(self, enabled_users_only=False): """ List all users activity Note: this call requires an API key with admin rights :return: A list of user activity logs, as a list of :class:`dataikuapi.dss.admin.DSSUserActivity` if as_objects is True, else as a list of dict :rtype: list of :class:`dataikuapi.dss.admin.DSSUserActivity` or a list of dict """ params = { "enabledUsersOnly": enabled_users_only } all_activity = self._perform_json("GET", "/admin/users-activity", params=params) return [DSSUserActivity(self, user_activity["login"], user_activity) for user_activity in all_activity] def list_expired_trial_users(self): """ List users whose trials have expired :return: A list of users """ users = self._perform_json("GET", "/admin/list-trial-expired-users") return users def get_authorization_matrix(self): """ Get the authorization matrix for all enabled users and groups Note: this call requires an API key with admin rights :return: The authorization matrix :rtype: A :class:`dataikuapi.dss.admin.DSSAuthorizationMatrix` authorization matrix handle """ resp = self._perform_json("GET", "/admin/authorization-matrix") return DSSAuthorizationMatrix(resp) def start_resync_users_from_supplier(self, logins): """ Starts a resync of multiple users from an external supplier (LDAP, Azure AD or custom auth) :param list logins: list of logins to resync :return: a :class:`dataikuapi.dss.future.DSSFuture` representing the sync process :rtype: :class:`dataikuapi.dss.future.DSSFuture` """ future_resp = self._perform_json("POST", "/admin/users/actions/resync-multi", body=logins) return DSSFuture.from_resp(self, future_resp) def start_resync_all_users_from_supplier(self): """ Starts a resync of all users from an external supplier (LDAP, Azure AD or custom auth) :return: a :class:`dataikuapi.dss.future.DSSFuture` representing the sync process :rtype: :class:`dataikuapi.dss.future.DSSFuture` """ future_resp = self._perform_json("POST", "/admin/users/actions/resync-multi") return DSSFuture.from_resp(self, future_resp) def start_fetch_external_groups(self, user_source_type): """ Fetch groups from external source :param user_source_type: 'LDAP', 'AZURE_AD' or 'CUSTOM' :rtype: :class:`dataikuapi.dss.future.DSSFuture` :return: a DSSFuture containing a list of group names """ future_resp = self._perform_json("GET", "/admin/external-groups", params={'userSourceType': user_source_type}) return DSSFuture.from_resp(self, future_resp) def start_fetch_external_users(self, user_source_type, login=None, group_name=None): """ Fetch users from external source filtered by login or group name: - if login is provided, will search for a user with an exact match in the external source (e.g. before login remapping) - else, - if group_name is provided, will search for members of the group in the external source - else will search for all users :param user_source_type: 'LDAP', 'AZURE_AD' or 'CUSTOM' :param login: optional - the login of the user in the external source :param group_name: optional - the group name of the group in the external source :rtype: :class:`dataikuapi.dss.future.DSSFuture` :return: a DSSFuture containing a list of ExternalUser """ future_resp = self._perform_json("GET", "/admin/external-users", params={'userSourceType': user_source_type, 'login': login, 'groupName': group_name}) return DSSFuture.from_resp(self, future_resp) def start_provision_users(self, user_source_type, users): """ Provision users of given source type :param string user_source_type: 'LDAP', 'AZURE_AD' or 'CUSTOM' :param list users: list of user attributes coming form the external source :rtype: :class:`dataikuapi.dss.future.DSSFuture` """ future_resp = self._perform_json("POST", "/admin/users/actions/provision", body={'userSourceType': user_source_type, 'users': users}) return DSSFuture.from_resp(self, future_resp) ######################################################## # Groups ######################################################## def list_groups(self): """ List all groups setup on the DSS instance Note: this call requires an API key with admin rights :returns: A list of groups, as an list of dicts :rtype: list of dicts """ return self._perform_json( "GET", "/admin/groups/") def get_group(self, name): """ Get a handle to interact with a specific group :param str name: the name of the desired group :returns: A :class:`dataikuapi.dss.admin.DSSGroup` group handle """ return DSSGroup(self, name) def create_group(self, name, description=None, source_type='LOCAL'): """ Create a group, and return a handle to interact with it Note: this call requires an API key with admin rights :param str name: the name of the new group :param str description: (optional) a description of the new group :param source_type: the type of the new group. Admissible values are 'LOCAL' and 'LDAP' :returns: A :class:`dataikuapi.dss.admin.DSSGroup` group handle """ resp = self._perform_text( "POST", "/admin/groups/", body={ "name" : name, "description" : description, "sourceType" : source_type }) return DSSGroup(self, name) ######################################################## # Connections ######################################################## def list_connections(self, as_type="dictitems"): """ List all connections setup on the DSS instance. .. note:: This call requires an API key with admin rights :param string as_type: how to return the connection. Possible values are "dictitems", "listitems" and "objects" :return: if **as_type** is dictitems, a dict of connection name to :class:`dataikuapi.dss.admin.DSSConnectionListItem`. if **as_type** is listitems, a list of :class:`dataikuapi.dss.admin.DSSConnectionListItem`. if **as_type** is objects, a list of :class:`dataikuapi.dss.admin.DSSConnection`. :rtype: dict or list """ items_dict = self._perform_json( "GET", "/admin/connections/") if as_type == "dictitems" or as_type == "dictitem": return {name:DSSConnectionListItem(self, items_dict[name]) for name in items_dict.keys()} if as_type == "listitems" or as_type == "listitem": return [DSSConnectionListItem(self, items_dict[name]) for name in items_dict.keys()] elif as_type == "objects" or as_type == "object": return [DSSConnection(self, name) for name in items_dict.keys()] else: raise ValueError("Unknown as_type") def list_connections_names(self, connection_type): """ List all connections names on the DSS instance. :param str connection_type: Returns only connections with this type. Use 'all' if you don't want to filter. :return: the list of connections names :rtype: List[str] """ return self._perform_json("GET", "/connections/get-names", params={"type": connection_type}) def get_connection(self, name): """ Get a handle to interact with a specific connection :param str name: the name of the desired connection :returns: A :class:`dataikuapi.dss.admin.DSSConnection` connection handle """ return DSSConnection(self, name) def create_connection(self, name, type, params=None, usable_by='ALL', allowed_groups=None): """ Create a connection, and return a handle to interact with it Note: this call requires an API key with admin rights :param name: the name of the new connection :param type: the type of the new connection :param dict params: the parameters of the new connection, as a JSON object (defaults to `{}`) :param usable_by: the type of access control for the connection. Either 'ALL' (=no access control) or 'ALLOWED' (=access restricted to users of a list of groups) :param list allowed_groups: when using access control (that is, setting usable_by='ALLOWED'), the list of names of the groups whose users are allowed to use the new connection (defaults to `[]`) :returns: A :class:`dataikuapi.dss.admin.DSSConnection` connection handle """ if params is None: params = {} if allowed_groups is None: allowed_groups = [] resp = self._perform_text( "POST", "/admin/connections/", body={ "name" : name, "type" : type, "params" : params, "usableBy" : usable_by, "allowedGroups" : allowed_groups }) return DSSConnection(self, name) ######################################################## # Code envs ######################################################## def list_code_envs(self, as_objects=False): """ List all code envs setup on the DSS instance Note: this call requires an API key with admin rights :param boolean as_objects: if True, each returned item will be a :class:`dataikuapi.dss.future.DSSCodeEnv` :returns: a list of code envs. Each code env is a dict containing at least "name", "type" and "language" """ list = self._perform_json( "GET", "/admin/code-envs/") if as_objects: return [DSSCodeEnv(self, e.get("envLang"), e.get("envName")) for e in list] else: return list def get_code_env(self, env_lang, env_name): """ Get a handle to interact with a specific code env :param env_lang: the language (PYTHON or R) of the new code env :param env_name: the name of the new code env :returns: A :class:`dataikuapi.dss.admin.DSSCodeEnv` code env handle """ return DSSCodeEnv(self, env_lang, env_name) def create_code_env(self, env_lang, env_name, deployment_mode, params=None): """ Create a code env, and return a handle to interact with it Note: this call requires an API key with admin rights :param env_lang: the language (PYTHON or R) of the new code env :param env_name: the name of the new code env :param deployment_mode: the type of the new code env :param params: the parameters of the new code env, as a JSON object :returns: A :class:`dataikuapi.dss.admin.DSSCodeEnv` code env handle """ params = params if params is not None else {} params['deploymentMode'] = deployment_mode resp = self._perform_json( "POST", "/admin/code-envs/%s/%s" % (env_lang, env_name), body=params) if resp is None: raise Exception('Env creation returned no data') if resp.get('messages', {}).get('error', False): raise Exception('Env creation failed : %s' % (json.dumps(resp.get('messages', {}).get('messages', {})))) return DSSCodeEnv(self, env_lang, env_name) def list_code_env_usages(self): """ List all usages of a code env in the instance :return: a list of objects where the code env is used """ return self._perform_json("GET", "/admin/code-envs/usages") def create_code_env_from_python_preset(self, preset_name, allow_update=True, interpreter=None, prefix=None): request_params = { 'allowUpdate': allow_update, 'interpreter': interpreter, 'prefix': prefix } return self._perform_json("POST", "/admin/code-envs/from-python-preset/%s" % (preset_name), params=request_params) ######################################################## # Clusters ######################################################## def list_clusters(self): """ List all clusters setup on the DSS instance Returns: List clusters (name, type, state) """ return self._perform_json( "GET", "/admin/clusters/") def get_cluster(self, cluster_id): """ Get a handle to interact with a specific cluster Args: name: the name of the desired cluster Returns: A :class:`dataikuapi.dss.admin.DSSCluster` cluster handle """ return DSSCluster(self, cluster_id) def create_cluster(self, cluster_name, cluster_type='manual', params=None, cluster_architecture='HADOOP'): """ Create a cluster, and return a handle to interact with it :param cluster_name: the name of the new cluster :param cluster_type: the type of the new cluster :param params: the parameters of the new cluster, as a JSON object :param cluster_architecture: the architecture of the new cluster. 'HADOOP' or 'KUBERNETES' :returns: A :class:`dataikuapi.dss.admin.DSSCluster` cluster handle """ definition = {} definition['name'] = cluster_name definition['type'] = cluster_type definition['architecture'] = cluster_architecture definition['params'] = params if params is not None else {} resp = self._perform_json( "POST", "/admin/clusters/", body=definition) if resp is None: raise Exception('Cluster creation returned no data') if resp.get('messages', {}).get('error', False): raise Exception('Cluster creation failed : %s' % (json.dumps(resp.get('messages', {}).get('messages', {})))) return DSSCluster(self, resp['id']) ######################################################## # Code studio templates ######################################################## def list_code_studio_templates(self, as_type='listitems'): """ List all code studio templates on the DSS instance :returns: List of templates (name, type) """ items = self._perform_json("GET", "/admin/code-studios/") if as_type == "listitems" or as_type == "listitem": return [DSSCodeStudioTemplateListItem(self, item) for item in items] elif as_type == "objects" or as_type == "object": return [DSSCodeStudioTemplate(self, item["id"]) for item in items] else: raise ValueError("Unknown as_type") def get_code_studio_template(self, template_id): """ Get a handle to interact with a specific code studio template :param str template_id: the template id of the desired code studio template :returns: A :class:`dataikuapi.dss.admin.DSSCodeStudioTemplate` code studio template handle """ return DSSCodeStudioTemplate(self, template_id) ######################################################## # Global API Keys ######################################################## def list_global_api_keys(self, as_type='listitems'): """ List all global API keys set up on the DSS instance .. note:: This call requires an API key with admin rights .. note:: If the secure API keys feature is enabled, the secret key of the listed API keys will not be present in the returned objects :param str as_type: How to return the global API keys. Possible values are "listitems" and "objects" :return: if as_type=listitems, each key as a :class:`dataikuapi.dss.admin.DSSGlobalApiKeyListItem`. if as_type=objects, each key is returned as a :class:`dataikuapi.dss.admin.DSSGlobalApiKey`. """ resp = self._perform_json( "GET", "/admin/global-api-keys/") if as_type == "listitems": return [DSSGlobalApiKeyListItem(self, item) for item in resp] elif as_type == 'objects': return [DSSGlobalApiKey(self, item["key"], item["id"]) for item in resp] else: raise ValueError("Unknown as_type") def get_global_api_key(self, key): """ Get a handle to interact with a specific Global API key .. deprecated:: 13.0.0 Use :meth:`DSSClient.get_global_api_key_by_id`. Calling this method with an invalid secret key will now result in an immediate error. :param str key: the secret key of the API key :returns: A :class:`dataikuapi.dss.admin.DSSGlobalApiKey` API key handle """ resp = self._perform_json( "GET", "/admin/globalAPIKeys/%s" % key) return DSSGlobalApiKey(self, key, resp['id']) def get_global_api_key_by_id(self, id_): """ Get a handle to interact with a specific Global API key :param str id_: the id the API key :returns: A :class:`dataikuapi.dss.admin.DSSGlobalApiKey` API key handle """ resp = self._perform_json( "GET", "/admin/global-api-keys/%s" % id_) return DSSGlobalApiKey(self, resp["key"], id_) def create_global_api_key(self, label=None, description=None, admin=False): """ Create a Global API key, and return a handle to interact with it .. note:: This call requires an API key with admin rights .. note:: The secret key of the created API key will always be present in the returned object, even if the secure API keys feature is enabled :param str label: the label of the new API key :param str description: the description of the new API key :param boolean admin: has the new API key admin rights (True or False) :returns: A :class:`dataikuapi.dss.admin.DSSGlobalApiKey` API key handle """ resp = self._perform_json( "POST", "/admin/global-api-keys/", body={ "label": label, "description": description, "globalPermissions": { "admin": admin } }) if resp is None: raise Exception('API key creation returned no data') if resp.get('messages', {}).get('error', False): raise Exception('API key creation failed : %s' % (json.dumps(resp.get('messages', {}).get('messages', {})))) if not resp.get('id', False): raise Exception('API key creation returned no key') return DSSGlobalApiKey(self, resp.get('key', ''), resp['id']) ######################################################## # Personal API Keys ######################################################## def list_personal_api_keys(self, as_type='listitems'): """ List all your personal API keys. :param str as_type: How to return the personal API keys. Possible values are "listitems" and "objects" :return: if as_type=listitems, each key as a :class:`dataikuapi.dss.admin.DSSPersonalApiKeyListItem`. if as_type=objects, each key is returned as a :class:`dataikuapi.dss.admin.DSSPersonalApiKey`. """ resp = self._perform_json( "GET", "/personal-api-keys/") if as_type == "listitems": return [DSSPersonalApiKeyListItem(self, item) for item in resp] elif as_type == 'objects': return [DSSPersonalApiKey(self, item.get('key', ''), item['id']) for item in resp] else: raise ValueError("Unknown as_type") def get_personal_api_key(self, id): """ Get a handle to interact with a specific Personal API key. :param str id: the id of the desired API key :returns: A :class:`dataikuapi.dss.admin.DSSPersonalApiKey` API key handle """ return DSSPersonalApiKey(self, "", id) def create_personal_api_key(self, label="", description="", as_type='dict'): """ Create a Personal API key associated with your user. :param str label: the label of the new API key :param str description: the description of the new API key :param str as_type: How to return the personal API keys. Possible values are "dict" and "object" :return: if as_type=dict, the new personal API key is returned as a dict. if as_type=object, the new personal API key is returned as a :class:`dataikuapi.dss.admin.DSSPersonalApiKey`. """ resp = self._perform_json( "POST", "/personal-api-keys/", body={"label": label, "description": description}) if resp is None: raise Exception('API key creation returned no data') if not resp.get('id', False): raise Exception('API key creation returned no key') if as_type == 'object': return DSSPersonalApiKey(self, resp.get('key', ''), resp["id"]) else: return resp def list_all_personal_api_keys(self, as_type='listitems'): """ List all personal API keys. Only admin can list all the keys. :param str as_type: How to return the personal API keys. Possible values are "listitems" and "objects" :return: if as_type=listitems, each key as a :class:`dataikuapi.dss.admin.DSSPersonalApiKeyListItem`. if as_type=objects, each key is returned as a :class:`dataikuapi.dss.admin.DSSPersonalApiKey`. """ resp = self._perform_json( "GET", "/admin/personal-api-keys/") if as_type == "listitems": return [DSSPersonalApiKeyListItem(self, item) for item in resp] elif as_type == 'objects': return [DSSPersonalApiKey(self, item.get('key', ''), item['id']) for item in resp] else: raise ValueError("Unknown as_type") def create_personal_api_key_for_user(self, user, label="", description="", as_type='object'): """ Create a Personal API key associated on behalf of a user. Only admin can create a key for another user. :param str label: the label of the new API key :param str description: the description of the new API key :param str user: the id of the user to impersonate :param str as_type: How to return the personal API keys. Possible values are "dict" and "object" :return: if as_type=dict, the new personal API key is returned as a dict. if as_type=object, the new personal API key is returned as a :class:`dataikuapi.dss.admin.DSSPersonalApiKey`. """ resp = self._perform_json( "POST", "/admin/personal-api-keys/", body={"user": user, "label": label, "description": description}) if resp is None: raise Exception('API key creation returned no data') if not resp.get('id', False): raise Exception('API key creation returned no key') if as_type == 'object': return DSSPersonalApiKey(self, resp.get('key', ''), resp["id"]) else: return resp ######################################################## # Meanings ######################################################## def list_meanings(self): """ List all user-defined meanings on the DSS instance Note: this call requires an API key with admin rights :returns: A list of meanings. Each meaning is a dict :rtype: list of dicts """ return self._perform_json( "GET", "/meanings/") def get_meaning(self, id): """ Get a handle to interact with a specific user-defined meaning Note: this call requires an API key with admin rights :param str id: the ID of the desired meaning :returns: A :class:`dataikuapi.dss.meaning.DSSMeaning` meaning handle """ return DSSMeaning(self, id) def create_meaning(self, id, label, type, description=None, values=None, mappings=None, pattern=None, normalizationMode=None, detectable=False): """ Create a meaning, and return a handle to interact with it Note: this call requires an API key with admin rights :param id: the ID of the new meaning :param type: the type of the new meaning. Admissible values are 'DECLARATIVE', 'VALUES_LIST', 'VALUES_MAPPING' and 'PATTERN' :param description (optional): the description of the new meaning :param values (optional): when type is 'VALUES_LIST', the list of values, or a list of {'value':'the value', 'color':'an optional color'} :param mappings (optional): when type is 'VALUES_MAPPING', the mapping, as a list of objects with this structure: {'from': 'value_1', 'to': 'value_a'} :param pattern (optional): when type is 'PATTERN', the pattern :param normalizationMode (optional): when type is 'VALUES_LIST', 'VALUES_MAPPING' or 'PATTERN', the normalization mode to use for value matching. One of 'EXACT', 'LOWERCASE', or 'NORMALIZED' (not available for 'PATTERN' type). Defaults to 'EXACT'. :param detectable (optional): whether DSS should consider assigning the meaning to columns set to 'Auto-detect'. Defaults to False. :returns: A :class:`dataikuapi.dss.meaning.DSSMeaning` meaning handle """ def make_entry(v): if isinstance(v, dku_basestring_type): return {'value':v} else: return v def make_mapping(v): return {'from':v.get('from', None), 'to':make_entry(v.get('to', None))} entries = None if values is not None: entries = [make_entry(v) for v in values] if mappings is not None: mappings = [make_mapping(v) for v in mappings] resp = self._perform_text( "POST", "/meanings/", body={ "id" : id, "label": label, "type": type, "description": description, "entries": entries, "mappings": mappings, "pattern": pattern, "normalizationMode": normalizationMode, "detectable": detectable }) return DSSMeaning(self, id) ######################################################## # Logs ######################################################## def list_logs(self): """ List all available log files on the DSS instance This call requires an API key with admin rights :returns: A list of log file names """ return self._perform_json( "GET", "/admin/logs/") def get_log(self, name): """ Get the contents of a specific log file This call requires an API key with admin rights :param str name: the name of the desired log file (obtained with :meth:`list_logs`) :returns: The full content of the log file, as a string """ return self._perform_json( "GET", "/admin/logs/%s" % name) def log_custom_audit(self, custom_type, custom_params=None): """ Log a custom entry to the audit trail :param str custom_type: value for customMsgType in audit trail item :param dict custom_params: value for customMsgParams in audit trail item (defaults to `{}`) """ if custom_params is None: custom_params = {} return self._perform_empty("POST", "/admin/audit/custom/%s" % custom_type, body = custom_params) ######################################################## # Monitoring ######################################################## def get_global_usage_summary(self, with_per_project=False): """ Gets a summary of the global usage of this DSS instance (number of projects, datasets, ...) :returns: a summary object """ data = self._perform_json( "GET", "/admin/monitoring/global-usage-summary", params={'withPerProject':with_per_project}) return DSSGlobalUsageSummary(data) ######################################################## # Variables ######################################################## def get_variables(self): """ Deprecated. Use :meth:`get_global_variables` """ warnings.warn("get_variables is deprecated, please use get_global_variables", DeprecationWarning) return self.get_global_variables() def get_global_variables(self): """ Get the DSS instance's variables, as a Python dictionary This call requires an API key with admin rights :returns: A :class:`dataikuapi.dss.admin.DSSInstanceVariables` handle """ variables = self._perform_json("GET", "/admin/variables/") return DSSInstanceVariables(self, variables) def set_variables(self, variables): """ Deprecated. Use :meth:`get_global_variables` and :meth:`dataikuapi.dss.admin.DSSInstanceVariables.save` Updates the DSS instance's variables This call requires an API key with admin rights It is not possible to update a single variable, you must set all of them at once. Thus, you should only use a ``variables`` parameter that has been obtained using :meth:`get_variables`. :param dict variables: the new dictionary of all variables of the instance """ warnings.warn("set_variables is deprecated, please use get_global_variables().save()", DeprecationWarning) return DSSInstanceVariables(self, variables).save() def get_resolved_variables(self, project_key=None, typed=False): """ Get a dictionary of resolved variables of the project. :param str project_key: the project key, defaults to the current project if any :param bool typed: if True, the variable values will be typed in the returned dict, defaults to False :returns: a dictionary with instance and project variables merged """ import dataiku return self._perform_json( "GET", "/projects/%s/variables-resolved" % (dataiku.default_project_key() if project_key is None else project_key), params={ "typed": "true" if typed else "false" }) ######################################################## # General settings ######################################################## def get_general_settings(self): """ Gets a handle to interact with the general settings. This call requires an API key with admin rights :returns: a :class:`dataikuapi.dss.admin.DSSGeneralSettings` handle """ return DSSGeneralSettings(self) ######################################################## # Bundles / Import (Automation node) ######################################################## class PermissionsPropagationPolicy(Enum): NONE = "NONE" READ_ONLY = "READ_ONLY" ALL = "ALL" def create_project_from_bundle_local_archive(self, archive_path, project_folder=None, permissions_propagation_policy=PermissionsPropagationPolicy.NONE): """ Create a project from a bundle archive. Warning: this method can only be used on an automation node. :param string archive_path: Path on the local machine where the archive is :param project_folder: the project folder in which the project will be created or None for root project folder :type project_folder: A :class:`dataikuapi.dss.projectfolder.DSSProjectFolder` :param permissions_propagation_policy: propagate the permissions that were set in the design node to the new project on the automation node (default: False) :type permissions_propagation_policy: A :class:`PermissionsPropagationPolicy` """ if isinstance(permissions_propagation_policy, DSSClient.PermissionsPropagationPolicy): permissions_propagation_policy = permissions_propagation_policy.value params = { "archivePath": osp.abspath(archive_path), "permissionsPropagationPolicy": permissions_propagation_policy, } if project_folder is not None: params["projectFolderId"] = project_folder.project_folder_id return self._perform_json("POST", "/projectsFromBundle/fromArchive", params=params) def create_project_from_bundle_archive(self, fp, project_folder=None): """ Create a project from a bundle archive (as a file object) Warning: this method can only be used on an automation node. :param string fp: A file-like object pointing to a bundle archive zip :param project_folder: the project folder in which the project will be created or None for root project folder :type project_folder: A :class:`dataikuapi.dss.projectfolder.DSSProjectFolder` """ params = {} if project_folder is not None: params['projectFolderId'] = project_folder.project_folder_id files = {'file': fp } return self._perform_json("POST", "/projectsFromBundle/", files=files, params=params) def prepare_project_import(self, f): """ Prepares import of a project archive. Warning: this method can only be used on a design node. :param file-like fp: the input stream, as a file-like object :returns: a :class:`TemporaryImportHandle` to interact with the prepared import """ val = self._perform_json_upload( "POST", "/projects/import/upload", "tmp-import.zip", f) return TemporaryImportHandle(self, val.json()["id"]) ######################################################## # API Deployer ######################################################## def get_apideployer(self): """Gets a handle to work with the API Deployer :rtype: :class:`~dataikuapi.dss.apideployer.DSSAPIDeployer` """ return DSSAPIDeployer(self) ######################################################## # Project Deployer ######################################################## def get_projectdeployer(self): """Gets a handle to work with the Project Deployer :rtype: :class:`~dataikuapi.dss.projectdeployer.DSSProjectDeployer` """ return DSSProjectDeployer(self) ######################################################## # Unified Monitoring ######################################################## def get_unified_monitoring(self): """Gets a handle to work with Unified Monitoring :rtype: :class:`~dataikuapi.dss.unifiedmonitoring.DSSUnifiedMonitoring` """ return DSSUnifiedMonitoring(self) ######################################################## # Data Catalog ######################################################## def catalog_index_connections(self, connection_names=None, all_connections=False, indexing_mode="FULL"): """ Triggers an indexing of multiple connections in the data catalog :param list connection_names: list of connections to index, ignored if `all_connections=True` (defaults to `[]`) :param bool all_connections: index all connections (defaults to `False`) """ if connection_names is None: connection_names = [] return self._perform_json("POST", "/catalog/index", body={ "connectionNames": connection_names, "indexAllConnections": all_connections, "indexingMode": indexing_mode }) ######################################################## # Model export ######################################################## def get_scoring_libs_stream(self): """ Get the scoring libraries jar required for scoring with model jars that don't include libraries. You need to close the stream after download. Failure to do so will result in the DSSClient becoming unusable. :returns: a jar file, as a stream :rtype: file-like """ return self._perform_raw("GET", "/resources/scoring-lib-jar") ######################################################## # Auth ######################################################## def get_auth_info(self, with_secrets=False): """ Returns various information about the user currently authenticated using this instance of the API client. This method returns a dict that may contain the following keys (may also contain others): * authIdentifier: login for a user, id for an API key * groups: list of group names (if context is an user) * secrets: list of dicts containing user secrets (if context is an user) :param: with_secrets boolean: Return user secrets :returns: a dict :rtype: dict """ return self._perform_json("GET", "/auth/info", params={"withSecrets": with_secrets}) def get_auth_info_from_browser_headers(self, headers_dict, with_secrets=False): """ Returns various information about the DSS user authenticated by the dictionary of HTTP headers provided in headers_dict. This is generally only used in webapp backends This method returns a dict that may contain the following keys (may also contain others): * authIdentifier: login for a user, id for an API key * groups: list of group names (if context is an user) * secrets: list of dicts containing user secrets (if context is an user) :param: headers_dict dict: Dictionary of HTTP headers :param: with_secrets boolean: Return user secrets :returns: a dict :rtype: dict """ return self._perform_json("POST", "/auth/info-from-browser-headers", params={"withSecrets": with_secrets}, body=headers_dict) def get_ticket_from_browser_headers(self, headers_dict): """ Returns a ticket for the DSS user authenticated by the dictionary of HTTP headers provided in headers_dict. This is only used in webapp backends This method returns a ticket to use as a X-DKU-APITicket header :param: headers_dict dict: Dictionary of HTTP headers :returns: a string :rtype: string """ return self._perform_json("POST", "/auth/ticket-from-browser-headers", body=headers_dict) ######################################################## # Container execution ######################################################## def push_base_images(self): """ Push base images for Kubernetes container-execution and Spark-on-Kubernetes """ resp = self._perform_json("POST", "/admin/container-exec/actions/push-base-images") return DSSFuture.from_resp(self, resp) def apply_kubernetes_namespaces_policies(self): """ Apply Kubernetes namespaces policies defined in the general settings """ resp = self._perform_json("POST", "/admin/container-exec/actions/apply-kubernetes-policies") return DSSFuture.from_resp(self, resp) def build_cde_plugins_image(self): """ Build and Push the image for containerized dss engine (CDE) with plugins :return: A :class:`~dataikuapi.dss.future.DSSFuture` representing the build process """ resp = self._perform_json("POST", "/admin/container-exec/actions/build-cde-plugins-image") return DSSFuture.from_resp(self, resp) ######################################################## # Global Instance Info ######################################################## def get_instance_info(self): """ Get global information about the DSS instance :returns: a :class:`DSSInstanceInfo` """ resp = self._perform_json("GET", "/instance-info") return DSSInstanceInfo(resp) ######################################################## # Instance Sanity Check ######################################################## def perform_instance_sanity_check(self, exclusion_list=[], wait=True): """ Run an Instance Sanity Check. This call requires an API key with admin rights. :param exclusion_list: a string list of codes to exclude in the sanity check, as returned by :meth:`get_sanity_check_codes` :return: a :class:`dataikuapi.dss.utils.DSSInfoMessages` if `wait` is `True`, or a :class:`dataikuapi.dss.future.DSSFuture` handle otherwise :rtype: :class:`dataikuapi.dss.utils.DSSInfoMessages` or :class:`dataikuapi.dss.future.DSSFuture` """ resp = self._perform_json("POST", "/admin/sanity-check/run", body=exclusion_list) future = DSSFuture.from_resp(self, resp, result_wrapper=DSSInfoMessages) if wait: return future.wait_for_result() else: return future def get_sanity_check_codes(self): """ Return the list of codes that can be generated by the sanity check. This call requires an API key with admin rights. :rtype: list[str] """ return self._perform_json("GET", "/admin/sanity-check/codes") ######################################################## # Licensing ######################################################## def get_licensing_status(self): """ Returns a dictionary with information about licensing status of this DSS instance :rtype: dict """ return self._perform_json("GET", "/admin/licensing/status") def set_license(self, license): """ Sets a new licence for DSS :param license: license (content of license file) :return: None """ self._perform_empty( "POST", "/admin/licensing/license", body=json.loads(license)) ######################################################## # Internal Request handling ######################################################## def _perform_http(self, method, path, params=None, body=None, stream=False, files=None, raw_body=None, headers=None): if body is not None: body = json.dumps(body) if raw_body is not None: body = raw_body try: http_res = self._session.request( method, "%s/dip/publicapi%s" % (self.host, path), params=params, data=body, files=files, stream=stream, headers=headers) http_res.raise_for_status() return http_res except exceptions.HTTPError: try: ex = http_res.json() except ValueError: ex = {"message": http_res.text} raise DataikuException("%s: %s" % (ex.get("errorType", "Unknown error"), ex.get("detailedMessage", ex.get("message", "No message")))) def _perform_empty(self, method, path, params=None, body=None, files = None, raw_body=None): self._perform_http(method, path, params=params, body=body, files=files, stream=False, raw_body=raw_body) def _perform_text(self, method, path, params=None, body=None,files=None, raw_body=None): return self._perform_http(method, path, params=params, body=body, files=files, stream=False, raw_body=raw_body).text def _perform_json(self, method, path, params=None, body=None,files=None, raw_body=None): return self._perform_http(method, path, params=params, body=body, files=files, stream=False, raw_body=raw_body).json() def _perform_raw(self, method, path, params=None, body=None,files=None, raw_body=None): return self._perform_http(method, path, params=params, body=body, files=files, stream=True, raw_body=raw_body) def _perform_json_upload(self, method, path, name, f): try: http_res = self._session.request( method, "%s/dip/publicapi%s" % (self.host, path), files = {'file': (name, f, {'Expires': '0'})} ) http_res.raise_for_status() return http_res except exceptions.HTTPError: ex = http_res.json() raise DataikuException("%s: %s" % (ex.get("errorType", "Unknown error"), ex.get("message", "No message"))) ######################################################## # Discussions ######################################################## def get_object_discussions(self, project_key, object_type, object_id): """ Get a handle to manage discussions on any object :param str project_key: identifier of the project to access :param str object_type: DSS object type :param str object_id: DSS object ID :returns: the handle to manage discussions :rtype: :class:`dataikuapi.discussion.DSSObjectDiscussions` """ return DSSObjectDiscussions(self, project_key, object_type, object_id) ######################################################## # Feature Store ######################################################## def get_feature_store(self): """ Get a handle to interact with the Feature Store. :return: a handle on the feature store :rtype: :class:`dataikuapi.feature_store.DSSFeatureStore` """ return DSSFeatureStore(self) ######################################################## # Workspaces ######################################################## def list_workspaces(self, as_objects=False): """ List the workspaces :returns: The list of workspaces. """ items = self._perform_json("GET", "/workspaces/") if as_objects: return [DSSWorkspace(self, item["workspaceKey"]) for item in items] else: return items def get_workspace(self, workspace_key): """ Get a handle to interact with a specific workspace :param str workspace_key: the workspace key of the desired workspace :returns: A :class:`dataikuapi.dss.workspace.DSSWorkspace` to interact with this workspace """ return DSSWorkspace(self, workspace_key) def create_workspace(self, workspace_key, name, permissions=None, description=None, color=None): """ Create a new workspace and return a workspace handle to interact with it :param str workspace_key: the identifier to use for the workspace. Must be globally unique :param str name: the display name for the workspace. :param [`dataikuapi.dss.workspace.DSSWorkspacePermissionItem`] permissions: Initial permissions for the workspace (can be modified later). :param str description: a description for the workspace. :param str color: The color to use (#RRGGBB format). A random color will be assigned if not specified :returns: A :class:`dataikuapi.dss.workspace.DSSWorkspace` workspace handle to interact with this workspace """ resp = self._perform_text("POST", "/workspaces/", body={ "workspaceKey": workspace_key, "displayName": name, "color": color, "description": description, "permissions": permissions }) return DSSWorkspace(self, workspace_key) ######################################################## # Data Collections ######################################################## def list_data_collections(self, as_type="listitems"): """ List the accessible data collections :param str as_type: How to return the list. Supported values are "listitems", "objects" and "dict" (defaults to **listitems**). :returns: The list of data collections. :rtype: a list of :class:`dataikuapi.dss.data_collection.DSSDataCollectionListItem` if as_type is "listitems", a list of :class:`dataikuapi.dss.data_collection.DSSDataCollection` if as_type is "objects", a list of dict if as_type is "dict" """ items = self._perform_json("GET", "/data-collections/") if as_type == "listitems" or as_type == "listitem": return [DSSDataCollectionListItem(self, item) for item in items] if as_type == "objects" or as_type == "object": return [DSSDataCollection(self, item["id"]) for item in items] else: return items def get_data_collection(self, id): """ Get a handle to interact with a specific data collection :param str id: the id of the data collection to fetch :rtype: :class:`dataikuapi.dss.data_collection.DSSDataCollection` """ return DSSDataCollection(self, id) def create_data_collection(self, displayName, id=None, tags=None, description=None, color=None, permissions=None): """ Create a new data collection and return a handle to interact with it :param str displayName: the display name for the data collection. :param str id: the identifier to use for the data_collection. Must be 8 alphanumerical characters if set, otherwise a random id will be generated. :param tags: The list of tags to use (defaults to *[]*) :type tags: list of str :param str description: a description for the data collection :param str color: The color to use (#RRGGBB format). A random color will be assigned if not specified :param permissions: Initial permissions for the data collection (can be modified later - current user will always be added as admin). :type permissions: a list of :class:`dict` :returns: Handle of the newly created Data Collection :rtype: :class:`dataikuapi.dss.data_collection.DSSDataCollection` """ res = self._perform_json("POST", "/data-collections/", body={ "id": id, "tags": tags if tags is not None else [], "displayName": displayName, "color": color, "description": description, "permissions": permissions }) return DSSDataCollection(self, res['id']) ######################################################## # Integration Channels ######################################################## def _map_channel_to_object_type(self, channel): if 'mail' == channel.get("family", None): return DSSMailMessagingChannel(self, channel) else: return DSSMessagingChannel(self, channel) def get_messaging_channel(self, channel_id): """ Get the messaging channel with the corresponding ID :param channel_id: ID of channel as specified Notifications & Integrations UI :return: A messaging channel object, such as DSSMessagingChannel, or a DSSMailMessagingChannel for mail a channel """ channel = self._perform_json('GET', '/messaging-channels/%s' % channel_id) return self._map_channel_to_object_type(channel) def list_messaging_channels(self, as_type="listitems", channel_type=None, channel_family=None): """ List all available messaging channels :param str as_type: How to return the list. Supported values are "listitems" and "objects" (defaults to **listitems**). :param str channel_type: a channel type to filter by, e.g. "smtp", "aws-ses-mail", "slack" :param str channel_family: a str to filter for family of channels with a similar interface - "mail" for all channels that send email-like messages :return: A list of messaging channels after the filtering specified, as listitems (DSSMessagingChannelListItem) or objects (DSSMessagingChannel / DSSMailMessagingChannel) """ query_string = "" if channel_type is not None: query_string = "?channelType=" + dku_quote_fn(channel_type) if channel_family is not None: query_string += "?" if query_string == "" else "&" query_string += "channelFamily=" + dku_quote_fn(channel_family) channels = self._perform_json('GET', '/messaging-channels/%s' % query_string) if as_type == "listitems" or as_type == "listitem": return [DSSMessagingChannelListItem(self, channel) for channel in channels] if as_type == "objects" or as_type == "object": return [self._map_channel_to_object_type(channel) for channel in channels] else: raise ValueError("Unknown as_type") ######################################################## # Data Quality ######################################################## def get_data_quality_status(self): """ Get the status of data-quality monitored projects, including the count of monitored datasets in Ok/Warning/Error/Empty statuses. :returns: The dict of data quality monitored project statuses. :rtype: dict with PROJECT_KEY as key """ return self._perform_json("GET", "/data-quality/status") ######################################################## # IAM ######################################################## def get_sso_settings(self): """ Get the Single Sign-On (SSO) settings :return: SSO settings :rtype: :class:`dataikuapi.iam.settings.SSOSettings` """ sso = self._perform_json("GET", "/admin/iam/sso-settings") return DSSSSOSettings(self, sso) def get_ldap_settings(self): """ Get the LDAP settings :return: LDAP settings :rtype: :class:`dataikuapi.iam.settings.LDAPSettings` """ ldap = self._perform_json("GET", "/admin/iam/ldap-settings") return DSSLDAPSettings(self, ldap) def get_azure_ad_settings(self): """ Get the Azure Active Directory (aka Microsoft Entra ID) settings :return: Azure AD settings :rtype: :class:`dataikuapi.iam.settings.AzureADSettings` """ ldap = self._perform_json("GET", "/admin/iam/azure-ad-settings") return DSSAzureADSettings(self, ldap) class TemporaryImportHandle(object): def __init__(self, client, import_id): self.client = client self.import_id = import_id def execute(self, settings=None): """ Executes the import with provided settings. :param dict settings: Dict of import settings (defaults to `{}`). The following settings are available: * targetProjectKey (string): Key to import under. Defaults to the original project key * remapping (dict): Dictionary of connection and code env remapping settings. See example of remapping dict: .. code-block:: python "remapping" : { "connections": [ { "source": "src_conn1", "target": "target_conn1" }, { "source": "src_conn2", "target": "target_conn2" } ], "codeEnvs" : [ { "source": "src_codeenv1", "target": "target_codeenv1" }, { "source": "src_codeenv2", "target": "target_codeenv2" } ] } @warning: You must check the 'success' flag """ # Empty JSON dicts can't be parsed properly if settings is None: settings = {} if settings == {}: settings["_"] = "_" return self.client._perform_json("POST", "/projects/import/%s/process" % (self.import_id), body = settings) class DSSInstanceInfo(object): """Global information about the DSS instance""" def __init__(self, data): """Do not call this directly, use :meth:`DSSClient.get_instance_info`""" self._data = data @property def raw(self): """Returns all data as a Python dictionary""" return self._data @property def node_id(self): """Returns the node id (as defined in Cloud Stacks or in install.ini)""" return self._data["nodeId"] @property def node_name(self): """Returns the node name as it appears in the navigation bar""" return self._data["nodeName"] @property def node_type(self): """ Returns the node type :return: One of DESIGN, AUTOMATION or DEPLOYER """ return self._data["nodeType"] dataiku-api-client-python-13.2.2/dataikuapi/fm/000077500000000000000000000000001471211575100213055ustar00rootroot00000000000000dataiku-api-client-python-13.2.2/dataikuapi/fm/__init__.py000066400000000000000000000000001471211575100234040ustar00rootroot00000000000000dataiku-api-client-python-13.2.2/dataikuapi/fm/future.py000066400000000000000000000061461471211575100232000ustar00rootroot00000000000000import sys, time class FMFuture(object): """ A future on the DSS instance """ def __init__( self, client, job_id, state=None, result_wrapper=lambda result: result ): self.client = client self.job_id = job_id self.state = state self.state_is_peek = True self.result_wrapper = result_wrapper @staticmethod def from_resp(client, resp, result_wrapper=lambda result: result): """ Creates a DSSFuture from a parsed JSON response :rtype: :class:`dataikuapi.fm.future.FMFuture` """ return FMFuture( client, resp.get("jobId", None), state=resp, result_wrapper=result_wrapper ) @classmethod def get_result_wait_if_needed(cls, client, ret): if "jobId" in ret: future = FMFuture(client, ret["jobId"], ret) future.wait_for_result() return future.get_result() else: return ret["result"] def abort(self): """ Abort the future """ return self.client._perform_tenant_empty("DELETE", "/futures/%s" % self.job_id) def get_state(self): """ Get the status of the future, and its result if it's ready """ self.state = self.client._perform_tenant_json( "GET", "/futures/%s" % self.job_id, params={"peek": False} ) self.state_is_peek = False return self.state def peek_state(self): """ Get the status of the future, and its result if it's ready """ self.state = self.client._perform_tenant_json( "GET", "/futures/%s" % self.job_id, params={"peek": True} ) self.state_is_peek = True return self.state def get_result(self): """ Get the future result if it's ready, raises an Exception otherwise """ if ( self.state is None or not self.state.get("hasResult", False) or self.state_is_peek ): self.get_state() if self.state.get("hasResult", False): return self.result_wrapper(self.state.get("result", None)) else: raise Exception("Result not ready") def has_result(self): """ Checks whether the future has a result ready """ if self.state is None or not self.state.get("hasResult", False): self.get_state() return self.state.get("hasResult", False) def wait_for_result(self): """ Wait and get the future result """ if self.state.get("hasResult", False): return self.result_wrapper(self.state.get("result", None)) if ( self.state is None or not self.state.get("hasResult", False) or self.state_is_peek ): self.get_state() while not self.state.get("hasResult", False): time.sleep(5) self.get_state() if self.state.get("hasResult", False): return self.result_wrapper(self.state.get("result", None)) else: raise Exception("No result") dataiku-api-client-python-13.2.2/dataikuapi/fm/instances.py000066400000000000000000000446721471211575100236630ustar00rootroot00000000000000from .future import FMFuture from ..dssclient import DSSClient from ..govern_client import GovernClient import sys if sys.version_info > (3, 4): from enum import Enum else: class Enum(object): pass class FMInstanceCreator(object): def __init__( self, client, label, instance_settings_template_id, virtual_network_id, image_id ): """ Helper to create a DSS instance. :param client: The FM client :type client: :class:`dataikuapi.fm.fmclient` :param str label: The label of the instance :param str instance_settings_template: The instance settings template id this instance should be based on :param str virtual_network: The virtual network where the instance should be spawned :param str image_id: The ID of the DSS runtime image (ex: dss-9.0.3-default) """ self.client = client self.data = {} self.data["label"] = label self.data["instanceSettingsTemplateId"] = instance_settings_template_id self.data["virtualNetworkId"] = virtual_network_id self.data["imageId"] = image_id # Set the default value for dssNodeType self.data["dssNodeType"] = "design" def with_dss_node_type(self, dss_node_type): """ Set the DSS node type of the instance to create. The default node type is `DESIGN`. :param dss_node_type: the type of the dss node to create. :type dss_node_type: :class:`dataikuapi.fm.instances.FMNodeType` :rtype: :class:`dataikuapi.fm.instances.FMInstanceCreator` """ # backward compatibility, was a string before . be sure the value falls into the enum value = dss_node_type # python2 does not support enum, keep string if sys.version_info > (3, 4): if isinstance(dss_node_type, str): value = FMNodeType[dss_node_type.upper()] self.data["dssNodeType"] = value.value else: value = FMNodeType.get_from_string(dss_node_type.upper()) self.data["dssNodeType"] = value; return self def with_cloud_instance_type(self, cloud_instance_type): """ Set the machine type for the DSS Instance :param str cloud_instance_type: the machine type to be used for the instance :rtype: :class:`dataikuapi.fm.instances.FMInstanceCreator` """ self.data["cloudInstanceType"] = cloud_instance_type return self def with_data_volume_options( self, data_volume_type=None, data_volume_size=None, data_volume_size_max=None, data_volume_IOPS=None, data_volume_encryption=None, data_volume_encryption_key=None, ): """ Set the options of the data volume to use with the DSS Instance :param str data_volume_type: Optional, data volume type :param int data_volume_size: Optional, data volume initial size :param int data_volume_size_max: Optional, data volume maximum size :param int data_volume_IOPS: Optional, data volume IOPS :param data_volume_encryption: Optional, encryption mode of the data volume :type data_volume_encryption: :class:`dataikuapi.fm.instances.FMInstanceEncryptionMode` :param str data_volume_encryption_key: Optional, the encryption key to use when data_volume_encryption_key is FMInstanceEncryptionMode.CUSTOM :rtype: :class:`dataikuapi.fm.instances.FMInstanceCreator` """ if not data_volume_encryption: data_volume_encryption = FMInstanceEncryptionMode.NONE if type(data_volume_encryption) is not FMInstanceEncryptionMode: raise TypeError( "data_volume encryption needs to be of type FMInstanceEncryptionMode" ) self.data["dataVolumeType"] = data_volume_type self.data["dataVolumeSizeGB"] = data_volume_size self.data["dataVolumeSizeMaxGB"] = data_volume_size_max self.data["dataVolumeIOPS"] = data_volume_IOPS self.data["volumesEncryption"] = data_volume_encryption.value self.data["volumesEncryptionKey"] = data_volume_encryption_key return self def with_cloud_tags(self, cloud_tags): """ Set the tags to be applied to the cloud resources created for this DSS instance :param dict cloud_tags: a key value dictionary of tags to be applied on the cloud resources :rtype: :class:`dataikuapi.fm.instances.FMInstanceCreator` """ self.data["cloudTags"] = cloud_tags return self def with_fm_tags(self, fm_tags): """ A list of tags to add on the DSS Instance in Fleet Manager :param list fm_tags: Optional, list of tags to be applied on the instance in the Fleet Manager :rtype: :class:`dataikuapi.fm.instances.FMInstanceCreator` """ self.data["fmTags"] = fm_tags return self class FMAWSInstanceCreator(FMInstanceCreator): def with_aws_root_volume_options( self, aws_root_volume_size=None, aws_root_volume_type=None, aws_root_volume_IOPS=None, ): """ Set the options of the root volume of the DSS Instance :param int aws_root_volume_size: Optional, the root volume size :param str aws_root_volume_type: Optional, the root volume type :param int aws_root_volume_IOPS: Optional, the root volume IOPS :rtype: :class:`dataikuapi.fm.instances.FMAWSInstanceCreator` """ self.data["awsRootVolumeSizeGB"] = aws_root_volume_size self.data["awsRootVolumeType"] = aws_root_volume_type self.data["awsRootVolumeIOPS"] = aws_root_volume_IOPS return self def create(self): """ Create the DSS instance :return: a newly created DSS instance :rtype: :class:`dataikuapi.fm.instances.FMAWSInstance` """ instance = self.client._perform_tenant_json( "POST", "/instances", body=self.data ) return FMAWSInstance(self.client, instance) class FMAzureInstanceCreator(FMInstanceCreator): def create(self): """ Create the DSS instance :return: a newly created DSS instance :rtype: :class:`dataikuapi.fm.instances.FMAzureInstance` """ instance = self.client._perform_tenant_json( "POST", "/instances", body=self.data ) return FMAzureInstance(self.client, instance) class FMGCPInstanceCreator(FMInstanceCreator): def create(self): """ Create the DSS instance :return: a newly created DSS instance :rtype: :class:`dataikuapi.fm.instances.FMGCPInstance` """ instance = self.client._perform_tenant_json( "POST", "/instances", body=self.data ) return FMGCPInstance(self.client, instance) class FMInstance(object): """ A handle to interact with a DSS instance. Do not create this directly, use :meth:`dataikuapi.fmclient.FMClient.get_instance` or * :meth:`dataikuapi.fmclient.FMClientAWS.new_instance_creator` * :meth:`dataikuapi.fmclient.FMClientAzure.new_instance_creator` * :meth:`dataikuapi.fmclient.FMClientGCP.new_instance_creator` """ def __init__(self, client, instance_data): self.client = client self.instance_data = instance_data self.id = instance_data["id"] def get_client(self): """ Get a Python client to communicate with a DSS instance :return: a Python client to communicate with the target instance :rtype: :class:`dataikuapi.dssclient.DSSClient` """ instance_status = self.get_status() external_url = instance_status.get("publicURL") admin_api_key_resp = self.client._perform_tenant_json( "GET", "/instances/%s/admin-api-key" % self.id ) api_key = admin_api_key_resp["adminAPIKey"] if not external_url: raise ValueError("No external URL available for node %s. This node may not be provisioned yet" % self.id) if self.instance_data.get("dssNodeType") == "govern": return GovernClient(external_url, api_key) else: return DSSClient(external_url, api_key) def reprovision(self): """ Reprovision the physical DSS instance :return: the `Future` object representing the reprovision process :rtype: :class:`dataikuapi.fm.future.FMFuture` """ future = self.client._perform_tenant_json( "GET", "/instances/%s/actions/reprovision" % self.id ) return FMFuture.from_resp(self.client, future) def deprovision(self): """ Deprovision the physical DSS instance :return: the `Future` object representing the deprovision process :rtype: :class:`dataikuapi.fm.future.FMFuture` """ future = self.client._perform_tenant_json( "GET", "/instances/%s/actions/deprovision" % self.id ) return FMFuture.from_resp(self.client, future) def restart_dss(self): """ Restart the DSS running on the physical instance :return: the `Future` object representing the restart process :rtype: :class:`dataikuapi.fm.future.FMFuture` """ future = self.client._perform_tenant_json( "GET", "/instances/%s/actions/restart-dss" % self.id ) return FMFuture.from_resp(self.client, future) def save(self): """ Update the instance """ self.client._perform_tenant_empty( "PUT", "/instances/%s" % self.id, body=self.instance_data ) self.instance_data = self.client._perform_tenant_json( "GET", "/instances/%s" % self.id ) def get_status(self): """ Get the physical DSS instance's status :return: the instance status :rtype: :class:`dataikuapi.fm.instances.FMInstanceStatus` """ status = self.client._perform_tenant_json( "GET", "/instances/%s/status" % self.id ) return FMInstanceStatus(status) def delete(self): """ Delete the DSS instance :return: the `Future` object representing the deletion process :rtype: :class:`dataikuapi.fm.future.FMFuture` """ future = self.client._perform_tenant_json( "GET", "/instances/%s/actions/delete" % self.id ) return FMFuture.from_resp(self.client, future) def start(self): """ Start the DSS instance :return: the `Future` object representing the starting process :rtype: :class:`dataikuapi.fm.future.FMFuture` """ future = self.client._perform_tenant_json( "GET", "/instances/%s/actions/start" % self.id ) return FMFuture.from_resp(self.client, future) def stop(self): """ Stop the DSS instance :return: the `Future` object representing the stopping process :rtype: :class:`dataikuapi.fm.future.FMFuture` """ future = self.client._perform_tenant_json( "GET", "/instances/%s/actions/stop" % self.id ) return FMFuture.from_resp(self.client, future) def get_initial_password(self): """ Get the initial DSS admin password. Can only be called once :return: a password for the 'admin' user. """ return self.client._perform_tenant_json( "GET", "/instances/%s/actions/get-initial-password" % self.id ) def reset_user_password(self, username, password): """ Reset the password for a user on the DSS instance :param string username: login :param string password: new password :return: the `Future` object representing the password reset process :rtype: :class:`dataikuapi.fm.future.FMFuture` """ future = self.client._perform_tenant_json( "GET", "/instances/%s/actions/reset-user-password" % self.id, params={ 'userName':username, 'password':password } ) return FMFuture.from_resp(self.client, future) def replay_setup_actions(self): """ Replay the setup actions on the DSS instance :return: the `Future` object representing the replay process :rtype: :class:`dataikuapi.fm.future.FMFuture` """ future = self.client._perform_tenant_json( "GET", "/instances/%s/actions/replay-setup-actions" % self.id ) return FMFuture.from_resp(self.client, future) def set_automated_snapshots(self, enable, period, keep=0): """ Set the automated snapshot policy for this instance :param boolean enable: Enable the automated snapshots :param int period: The time period between 2 snapshot in hours :param int keep: Optional, the number of snapshot to keep. Use 0 to keep all snapshots. Defaults to 0. """ self.instance_data["enableAutomatedSnapshot"] = enable self.instance_data["automatedSnapshotPeriod"] = period self.instance_data["automatedSnapshotRetention"] = keep return self def set_custom_certificate(self, pem_data): """ Set the custom certificate for this instance Only needed when Virtual Network HTTPS Strategy is set to Custom Certificate :param str pem_data: The SSL certificate """ self.instance_data["sslCertificatePEM"] = pem_data return self ######################################################## # Snapshots ######################################################## def list_snapshots(self): """ List all the snapshots of this instance :return: list of snapshots :rtype: list of :class:`dataikuapi.fm.instances.FMSnapshot` """ snapshots = self.client._perform_tenant_json("GET", "/instances/%s/snapshots" % self.id) return [FMSnapshot(self.client, self.id, x["id"], x) for x in snapshots] def get_snapshot(self, snapshot_id): """ Get a snapshot of this instance :param str snapshot_id: identifier of the snapshot :return: Snapshot :rtype: :class:`dataikuapi.fm.instances.FMSnapshot` """ return FMSnapshot(self.client, self.id, snapshot_id) def snapshot(self, reason_for_snapshot=None): """ Create a snapshot of the instance :return: Snapshot :rtype: :class:`dataikuapi.fm.instances.FMSnapshot` """ snapshot = self.client._perform_tenant_json( "POST", "/instances/%s/snapshots" % self.id, params={ "reasonForSnapshot":reason_for_snapshot } ) return FMSnapshot(self.client, self.id, snapshot["id"], snapshot) class FMAWSInstance(FMInstance): def set_elastic_ip(self, enable, elasticip_allocation_id): """ Set a public elastic ip for this instance :param boolan enable: Enable the elastic ip allocation :param str elasticip_allocation_id: AWS ElasticIP allocation ID """ self.instance_data["awsAssignElasticIP"] = enable self.instance_data["awsElasticIPAllocationId"] = elasticip_allocation_id return self class FMAzureInstance(FMInstance): def set_elastic_ip(self, enable, public_ip_id): """ Set a public elastic ip for this instance :param boolan enable: Enable the elastic ip allocation :param str public_ip_id: Azure Public IP ID """ self.instance_data["azureAssignElasticIP"] = enable self.instance_data["azurePublicIPId"] = public_ip_id return self class FMGCPInstance(FMInstance): def set_public_ip(self, enable, public_ip_id): """ Set a public ip for this instance :param boolan enable: Enable the public ip allocation :param str public_ip_id: GCP Public IP ID """ self.instance_data["gcpAssignPublicIP"] = enable self.instance_data["gcpPublicIPId"] = public_ip_id return self class FMNodeType(Enum): DESIGN = "design" DEPLOYER = "deployer" AUTOMATION = "automation" GOVERN = "govern" # Python2 emulated enum. to be removed on Python2 support removal @staticmethod def get_from_string(s): if s == "DESIGN": return FMNodeType.DESIGN if s == "DEPLOYER": return FMNodeType.DEPLOYER if s == "AUTOMATION": return FMNodeType.AUTOMATION if s == "GOVERN": return FMNodeType.GOVERN raise Exception("Invalid Node Type " + s) class FMInstanceEncryptionMode(Enum): NONE = "NONE" DEFAULT_KEY = "DEFAULT_KEY" TENANT = "TENANT" CUSTOM = "CUSTOM" class FMInstanceStatus(dict): """A class holding read-only information about an Instance. This class should not be created directly. Instead, use :meth:`FMInstance.get_status` """ def __init__(self, data): """Do not call this directly, use :meth:`FMInstance.get_status`""" super(FMInstanceStatus, self).__init__(data) class FMSnapshot(object): """ A handle to interact with a snapshot of a DSS instance. Do not create this directly, use :meth:`FMInstance.snapshot` """ def __init__(self, client, instance_id, snapshot_id, snapshot_data=None): self.client = client self.instance_id = instance_id self.snapshot_id = snapshot_id self.snapshot_data = snapshot_data def get_info(self): """ Get the information about this snapshot :return: a dict """ if self.snapshot_data is None: self.snapshot_data = self.client._perform_tenant_json( "GET", "/instances/%s/snapshots/%s" % (self.instance_id, self.snapshot_id) ) return self.snapshot_data def reprovision(self): """ Reprovision the physical DSS instance from this snapshot :return: the `Future` object representing the reprovision process :rtype: :class:`dataikuapi.fm.future.FMFuture` """ future = self.client._perform_tenant_json( "POST", "/instances/%s/snapshots/%s/reprovision" % (self.instance_id, self.snapshot_id) ) return FMFuture.from_resp(self.client, future) def delete(self): """ Delete the snapshot :return: the `Future` object representing the deletion process :rtype: :class:`dataikuapi.fm.future.FMFuture` """ future = self.client._perform_tenant_json( "DELETE", "/instances/%s/snapshots/%s" % (self.instance_id, self.snapshot_id) ) return FMFuture.from_resp(self.client, future) dataiku-api-client-python-13.2.2/dataikuapi/fm/instancesettingstemplates.py000066400000000000000000000452031471211575100271670ustar00rootroot00000000000000import json from .future import FMFuture import sys if sys.version_info > (3, 4): from enum import Enum else: class Enum(object): pass class FMInstanceSettingsTemplateCreator(object): def __init__(self, client, label): """ A builder class to create an Instance Settings Template :param str label: The label of the Virtual Network """ self.data = {} self.data["label"] = label self.client = client def create(self): """ Create a new instance settings template. :return: a newly created template :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMInstanceSettingsTemplate` """ template = self.client._perform_tenant_json( "POST", "/instance-settings-templates", body=self.data ) return FMInstanceSettingsTemplate(self.client, template) def with_setup_actions(self, setup_actions): """ Add setup actions :param list setup_actions: List of setup actions to be played on an instance :type setup_actions: list of :class:`dataikuapi.fm.instancesettingstemplates.FMSetupActions` :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMInstanceSettingsTemplateCreator` """ self.data["setupActions"] = setup_actions return self def with_license(self, license_file_path=None, license_string=None): """ Override global license :param str license_file_path: Optional, load the license from a json file :param str license_string: Optional, load the license from a json string :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMInstanceSettingsTemplateCreator` """ if license_file_path is not None: with open(license_file_path) as json_file: license = json.load(json_file) elif license_string is not None: license = json.loads(license_string) else: raise ValueError( "a valid license_file_path or license_string needs to be provided" ) self.data["license"] = json.dumps(license, indent=2) return self class FMAWSInstanceSettingsTemplateCreator(FMInstanceSettingsTemplateCreator): def with_aws_keypair(self, aws_keypair_name): """ Add an AWS Keypair to the DSS instance. Needed to get SSH access to the DSS instance, using the centos user. :param str aws_keypair_name: Name of an AWS key pair to add to the instance. :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMAWSInstanceSettingsTemplateCreator` """ self.data["awsKeyPairName"] = aws_keypair_name return self def with_startup_instance_profile(self, startup_instance_profile_arn): """ Add an Instance Profile to be assigned to the DSS instance on startup :param str startup_instance_profile_arn: ARN of the Instance profile assigned to the DSS instance at startup time :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMAWSInstanceSettingsTemplateCreator` """ self.data["startupInstanceProfileArn"] = startup_instance_profile_arn return self def with_runtime_instance_profile(self, runtime_instance_profile_arn): """ Add an Instance Profile to be assigned to the DSS instance when running :param str runtime_instance_profile_arn: ARN of the Instance profile assigned to the DSS instance during runtime :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMAWSInstanceSettingsTemplateCreator` """ self.data["runtimeInstanceProfileArn"] = runtime_instance_profile_arn return self def with_restrict_aws_metadata_server_access( self, restrict_aws_metadata_server_access=True ): """ Restrict AWS metadata server access on the DSS instance. :param boolean restrict_aws_metadata_server_access: Optional, If true, restrict the access to the metadata server access. Defaults to true :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMAWSInstanceSettingsTemplateCreator` """ self.data[ "restrictAwsMetadataServerAccess" ] = restrict_aws_metadata_server_access return self def with_default_aws_api_access_mode(self): """ The DSS Instance will use the Runtime Instance Profile to access AWS API. :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMAWSInstanceSettingsTemplateCreator` """ self.data["dataikuAwsAPIAccessMode"] = "NONE" return self def with_keypair_aws_api_access_mode( self, aws_access_key_id, aws_keypair_storage_mode="NONE", aws_secret_access_key=None, aws_secret_access_key_aws_secret_name=None, aws_secrets_manager_region=None, ): """ DSS Instance will use an Access Key to authenticate against the AWS API. :param str aws_access_key_id: AWS Access Key ID. :param str aws_keypair_storage_mode: Optional, the storage mode of the AWS api key. Accepts "NONE", "INLINE_ENCRYPTED" or "AWS_SECRETS_MANAGER". Defaults to "NONE" :param str aws_secret_access_key: Optional, AWS Access Key Secret. Only needed if keypair_storage_mode is "INLINE_ENCRYPTED" :param str aws_secret_access_key_aws_secret_name: Optional, ASM secret name. Only needed if aws_keypair_storage_mode is "AWS_SECRET_MANAGER" :param str aws_secrets_manager_region: Optional, Secret Manager region to use. Only needed if aws_keypair_storage_mode is "AWS_SECRET_MANAGER" :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMAWSInstanceSettingsTemplateCreator` """ if aws_keypair_storage_mode not in [ "NONE", "INLINE_ENCRYPTED", "AWS_SECRETS_MANAGER", ]: raise ValueError( 'aws_keypair_storage_mode should be either "NONE", "INLINE_ENCRYPTED" or "AWS_SECRET_MANAGER"' ) self.data["dataikuAwsAPIAccessMode"] = "KEYPAIR" self.data["dataikuAwsKeypairStorageMode"] = aws_keypair_storage_mode if aws_keypair_storage_mode == "NONE": return self self.data["dataikuAwsAccessKeyId"] = aws_access_key_id if aws_keypair_storage_mode == "INLINE_ENCRYPTED": if aws_secret_access_key == None: raise ValueError( 'When aws_keypair_storage_mode is "INLINE_ENCRYPTED", aws_secret_access_key should be provided' ) self.data["dataikuAwsSecretAccessKey"] = aws_secret_access_key elif aws_keypair_storage_mode == "AWS_SECRETS_MANAGER": if aws_secret_access_key_aws_secret_name == None: raise ValueError( 'When aws_keypair_storage_mode is "AWS_SECRETS_MANAGER", aws_secret_access_key_aws_secret_name should be provided' ) self.data[ "dataikuAwsSecretAccessKeyAwsSecretName" ] = aws_secret_access_key_aws_secret_name self.data["awsSecretsManagerRegion"] = aws_secrets_manager_region return self class FMAzureInstanceSettingsTemplateCreator(FMInstanceSettingsTemplateCreator): def with_ssh_key(self, ssh_public_key): """ Add an SSH public key to the DSS Instance. Needed to access it through SSH, using the centos user. :param str ssh_public_key: The content of the public key to add to the instance. :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMAzureInstanceSettingsTemplateCreator` """ self.data["azureSshKey"] = ssh_public_key return self def with_startup_managed_identity(self, startup_managed_identity): """ Add a managed identity to be assign to the DSS instance on startup :param str startup_managed_identity: Managed Identity ID :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMAzureInstanceSettingsTemplateCreator` """ self.data["startupManagedIdentity"] = startup_managed_identity return self def with_runtime_managed_identity(self, runtime_managed_identity): """ Add a managed identity to be assign to the DSS instance when running :param str runtime_managed_identity: Managed Identity ID :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMAzureInstanceSettingsTemplateCreator` """ self.data["runtimeManagedIdentity"] = runtime_managed_identity return self class FMGCPInstanceSettingsTemplateCreator(FMInstanceSettingsTemplateCreator): def with_ssh_key(self, ssh_public_key): """ Add an SSH public key to the DSS Instance. Needed to access it through SSH, using the centos user. :param str ssh_public_key: The content of the public key to add to the instance. :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMGCPInstanceSettingsTemplateCreator` """ self.data["gcpSshKey"] = ssh_public_key return self def with_restrict_metadata_server_access(self, restrict_metadata_server_access=True): """ Restrict GCloud metadata server access on the DSS instance. :param boolean restrict_metadata_server_access: Optional, If true, restrict the access to the metadata server access. Defaults to true :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMGCPInstanceSettingsTemplateCreator` """ self.data["restrictGcpMetadataServerAccess"] = restrict_metadata_server_access return self def with_block_project_wide_keys(self, block_project_wide_keys=True): """ Restrict GCloud metadata server access on the DSS instance. :param boolean block_project_wide_keys: Optional, If true, block project-wide ssh keys on the instance. Defaults to true :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMGCPInstanceSettingsTemplateCreator` """ self.data["gcpBlockProjectWideKeys"] = block_project_wide_keys return self def with_runtime_service_account(self, startup_service_account): """ Add a service account to be assigned to the DSS instance on startup :param str startup_service_account: service account email :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMGCPInstanceSettingsTemplateCreator` """ self.data["startupServiceAccount"] = startup_service_account return self class FMInstanceSettingsTemplate(object): def __init__(self, client, ist_data): self.client = client self.id = ist_data["id"] self.ist_data = ist_data def save(self): """ Update this template """ self.client._perform_tenant_empty( "PUT", "/instance-settings-templates/%s" % self.id, body=self.ist_data ) self.ist_data = self.client._perform_tenant_json( "GET", "/instance-settings-templates/%s" % self.id ) def delete(self): """ Delete this template :return: the `Future` object representing the deletion process :rtype: :class:`dataikuapi.fm.future.FMFuture` """ future = self.client._perform_tenant_json( "DELETE", "/instance-settings-templates/%s" % self.id ) return FMFuture.from_resp(self.client, future) def add_setup_action(self, setup_action): """ Add a setup action :param setup_action: the action to add :type setup_action: :class:`dataikuapi.fm.instancesettingstemplates.FMSetupAction` :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMInstanceSettingsTemplate` """ self.ist_data["setupActions"].append(setup_action) return self class FMSetupAction(dict): def __init__(self, setupActionType, params=None): """ A class representing a setup action Do not create this directly, use the static methods in this class, for example: - :meth:`dataikuapi.fm.instancesettingstemplates.FMSetupAction.add_authorized_key` """ data = { "type": setupActionType.value, } if params: data["params"] = params else: data["params"] = {} super(FMSetupAction, self).__init__(data) @staticmethod def add_authorized_key(ssh_key): """ Return an ADD_AUTHORIZED_KEY setup action :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMSetupAction` """ return FMSetupAction(FMSetupActionType.ADD_AUTHORIZED_KEY, {"sshKey": ssh_key}) @staticmethod def run_ansible_task(stage, yaml_string): """ Return a RUN_ANSIBLE_TASK setup action :param stage: the action stage :type stage: :class:`dataikuapi.fm.instancesettingstemplates.FMSetupActionStage` :param str yaml_string: a yaml encoded string defining the ansibles tasks to run :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMSetupAction` """ return FMSetupAction( FMSetupActionType.RUN_ANSIBLE_TASKS, {"stage": stage.value, "ansibleTasks": yaml_string}, ) @staticmethod def install_system_packages(packages): """ Return an INSTALL_SYSTEM_PACKAGES setup action :param list packages: List of packages to install :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMSetupAction` """ return FMSetupAction( FMSetupActionType.INSTALL_SYSTEM_PACKAGES, {"packages": packages} ) @staticmethod def setup_advanced_security(basic_headers=True, hsts=False): """ Return a SETUP_ADVANCED_SECURITY setup action :param boolean basic_headers: Optional, Prevent browsers to render Web content served by DSS to be embedded into a frame, iframe, embed or object tag. Defaults to True :param boolean hsts: Optional, Enforce HTTP Strict Transport Security. Defaults to False :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMSetupAction` """ return FMSetupAction( FMSetupActionType.SETUP_ADVANCED_SECURITY, {"basic_headers": basic_headers, "hsts": hsts}, ) @staticmethod def install_jdbc_driver( database_type, url, paths_in_archive=None, http_headers=None, http_username=None, http_password=None, datadir_subdirectory=None, ): """ Return a INSTALL_JDBC_DRIVER setup action :param database_type: the database type :type database_type: :class:`dataikuapi.fm.instancesettingstemplates.FMSetupActionAddJDBCDriverDatabaseType` :param str url: The full address to the driver. Supports http(s)://, s3://, abs:// or file:// endpoints :param list paths_in_archive: Optional, must be used when the driver is shipped as a tarball or a ZIP file. Add here all the paths to find the JAR files in the driver archive. Paths are relative to the top of the archive. Wildcards are supported. :param dict http_headers: Optional, If you download the driver from a HTTP(S) endpoint, add here the headers you want to add to the query. This setting is ignored for any other type of download. :param str http_username: Optional, If the HTTP(S) endpoint expect a Basic Authentication, add here the username. To explicitely specify which Assigned Identity use if the machine have several, set the client_id here. To authenticate with a SAS Token on Azure Blob Storage (not recommended), use "token" as the value here. :param str http_password: Optional, If the HTTP(S) endpoint expect a Basic Authentication, add here the password. To authenticate with a SAS Token on Azure Blob Storage (not recommended), store the token in this field. :param str datadir_subdirectory: Optional, Some drivers are shipped with a high number of JAR files along with them. In that case, you might want to install them under an additional level in the DSS data directory. Set the name of this subdirectory here. Not required for most drivers. :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMSetupAction` """ if paths_in_archive: if not isinstance(paths_in_archive, list) and not isinstance(paths_in_archive, tuple): raise ValueError( "paths_in_archive is expected to be a list" ) if http_headers: if not isinstance(http_headers, dict): raise ValueError( "http_headers is expected to be a dict" ) return FMSetupAction( FMSetupActionType.INSTALL_JDBC_DRIVER, { "url": url, "dbType": database_type.value, "pathsInArchive": paths_in_archive, "headers": http_headers, "username": http_username, "password": http_password, "subpathInDatadir": datadir_subdirectory, }, ) @staticmethod def setup_k8s_and_spark(): """ Return a SETUP_K8S_AND_SPARK setup action :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMSetupAction` """ return FMSetupAction(FMSetupActionType.SETUP_K8S_AND_SPARK) class FMSetupActionType(Enum): RUN_ANSIBLE_TASKS = "RUN_ANSIBLE_TASKS" INSTALL_SYSTEM_PACKAGES = "INSTALL_SYSTEM_PACKAGES" INSTALL_DSS_PLUGINS_FROM_STORE = "INSTALL_DSS_PLUGINS_FROM_STORE" SETUP_K8S_AND_SPARK = "SETUP_K8S_AND_SPARK" SETUP_RUNTIME_DATABASE = "SETUP_RUNTIME_DATABASE" SETUP_MUS_AUTOCREATE = "SETUP_MUS_AUTOCREATE" SETUP_UI_CUSTOMIZATION = "SETUP_UI_CUSTOMIZATION" SETUP_MEMORY_SETTINGS = "SETUP_MEMORY_SETTINGS" SETUP_GRAPHICS_EXPORT = "SETUP_GRAPHICS_EXPORT" ADD_AUTHORIZED_KEY = "ADD_AUTHORIZED_KEY" INSTALL_JDBC_DRIVER = "INSTALL_JDBC_DRIVER" SETUP_ADVANCED_SECURITY = "SETUP_ADVANCED_SECURITY" SETUP_INSTANCE_NAME = "SETUP_INSTANCE_NAME" SETUP_NONE_K8S_AND_SPARK = "SETUP_NONE_K8S_AND_SPARK" SETUP_SECURE_COOKIES = "SETUP_SECURE_COOKIES" ADD_ENV_VARIABLES = "ADD_ENV_VARIABLES" ADD_PROPERTIES = "ADD_PROPERTIES" ADD_SSH_KEY = "ADD_SSH_KEY" ADD_CA_TO_TRUSTSTORE = "ADD_CA_TO_TRUSTSTORE" SETUP_PROXY = "SETUP_PROXY" INSTALL_CODE_ENV_WITH_PRESET = "INSTALL_CODE_ENV_WITH_PRESET" UPDATE_DSS_PORT = "UPDATE_DSS_PORT" class FMSetupActionStage(Enum): after_dss_startup = "after_dss_startup" after_install = "after_install" before_install = "before_install" class FMSetupActionAddJDBCDriverDatabaseType(Enum): mysql = "mysql" mssql = "mssql" oracle = "oracle" mariadb = "mariadb" snowflake = "snowflake" athena = "athena" bigquery = "bigquery" dataiku-api-client-python-13.2.2/dataikuapi/fm/tenant.py000066400000000000000000000153271471211575100231600ustar00rootroot00000000000000import json class FMCloudCredentials(object): """ A Tenant Cloud Credentials in the FM instance """ def __init__(self, client, cloud_credentials): self.client = client self.cloud_credentials = cloud_credentials def set_cmk_key(self, cmk_key_id): self.cloud_credentials["awsCMKId"] = cmk_key_id return self def set_static_license(self, license_file_path=None, license_string=None): """ Set a default static license for the DSS instances :param str license_file_path: Optional, load the license from a json file :param str license_string: Optional, load the license from a json string """ if license_file_path is not None: with open(license_file_path) as json_file: license = json.load(json_file) elif license_string is not None: license = json.loads(license_string) else: raise ValueError( "a valid license_file_path or license_string needs to be provided" ) self.cloud_credentials["licenseMode"] = "STATIC" self.cloud_credentials["license"] = json.dumps(license, indent=2) return self def set_azure_keyvault(self, azure_keyvault_id, azure_key_name, azure_key_version): """ Set the Azure Key Vault configuration :param str azure_keyvault_id: the Azure Key Vault resource ID :param str azure_key_name: the name of the key to use in the vault :param str azure_key_version: the version of the key to use in the vault """ self.cloud_credentials["azureKeyVaultId"] = azure_keyvault_id self.cloud_credentials["azureKeyName"] = azure_key_name self.cloud_credentials["azureKeyVersion"] = azure_key_version return self def set_aws_cmk(self, aws_cmk_id): """ Set AWS Customer Managed Key (CMK) configuration :param str aws_cmk_id: the ID of the key to use """ self.cloud_credentials["awsCMKId"] = aws_cmk_id return self def set_gcp_key(self, gcp_location_id, gcp_key_ring, gcp_crypto_key, gcp_crypto_key_version): """ Set GCP Key Management Service (KMS) configuration :param str gcp_location_id: the location ID of the key ring :param str gcp_key_ring: the name of the key ring :param str gcp_crypto_key: the name of the key to use in the key ring :param str gcp_crypto_key_version: the version of the key to use in the key ring """ self.cloud_credentials["gcpLocationId"] = gcp_location_id self.cloud_credentials["gcpKeyRing"] = gcp_key_ring self.cloud_credentials["gcpCryptoKey"] = gcp_crypto_key self.cloud_credentials["gcpCryptoKeyVersion"] = gcp_crypto_key_version return self def set_automatically_updated_license(self, license_token): """ Set an automatically updated license for the DSS instances :param str license_token: License token """ if license_token is None: raise ValueError("a valid license_token needs to be provided") self.cloud_credentials["licenseMode"] = "AUTO_UPDATE" self.cloud_credentials["licenseToken"] = license_token return self def set_authentication(self, authentication): """ Set the authentication for the tenant :param authentication: the authentication to be used :type authentication: :class:`dataikuapi.fm.tenant.FMCloudAuthentication` """ self.cloud_credentials.update(authentication) return self def save(self): """ Saves back the settings to the project """ self.client._perform_tenant_empty( "PUT", "/cloud-credentials", body=self.cloud_credentials ) class FMCloudTags(object): """ A Tenant Cloud Tags in the FM instance """ def __init__(self, client, cloud_tags): self.client = client self.cloud_tags = json.loads(cloud_tags["msg"]) @property def tags(self): return self.cloud_tags def save(self): """ Saves the tags on FM """ self.client._perform_tenant_empty("PUT", "/cloud-tags", body=self.cloud_tags) class FMCloudAuthentication(dict): def __init__(self, data): """ A class holding the Cloud Authentication information Do not create this directly, use: - :meth:`dataikuapi.fm.tenant.FMCloudAuthentication.aws_same_as_fm` to use the same authentication as Fleet Manager - :meth:`dataikuapi.fm.tenant.FMCloudAuthentication.aws_iam_role` to use a custom IAM Role - :meth:`dataikuapi.fm.tenant.FMCloudAuthentication.aws_keypair` to use a AWS Access key ID and AWS Secret Access Key pair """ super(FMCloudAuthentication, self).__init__(data) @staticmethod def aws_same_as_fm(): """ AWS Only: use the same authentication as Fleet Manager """ return FMCloudAuthentication({"awsAuthenticationMode": "SAME_AS_FM"}) @staticmethod def aws_iam_role(role_arn): """ AWS Only: use an IAM Role :param str role_arn: ARN of the IAM Role """ return FMCloudAuthentication( {"awsAuthenticationMode": "IAM_ROLE", "awsIAMRoleARN": role_arn} ) @staticmethod def aws_keypair(access_key_id, secret_access_key): """ AWS Only: use an AWS Access Key :param str access_key_id: AWS Access Key ID :param str secret_access_key: AWS Secret Access Key """ return FMCloudAuthentication( { "awsAuthenticationMode": "KEYPAIR", "awsAccessKeyId": access_key_id, "awsSecretAccessKey": secret_access_key, } ) @staticmethod def azure(subscription, tenant_id, environment, client_id): """ Azure Only :param str subscription: Azure Subscription :param str tenant_id: Azure Tenant Id :param str environment: Azure Environment :param str client_id: Azure Client Id """ data = { "azureSubscription": subscription, "azureTenantId": tenant_id, "azureEnvironment": environment, "azureFMAppClientId": client_id, } return FMCloudAuthentication(data) @staticmethod def gcp(project_id, service_account_key): """ GCP Only :param str project_id: GCP project :param str service_account_key: Optional, service account key (JSON) """ data = { "gcpProjectId": project_id, "gcpServiceAccountKey": service_account_key } return FMCloudAuthentication(data) dataiku-api-client-python-13.2.2/dataikuapi/fm/virtualnetworks.py000066400000000000000000000324541471211575100251520ustar00rootroot00000000000000from .future import FMFuture class FMVirtualNetworkCreator(object): def __init__(self, client, label): """ A builder class to create virtual networks :param str label: The label of the virtual network """ self.client = client self.data = {} self.use_default_values = False self.data["label"] = label self.data["mode"] = "EXISTING_MONOTENANT" def with_internet_access_mode(self, internet_access_mode): """ Set the Internet access mode :param str internet_access_mode: The internet access mode of the instances created in this virtual network. Accepts "YES", "NO", "EGRESS_ONLY". Defaults to "YES" """ if internet_access_mode not in ["YES", "NO", "EGRESS_ONLY"]: raise ValueError( 'internet_access_mode should be either "YES", "NO", or "EGRESS_ONLY"' ) self.data["internetAccessMode"] = internet_access_mode return self def with_default_values(self): """ Set the VPC and Subnet to their default values: the vpc and subnet of the FM instance """ self.use_default_values = True return self class FMAWSVirtualNetworkCreator(FMVirtualNetworkCreator): def with_vpc(self, aws_vpc_id, aws_subnet_id): """ Setup the VPC and Subnet to be used by the virtual network :param str aws_vpc_id: ID of the VPC to use :param str aws_subnet_id: ID of the subnet to use """ self.data["awsVpcId"] = aws_vpc_id self.data["awsSubnetId"] = aws_subnet_id return self def with_auto_create_security_groups(self): """ Automatically create the AWS Security Groups when creating this virtual network """ self.data["awsAutoCreateSecurityGroups"] = True return self def with_aws_security_groups(self, *aws_security_groups): """ Use pre-created AWS Security Groups :param str aws_security_groups: Up to 5 security group ids to assign to the instances created in this virtual network """ self.data["awsAutoCreateSecurityGroups"] = False self.data["awsSecurityGroups"] = aws_security_groups return self def create(self): """ Create a new virtual network :return: a newly created network :rtype: :class:`dataikuapi.fm.virtualnetworks.FMAWSVirtualNetwork` """ vn = self.client._perform_tenant_json( "POST", "/virtual-networks", body=self.data, params={ 'useDefaultValues':self.use_default_values } ) return FMAWSVirtualNetwork(self.client, vn) class FMAzureVirtualNetworkCreator(FMVirtualNetworkCreator): def with_azure_virtual_network(self, azure_vn_id, azure_subnet_id): """ Setup the Azure Virtual Network and Subnet to be used by the virtual network :param str azure_vn_id: Resource ID of the Azure Virtual Network to use :param str azure_subnet_id: Resource ID of the subnet to use """ self.data["azureVnId"] = azure_vn_id self.data["azureSubnetId"] = azure_subnet_id return self def with_auto_update_security_groups(self, auto_update_security_groups=True): """ Auto update the security groups of the Azure Virtual Network :param boolean auto_update_security_groups: Optional, Auto update the subnet security group. Defaults to True """ self.data["azureAutoUpdateSecurityGroups"] = auto_update_security_groups return self def create(self): """ Create a new virtual network :return: a newly created network :rtype: :class:`dataikuapi.fm.virtualnetworks.FMAzureVirtualNetwork` """ vn = self.client._perform_tenant_json( "POST", "/virtual-networks", body=self.data, params={ 'useDefaultValues':self.use_default_values } ) return FMAzureVirtualNetwork(self.client, vn) class FMGCPVirtualNetworkCreator(FMVirtualNetworkCreator): def with_vpc(self, project_id, network, subnetwork): """ Setup the VPC which the virtual network will use :param str project_id: ID of the project in which the network is defined :param str network: name of the network :param str subnetwork: name of the subnetwork """ self.data["gcpProjectId"] = project_id self.data["gcpNetwork"] = network self.data["gcpSubnetwork"] = subnetwork return self def with_network_tags(self, *network_tags): """ Use network tags on the instances created in the virtual network :param str network_tags: network tags to assign to the instances created in this virtual network. """ self.data["gcpNetworkTags"] = network_tags return self def create(self): """ Create the virtual network :return: a newly created network :rtype: :class:`dataikuapi.fm.virtualnetworks.FMGCPVirtualNetwork` """ vn = self.client._perform_tenant_json( "POST", "/virtual-networks", body=self.data, params={ 'useDefaultValues':self.use_default_values } ) return FMGCPVirtualNetwork(self.client, vn) class FMVirtualNetwork(object): def __init__(self, client, vn_data): self.client = client self.vn_data = vn_data self.id = self.vn_data["id"] def save(self): """ Update this virtual network. """ self.client._perform_tenant_empty( "PUT", "/virtual-networks/%s" % self.id, body=self.vn_data ) self.vn_data = self.client._perform_tenant_json( "GET", "/virtual-networks/%s" % self.id ) def delete(self): """ Delete this virtual network. :return: the `Future` object representing the deletion process :rtype: :class:`dataikuapi.fm.future.FMFuture` """ future = self.client._perform_tenant_json( "DELETE", "/virtual-networks/%s" % self.id ) return FMFuture.from_resp(self.client, future) def set_fleet_management( self, enable, event_server=None, deployer_management="NO_MANAGED_DEPLOYER", govern_server=None ): """ When enabled, all instances in this virtual network know each other and can centrally manage deployer and logs centralization :param boolean enable: Enable or not the Fleet Management :param str event_server: Optional, Node name of the node that should act as the centralized event server for logs concentration. Audit logs of all design, deployer and automation nodes will automatically be sent there. :param str deployer_management: Optional, Accepts: - "NO_MANAGED_DEPLOYER": Do not manage the deployer. This is the default mode. - "CENTRAL_DEPLOYER": Central deployer. Recommended if you have more than one design node or may have more than one design node in the future. - "EACH_DESIGN_NODE": Deployer from design. Recommended if you have a single design node and want a simpler setup. :param str govern_server: Optional, node name of the node that should act as the centralized Govern server. """ self.vn_data["managedNodesDirectory"] = enable self.vn_data["eventServerNodeLabel"] = event_server self.vn_data["nodesDirectoryDeployerMode"] = deployer_management self.vn_data["governServerNodeLabel"] = govern_server return self def set_https_strategy(self, https_strategy): """ Set the HTTPS strategy for this virtual network :param https_strategy: the strategy to set :type https_strategy: :class:`dataikuapi.fm.virtualnetworks.FMHTTPSStrategy` """ self.vn_data.update(https_strategy) return self class FMAWSVirtualNetwork(FMVirtualNetwork): def set_dns_strategy( self, assign_domain_name, aws_private_ip_zone53_id=None, aws_public_ip_zone53_id=None, ): """ Set the DNS strategy for this virtual network :param boolean assign_domain_name: If false, don't assign domain names, use ip_only :param str aws_private_ip_zone53_id: Optional, AWS Only, the ID of the AWS Route53 Zone to use for private ip :param str aws_public_ip_zone53_id: Optional, AWS Only, the ID of the AWS Route53 Zone to use for public ip """ if assign_domain_name: self.vn_data["dnsStrategy"] = "VN_SPECIFIC_CLOUD_DNS_SERVICE" self.vn_data["awsRoute53PrivateIPZoneId"] = aws_private_ip_zone53_id self.vn_data["awsRoute53PublicIPZoneId"] = aws_public_ip_zone53_id else: self.vn_data["dnsStrategy"] = "NONE" return self class FMAzureVirtualNetwork(FMVirtualNetwork): def set_dns_strategy(self, assign_domain_name, azure_dns_zone_id=None): """ Set the DNS strategy for this virtual network :param boolean assign_domain_name: If false, don't assign domain names, use ip_only :param str azure_dns_zone_id: Optional, Azure Only, the ID of the Azure DNS zone to use """ if assign_domain_name: self.vn_data["dnsStrategy"] = "VN_SPECIFIC_CLOUD_DNS_SERVICE" self.vn_data["azureDnsZoneId"] = azure_dns_zone_id else: self.vn_data["dnsStrategy"] = "NONE" return self class FMGCPVirtualNetwork(FMVirtualNetwork): def set_assign_public_ip(self, public_ip=True): """ Sets whether the instances on this network will have a publicly accessible IP :param bool public_ip: if False, the instances will not be accessible from outside GCP """ if public_ip is not None: self.vn_data["gcpAssignPublicIP"] = public_ip return self def set_location_for_created_resources(self, project_id=None, zone=None): """ Set the location in GCP of the instances created using this virtual network :param str project_id: Optional, the project in which instances should be created. If empty string, then the project of the FM instance is used :param str zone: Optional, the zone in which instances should be created. If empty string, then the zone of the FM instance is used """ if zone == '': self.vn_data["gcpZone"] = None elif zone is not None: self.vn_data["gcpZone"] = zone if project_id == '': self.vn_data["gcpProjectIdForCreatedResources"] = None elif project_id is not None: self.vn_data["gcpProjectIdForCreatedResources"] = project_id return self def set_dns_strategy(self, assign_domain_name, private_ip_zone_id=None,public_ip_zone_id=None): """ Set the DNS strategy for this virtual network :param boolean assign_domain_name: If false, don't assign domain names, use ip_only :param str private_ip_zone_id: Optional, the ID of the Cloud DNS Zone to use for private ip :param str public_ip_zone_id: Optional, the ID of the Cloud DNS Zone to use for public ip """ if assign_domain_name: self.vn_data["dnsStrategy"] = "VN_SPECIFIC_CLOUD_DNS_SERVICE" self.vn_data["gcpCloudDnsPrivateIPZoneId"] = private_ip_zone_id self.vn_data["gcpCloudDnsPublicIPZoneId"] = public_ip_zone_id else: self.vn_data["dnsStrategy"] = "NONE" return self class FMHTTPSStrategy(dict): def __init__(self, data, https_strategy, http_redirect=False): """ A class holding HTTPS Strategy for virtual networks Do not create this directly, use: - :meth:`dataikuapi.fm.virtualnetwork.FMHTTPSStrategy.disable` to use HTTP only - :meth:`dataikuapi.fm.virtualnetwork.FMHTTPSStrategy.self_signed` to use self-signed certificates - :meth:`dataikuapi.fm.virtualnetwork.FMHTTPSStrategy.custom_cert` to use custom certificates - :meth:`dataikuapi.fm.virtualnetwork.FMHTTPSStrategy.lets_encrypt` to use Let's Encrypt """ super(FMHTTPSStrategy, self).__init__(data) self["httpsStrategy"] = https_strategy if http_redirect: self["httpStrategy"] = "REDIRECT" else: self["httpStrategy"] = "DISABLE" @staticmethod def disable(): """ Use HTTP only """ return FMHTTPSStrategy({}, "NONE", False) @staticmethod def self_signed(http_redirect): """ Use self-signed certificates :param bool http_redirect: If true, HTTP is redirected to HTTPS. If false, HTTP is disabled. Defaults to false """ return FMHTTPSStrategy({}, "SELF_SIGNED", http_redirect) @staticmethod def custom_cert(http_redirect): """ Use a custom certificate for each instance :param bool http_redirect: If true, HTTP is redirected to HTTPS. If false, HTTP is disabled. Defaults to false """ return FMHTTPSStrategy({}, "CUSTOM_CERTIFICATE", http_redirect) @staticmethod def lets_encrypt(contact_mail): """ Use Let's Encrypt to generate https certificates :param str contact_mail: The contact email provided to Let's Encrypt """ return FMHTTPSStrategy({"contactMail": contact_mail}, "LETSENCRYPT", True) dataiku-api-client-python-13.2.2/dataikuapi/fmclient.py000066400000000000000000000401461471211575100230630ustar00rootroot00000000000000import json from requests import Session from requests import exceptions from requests.auth import HTTPBasicAuth import os.path as osp from .utils import DataikuException from .iam.settings import FMSSOSettings, FMLDAPSettings, FMAzureADSettings from .fm.tenant import FMCloudCredentials, FMCloudTags from .fm.virtualnetworks import ( FMVirtualNetwork, FMAWSVirtualNetworkCreator, FMAzureVirtualNetworkCreator, FMGCPVirtualNetworkCreator, FMAWSVirtualNetwork, FMAzureVirtualNetwork, FMGCPVirtualNetwork ) from .fm.instances import ( FMInstance, FMInstanceEncryptionMode, FMAWSInstanceCreator, FMAzureInstanceCreator, FMGCPInstanceCreator, FMAWSInstance, FMAzureInstance, FMGCPInstance ) from .fm.instancesettingstemplates import ( FMInstanceSettingsTemplate, FMAWSInstanceSettingsTemplateCreator, FMAzureInstanceSettingsTemplateCreator, FMGCPInstanceSettingsTemplateCreator ) import sys if sys.version_info > (3, 4): from enum import Enum else: class Enum(object): pass class FMClient(object): def __init__( self, host, api_key_id, api_key_secret, tenant_id="main", extra_headers=None, insecure_tls=False, ): """ Base class for the different FM Clients Do not create this class, instead use :class:`dataikuapi.FMClientAWS`, :class:`dataikuapi.FMClientAzure` or :class:`dataikuapi.FMClientGCP` """ if self.cloud == None: raise NotImplementedError( "Do not use FMClient directly, instead use FMClientAWS, FMClientAzure or FMClientGCP" ) self.api_key_id = api_key_id self.api_key_secret = api_key_secret self.host = host self.__tenant_id = tenant_id self._session = Session() if insecure_tls: self._session.verify = False if self.api_key_id is not None and self.api_key_secret is not None: self._session.auth = HTTPBasicAuth(self.api_key_id, self.api_key_secret) else: raise ValueError("API Key ID and API Key secret are required") if extra_headers is not None: self._session.headers.update(extra_headers) ######################################################## # Tenant ######################################################## def get_tenant_id(self): return self.__tenant_id def get_cloud_credentials(self): """ Get the cloud credentials :return: cloud credentials :rtype: :class:`dataikuapi.fm.tenant.FMCloudCredentials` """ creds = self._perform_tenant_json("GET", "/cloud-credentials") return FMCloudCredentials(self, creds) def get_sso_settings(self): """ Get the Single Sign-On (SSO) settings :return: SSO settings :rtype: :class:`dataikuapi.iam.settings.SSOSettings` """ sso = self._perform_tenant_json("GET", "/iam/sso-settings") return FMSSOSettings(self, sso) def get_ldap_settings(self): """ Get the LDAP settings :return: LDAP settings :rtype: :class:`dataikuapi.iam.settings.LDAPSettings` """ ldap = self._perform_tenant_json("GET", "/iam/ldap-settings") return FMLDAPSettings(self, ldap) def get_azure_ad_settings(self): """ Get the Azure Active Directory (aka Microsoft Entra ID) settings :return: Azure AD settings :rtype: :class:`dataikuapi.iam.settings.AzureADSettings` """ ldap = self._perform_tenant_json("GET", "/iam/azure-ad-settings") return FMAzureADSettings(self, ldap) def get_cloud_tags(self): """ Get the tenant's cloud tags :return: tenant's cloud tags :rtype: :class:`dataikuapi.fm.tenant.FMCloudTags` """ tags = self._perform_tenant_json("GET", "/cloud-tags") return FMCloudTags(self, tags) ######################################################## # VirtualNetwork ######################################################## def _make_virtual_network(self, vn): if self.cloud == "AWS": return FMAWSVirtualNetwork(self, vn) elif self.cloud == "Azure": return FMAzureVirtualNetwork(self, vn) elif self.cloud == "GCP": return FMGCPVirtualNetwork(self, vn) else: raise Exception("Unknown cloud type %s" % self.cloud) def list_virtual_networks(self): """ List all virtual networks :return: list of virtual networks :rtype: list of :class:`dataikuapi.fm.virtualnetworks.FMVirtualNetwork` """ vns = self._perform_tenant_json("GET", "/virtual-networks") return [self._make_virtual_network(x) for x in vns] def get_virtual_network(self, virtual_network_id): """ Get a virtual network by its id :param str virtual_network_id: the id of the network to retrieve :return: the requested virtual network :rtype: :class:`dataikuapi.fm.virtualnetworks.FMVirtualNetwork` """ vn = self._perform_tenant_json( "GET", "/virtual-networks/%s" % virtual_network_id ) return self._make_virtual_network(vn) ######################################################## # Instance settings template ######################################################## def list_instance_templates(self): """ List all instance settings templates :return: list of instance settings template :rtype: list of :class:`dataikuapi.fm.tenant.FMInstanceSettingsTemplate` """ templates = self._perform_tenant_json("GET", "/instance-settings-templates") return [FMInstanceSettingsTemplate(self, x) for x in templates] def get_instance_template(self, template_id): """ Get an instance setting template template by its id :param str template_id: the id of the template to retrieve :return: the requested instance settings template :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMInstanceSettingsTemplate` """ template = self._perform_tenant_json( "GET", "/instance-settings-templates/%s" % template_id ) return FMInstanceSettingsTemplate(self, template) ######################################################## # Instance ######################################################## def _make_instance(self, i): if self.cloud == "AWS": return FMAWSInstance(self, i) elif self.cloud == "Azure": return FMAzureInstance(self, i) elif self.cloud == "GCP": return FMGCPInstance(self, i) else: raise Exception("Unknown cloud type %s" % self.cloud) def list_instances(self): """ List all DSS instances :return: list of instances :rtype: list of :class:`dataikuapi.fm.instances.FMInstance` """ instances = self._perform_tenant_json("GET", "/instances") return [self._make_instance(x) for x in instances] def get_instance(self, instance_id): """ Get a DSS instance by its id :param str instance_id: the id of the instance to retrieve :return: the requested instance if any :rtype: :class:`dataikuapi.fm.instances.FMInstance` """ instance = self._perform_tenant_json("GET", "/instances/%s" % instance_id) return self._make_instance(instance) def list_instance_images(self): """ List all available images to create new instances :return: list of images, as a pair of id and label """ return self._perform_tenant_json("GET", "/images") ######################################################## # Internal Request handling ######################################################## def _perform_http( self, method, path, params=None, body=None, stream=False, files=None, raw_body=None, ): if body is not None: body = json.dumps(body) if raw_body is not None: body = raw_body try: http_res = self._session.request( method, "%s/api/public%s" % (self.host, path), params=params, data=body, files=files, stream=stream, ) http_res.raise_for_status() return http_res except exceptions.HTTPError: try: ex = http_res.json() except ValueError: ex = {"message": http_res.text} raise DataikuException( "%s: %s" % ( ex.get("errorType", "Unknown error"), ex.get("message", "No message"), ) ) def _perform_empty( self, method, path, params=None, body=None, files=None, raw_body=None ): self._perform_http( method, path, params=params, body=body, files=files, stream=False, raw_body=raw_body, ) def _perform_json( self, method, path, params=None, body=None, files=None, raw_body=None ): return self._perform_http( method, path, params=params, body=body, files=files, stream=False, raw_body=raw_body, ).json() def _perform_tenant_json( self, method, path, params=None, body=None, files=None, raw_body=None ): return self._perform_json( method, "/tenants/%s%s" % (self.__tenant_id, path), params=params, body=body, files=files, raw_body=raw_body, ) def _perform_tenant_empty( self, method, path, params=None, body=None, files=None, raw_body=None ): self._perform_empty( method, "/tenants/%s%s" % (self.__tenant_id, path), params=params, body=body, files=files, raw_body=raw_body, ) class FMClientAWS(FMClient): def __init__( self, host, api_key_id, api_key_secret, tenant_id="main", extra_headers=None, insecure_tls=False, ): """ AWS Only - Instantiate a new FM API client on the given host with the given API key. The API key will define which operations are allowed for the client. :param str host: Full url of the FM """ self.cloud = "AWS" super(FMClientAWS, self).__init__( host, api_key_id, api_key_secret, tenant_id, extra_headers, insecure_tls ) def new_virtual_network_creator(self, label): """ Instantiate a new virtual network creator :param str label: The label of the network :rtype: :class:`dataikuapi.fm.virtualnetworks.FMAWSVirtualNetworkCreator` """ return FMAWSVirtualNetworkCreator(self, label) def new_instance_template_creator(self, label): """ Instantiate a new instance template creator :param str label: The label of the instance :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMAWSInstanceSettingsTemplateCreator` """ return FMAWSInstanceSettingsTemplateCreator(self, label) def new_instance_creator( self, label, instance_settings_template_id, virtual_network_id, image_id ): """ Instantiate a new instance creator :param str label: The label of the instance :param str instance_settings_template: The instance settings template id this instance should be based on :param str virtual_network: The virtual network where the instance should be spawned :param str image_id: The ID of the DSS runtime image (ex: dss-9.0.3-default) :rtype: :class:`dataikuapi.fm.instances.FMAWSInstanceCreator` """ return FMAWSInstanceCreator( self, label, instance_settings_template_id, virtual_network_id, image_id ) class FMClientAzure(FMClient): def __init__( self, host, api_key_id, api_key_secret, tenant_id="main", extra_headers=None, insecure_tls=False, ): """ Azure Only - Instantiate a new FM API client on the given host with the given API key. The API key will define which operations are allowed for the client. :param str host: Full url of the FM """ self.cloud = "Azure" super(FMClientAzure, self).__init__( host, api_key_id, api_key_secret, tenant_id, extra_headers, insecure_tls ) def new_virtual_network_creator(self, label): """ Instantiate a new virtual network creator :param str label: The label of the network :rtype: :class:`dataikuapi.fm.virtualnetworks.FMAzureVirtualNetworkCreator` """ return FMAzureVirtualNetworkCreator(self, label) def new_instance_template_creator(self, label): """ Instantiate a new instance template creator :param str label: The label of the instance :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMAzureInstanceSettingsTemplateCreator` """ return FMAzureInstanceSettingsTemplateCreator(self, label) def new_instance_creator( self, label, instance_settings_template_id, virtual_network_id, image_id ): """ Instantiate a new instance creator :param str label: The label of the instance :param str instance_settings_template: The instance settings template id this instance should be based on :param str virtual_network: The virtual network where the instance should be spawned :param str image_id: The ID of the DSS runtime image (ex: dss-9.0.3-default) :rtype: :class:`dataikuapi.fm.instances.FMAzureInstanceCreator` """ return FMAzureInstanceCreator( self, label, instance_settings_template_id, virtual_network_id, image_id ) class FMClientGCP(FMClient): def __init__( self, host, api_key_id, api_key_secret, tenant_id="main", extra_headers=None, insecure_tls=False, ): """ GCP Only - Instantiate a new FM API client on the given host with the given API key. The API key will define which operations are allowed for the client. :param str host: Full url of the FM """ self.cloud = "GCP" super(FMClientGCP, self).__init__( host, api_key_id, api_key_secret, tenant_id, extra_headers, insecure_tls ) def new_virtual_network_creator(self, label): """ Instantiate a new virtual network creator :param str label: The label of the network :rtype: :class:`dataikuapi.fm.virtualnetworks.FMGCPVirtualNetworkCreator` """ return FMGCPVirtualNetworkCreator(self, label) def new_instance_template_creator(self, label): """ Instantiate a new instance template creator :param str label: The label of the instance :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMGCPInstanceSettingsTemplateCreator` """ return FMGCPInstanceSettingsTemplateCreator(self, label) def new_instance_creator( self, label, instance_settings_template_id, virtual_network_id, image_id ): """ Instantiate a new instance creator :param str label: The label of the instance :param str instance_settings_template: The instance settings template id this instance should be based on :param str virtual_network: The virtual network where the instance should be spawned :param str image_id: The ID of the DSS runtime image (ex: dss-9.0.3-default) :rtype: :class:`dataikuapi.fm.instances.FMGCPInstanceCreator` """ return FMGCPInstanceCreator( self, label, instance_settings_template_id, virtual_network_id, image_id ) dataiku-api-client-python-13.2.2/dataikuapi/govern/000077500000000000000000000000001471211575100222035ustar00rootroot00000000000000dataiku-api-client-python-13.2.2/dataikuapi/govern/__init__.py000066400000000000000000000000001471211575100243020ustar00rootroot00000000000000dataiku-api-client-python-13.2.2/dataikuapi/govern/admin.py000066400000000000000000000353051471211575100236530ustar00rootroot00000000000000from datetime import datetime from dataikuapi.govern.future import GovernFuture from ..utils import _timestamp_ms_to_zoned_datetime class GovernUser(object): """ A handle for a user on the Govern instance. Do not create this object directly, use :meth:`~dataikuapi.govern_client.GovernClient.get_user` instead. """ def __init__(self, client, login): self.client = client self.login = login def delete(self): """ Delete the user """ return self.client._perform_empty( "DELETE", "/admin/users/%s" % self.login) def get_settings(self): """ Get the settings of the user :rtype: :class:`GovernUserSettings` """ raw = self.client._perform_json("GET", "/admin/users/%s" % self.login) return GovernUserSettings(self.client, self.login, raw) def get_activity(self): """ Gets the activity of the user. :return: the user's activity :rtype: :class:`GovernUserActivity` """ activity = self.client._perform_json("GET", "/admin/users/%s/activity" % self.login) return GovernUserActivity(self.client, self.login, activity) def get_client_as(self): """ Get a :class:`~dataikuapi.govern_client.GovernClient` that has the permissions of this user. This allows administrators to impersonate actions on behalf of other users, in order to perform actions on their behalf """ from dataikuapi.govern_client import GovernClient if self.client.api_key is not None: return GovernClient(self.client.host, self.client.api_key, extra_headers={"X-DKU-ProxyUser": self.login}, insecure_tls=not self.client._session.verify) elif self.client.internal_ticket is not None: return GovernClient(self.client.host, internal_ticket=self.client.internal_ticket, extra_headers={"X-DKU-ProxyUser": self.login}, insecure_tls=not self.client._session.verify) else: raise ValueError("Don't know how to proxy this client") ######################################################## # Supplier interaction ######################################################## def start_resync_from_supplier(self): """ Starts a resync of the user from an external supplier (LDAP, Azure AD or custom auth) :return: a :class:`dataikuapi.govern.future.GovernFuture` representing the sync process :rtype: :class:`dataikuapi.govern.future.GovernFuture` """ future_resp = self.client._perform_json("POST", "/admin/users/%s/actions/resync" % self.login) return GovernFuture.from_resp(self.client, future_resp) class GovernOwnUser(object): """ A handle to interact with your own user Do not create this object directly, use :meth:`~dataikuapi.govern_client.GovernClient.get_own_user` instead. """ def __init__(self, client): self.client = client def get_settings(self): """ Get your own settings :rtype: :class:`GovernOwnUserSettings` """ raw = self.client._perform_json("GET", "/current-user") return GovernOwnUserSettings(self.client, raw) class GovernUserSettingsBase(object): """ Settings for a Govern user. Do not create this object directly, use :meth:`GovernUser.get_settings` or :meth:`GovernOwnUser.get_settings` instead. """ def __init__(self, settings): self.settings = settings def get_raw(self): """ :return: the raw settings of the user, as a dict. Modifications made to the returned object are reflected when saving :rtype: dict """ return self.settings class GovernUserSettings(GovernUserSettingsBase): """ Settings for a Govern user. Do not create this object directly, use :meth:`GovernUser.get_settings` instead. """ def __init__(self, client, login, settings): super(GovernUserSettings, self).__init__(settings) self.client = client self.login = login @property def enabled(self): """ Whether this user is enabled :rtype: boolean """ return self.settings["enabled"] @enabled.setter def enabled(self, new_value): self.settings["enabled"] = new_value @property def creation_date(self): """ Get the creation date of the user as a :class:`datetime.datetime` :return: the creation date :rtype: :class:`datetime.datetime` or None """ timestamp = self.settings["creationDate"] if "creationDate" in self.settings else None return _timestamp_ms_to_zoned_datetime(timestamp) def save(self): """Saves the settings""" self.client._perform_json("PUT", "/admin/users/%s" % self.login, body=self.settings) class GovernOwnUserSettings(GovernUserSettingsBase): """ Settings for the current Govern user. Do not create this object directly, use :meth:`GovernOwnUser.get_settings` instead. """ def __init__(self, client, settings): super(GovernOwnUserSettings, self).__init__(settings) self.client = client def save(self): """Saves the settings back to the current user""" self.client._perform_empty("PUT", "/current-user", body=self.settings) class GovernUserActivity(object): """ Activity for a Govern user. .. important:: Do not instantiate directly, use :meth:`GovernUser.get_activity` or :meth:`dataikuapi.GovernClient.list_users_activity()` instead. """ def __init__(self, client, login, activity): self.client = client self.login = login self.activity = activity def get_raw(self): """ Get the raw activity of the user as a dict. :return: the raw activity. Fields are * **login** : the login of the user for this activity * **lastSuccessfulLogin** : timestamp in milliseconds of the last time the user logged into Govern * **lastFailedLogin** : timestamp in milliseconds of the last time Govern recorded a login failure for this user * **lastSessionActivity** : timestamp in milliseconds of the last time the user opened a tab :rtype: dict """ return self.activity @property def last_successful_login(self): """ Get the last successful login of the user Returns None if there was no successful login for this user. :return: the last successful login :rtype: :class:`datetime.datetime` or None """ timestamp = self.activity["lastSuccessfulLogin"] return _timestamp_ms_to_zoned_datetime(timestamp) @property def last_failed_login(self): """ Get the last failed login of the user Returns None if there were no failed login for this user. :return: the last failed login :rtype: :class:`datetime.datetime` or None """ timestamp = self.activity["lastFailedLogin"] return _timestamp_ms_to_zoned_datetime(timestamp) @property def last_session_activity(self): """ Get the last session activity of the user The last session activity is the last time the user opened a new Govern tab or refreshed his session. Returns None if there is no session activity yet. :return: the last session activity :rtype: :class:`datetime.datetime` or None """ timestamp = self.activity["lastSessionActivity"] return _timestamp_ms_to_zoned_datetime(timestamp) class GovernAuthorizationMatrix(object): """ The authorization matrix of all groups and enabled users of the Govern instance. .. important:: Do not instantiate directly, use :meth:`dataikuapi.GovernClient.get_authorization_matrix` instead. """ def __init__(self, authorization_matrix): self.authorization_matrix = authorization_matrix @property def raw(self): """ Get the raw authorization matrix as a dict :return: the authorization matrix. There are 2 parts in the matrix, each as a top-level field and with similar structures, **perUser** and **perGroup**. :rtype: dict """ return self.authorization_matrix class GovernGroup(object): """ A group on the Govern instance. Do not create this object directly, use :meth:`~dataikuapi.govern_client.GovernClient.get_group` instead. """ def __init__(self, client, name): self.client = client self.name = name def delete(self): """ Delete the group """ return self.client._perform_empty( "DELETE", "/admin/groups/%s" % self.name) def get_definition(self): """ Get the group's definition (name, description, admin abilities, type, ldap name mapping) :return: the group's definition, as a dict """ return self.client._perform_json( "GET", "/admin/groups/%s" % self.name) def set_definition(self, definition): """ Set the group's definition. You should only :meth:`set_definition` using an object that you obtained through :meth:`get_definition`, not create a new dict. :param: dict definition: the definition for the group, as a dict :return: a dict - the definition of the group :rtype: dict """ return self.client._perform_json( "PUT", "/admin/groups/%s" % self.name, body=definition) class GovernGlobalApiKey(object): """ A global API key on the Govern instance """ def __init__(self, client, key, id_): self.client = client self.key = key self.id_ = id_ ######################################################## # Key deletion ######################################################## def delete(self): """ Delete the api key .. note:: This call requires an API key with admin rights """ return self.client._perform_empty( "DELETE", "/admin/global-api-keys/%s" % self.id_) ######################################################## # Key description ######################################################## def get_definition(self): """ Get the API key's definition .. note:: This call requires an API key with admin rights .. note:: If the secure API keys feature is enabled, the secret key of this API key will not be present in the returned dict :return: the API key definition, as a dict. The dict additionally contains the definition of the permissions attached to the key. :rtype: dict """ return self.client._perform_json( "GET", "/admin/global-api-keys/%s" % self.id_) def set_definition(self, definition): """ Set the API key's definition .. note:: This call requires an API key with admin rights .. important:: You should only :meth:`set_definition` using an object that you obtained through :meth:`get_definition`, not create a new dict. You may not use this method to update the 'key' field. Usage example .. code-block:: python # make an API key able to create projects key = client.get_global_api_key('my_api_key_secret') definition = key.get_definition() definition["globalPermissions"]["admin"] = True key.set_definition(definition) :param dict definition: the definition for the API key """ return self.client._perform_empty( "PUT", "/admin/global-api-keys/%s" % self.id_, body = definition) class GovernGlobalApiKeyListItem(dict): """ An item in a list of global API keys. .. important:: Do not instantiate directly, use :meth:`dataikuapi.GovernClient.list_global_api_keys` instead. """ def __init__(self, client, data): super(GovernGlobalApiKeyListItem, self).__init__(data) self.client = client def to_global_api_key(self): """ Gets a handle corresponding to this item :rtype: :class:`GovernGlobalApiKey` """ return GovernGlobalApiKey(self.client, self["key"], self["id"]) @property def id(self): """ Get the identifier of the API key :rtype: string """ return self["id"] @property def user_for_impersonation(self): """ Get the user associated to the API key :rtype: string """ return self.get("dssUserForImpersonation") @property def key(self): """ Get the API key If the secure API keys feature is enabled, this key field will not be available :rtype: string """ return self["key"] @property def label(self): """ Get the label of the API key :rtype: string """ return self["label"] @property def description(self): """ Get the description of the API key :rtype: string """ return self.get("description") @property def created_on(self): """ Get the timestamp of when the API key was created :rtype: :class:`datetime.datetime` """ timestamp = self["createdOn"] return _timestamp_ms_to_zoned_datetime(timestamp) @property def created_by(self): """ Get the login of the user who created the API key :rtype: string """ return self["createdBy"] class GovernGeneralSettings(object): """ The general settings of the Govern instance. Do not create this object directly, use :meth:`~dataikuapi.govern_client.GovernClient.get_general_settings` instead. """ def __init__(self, client): self.client = client self.settings = self.client._perform_json("GET", "/admin/general-settings") ######################################################## # Update settings on instance ######################################################## def save(self): """ Save the changes that were made to the settings on the Govern instance Note: this call requires an API key with admin rights """ return self.client._perform_empty("PUT", "/admin/general-settings", body=self.settings) ######################################################## # Value accessors ######################################################## def get_raw(self): """ Get the settings as a dictionary """ return self.settings dataiku-api-client-python-13.2.2/dataikuapi/govern/admin_blueprint_designer.py000066400000000000000000000447471471211575100276310ustar00rootroot00000000000000class GovernAdminBlueprintDesigner(object): """ Handle to interact with the blueprint designer Do not create this directly, use :meth:`~dataikuapi.govern_client.GovernClient.get_blueprint_designer` """ def __init__(self, client): self.client = client def list_blueprints(self): """ List blueprints :return: the list of blueprints :rtype: list of :class:`~dataikuapi.govern.admin_blueprint_designer.GovernAdminBlueprintListItem` """ blueprints = self.client._perform_json("GET", "/admin/blueprints") return [GovernAdminBlueprintListItem(self.client, blueprint) for blueprint in blueprints] def get_blueprint(self, blueprint_id): """ Get a specific blueprint. :param str blueprint_id: the ID of the blueprint :return: a blueprint object :rtype: :class:`~dataikuapi.govern.admin_blueprint_designer.GovernAdminBlueprint` """ return GovernAdminBlueprint(self.client, blueprint_id) def create_blueprint(self, new_identifier, blueprint): """ Create a new blueprint and returns a handle to interact with it. :param str new_identifier: the new identifier for the blueprint. Allowed characters are letters, digits, hyphen, and underscore. :param dict blueprint: the blueprint definition :return: The handle for the newly created blueprint :rtype: :class:`~dataikuapi.govern.admin_blueprint_designer.GovernAdminBlueprint` """ result = self.client._perform_json("POST", "/admin/blueprints", params={"newIdentifier": new_identifier}, body=blueprint) return GovernAdminBlueprint(self.client, result["blueprint"]["id"]) class GovernAdminBlueprintListItem(object): """ An item in a list of blueprints. Do not create this directly, use :meth:`~dataikuapi.govern.admin_blueprint_designer.GovernAdminBlueprintDesigner.list_blueprints` """ def __init__(self, client, data): self.client = client self._data = data def get_raw(self): """ Get the raw content of the blueprint list item :return: the raw content of the blueprint list item as a dict :rtype: dict """ return self._data def to_blueprint(self): """ Gets the :class:`~dataikuapi.govern.admin_blueprint_designer.GovernAdminBlueprint` corresponding to this blueprint object :return: the blueprint object :rtype: a :class:`~dataikuapi.govern.admin_blueprint_designer.GovernAdminBlueprint` """ return GovernAdminBlueprint(self.client, self._data["blueprint"]["id"]) class GovernAdminBlueprint(object): """ A handle to interact with a blueprint as an admin on the Govern instance. Do not create this directly, use :meth:`~dataikuapi.govern.admin_blueprint_designer.GovernAdminBlueprintDesigner.get_blueprint` """ def __init__(self, client, blueprint_id): self.client = client self.blueprint_id = blueprint_id def get_definition(self): """ Get the definition of the blueprint as an object. To modify the definition, call :meth:`~dataikuapi.govern.admin_blueprint_designer.GovernAdminBlueprintDefinition.save` on the returned object. :return: The blueprint definition as an object. :rtype: :class:`~dataikuapi.govern.admin_blueprint_designer.GovernAdminBlueprintDefinition` """ blueprint = self.client._perform_json("GET", "/admin/blueprint/%s" % self.blueprint_id) return GovernAdminBlueprintDefinition(self.client, self.blueprint_id, blueprint["blueprint"]) def list_versions(self): """ List versions of this blueprint. :return: The list of the versions of the blueprint :rtype: list of :class:`~dataikuapi.govern.admin_blueprint_designer.GovernAdminBlueprintVersionListItem` """ versions = self.client._perform_json("GET", "/admin/blueprint/%s/versions" % self.blueprint_id) return [GovernAdminBlueprintVersionListItem(self.client, self.blueprint_id, version) for version in versions] def create_version(self, new_identifier, name=None, origin_version_id=None): """ Create a new blueprint version and returns a handle to interact with it. :param str new_identifier: The new identifier of the blueprint version. Allowed characters are letters, digits, hyphen, and underscore. :param str name: (Optional) The name of the blueprint version. :param str origin_version_id: (Optional) The blueprint version ID of the origin version ID if there is one. :return: The handle of the newly created blueprint :rtype: :class:`~dataikuapi.govern.admin_blueprint_designer.GovernAdminBlueprintVersion` """ params = {"newIdentifier": new_identifier} if name is not None: params["name"] = name if origin_version_id is not None: params["originVersionId"] = origin_version_id version = self.client._perform_json("POST", "/admin/blueprint/%s/versions" % self.blueprint_id, params=params) return GovernAdminBlueprintVersion(self.client, self.blueprint_id, version["blueprintVersion"]["id"]["versionId"]) def get_version(self, version_id): """ Get a blueprint version and return a handle to interact with it. :param str version_id: ID of the version :rtype: :class:`~dataikuapi.govern.admin_blueprint_designer.GovernAdminBlueprintVersion` """ return GovernAdminBlueprintVersion(self.client, self.blueprint_id, version_id) def delete(self): """ Delete the blueprint. To delete a blueprint, all related blueprint versions and artifacts must be deleted beforehand. :return: None """ self.client._perform_empty("DELETE", "/admin/blueprint/%s" % self.blueprint_id) class GovernAdminBlueprintDefinition(object): """ The definition of a blueprint. Do not create this class directly, instead use :meth:`~dataikuapi.govern.admin_blueprint_designer.GovernAdminBlueprint.get_definition` """ def __init__(self, client, blueprint_id, definition): self.client = client self.blueprint_id = blueprint_id self.definition = definition def get_raw(self): """ Get raw definition of the blueprint :return: the raw definition of blueprint, as a dict. Modifications made to the returned object are reflected when saving :rtype: dict """ return self.definition def save(self): """ Save this settings back to the blueprint. :return: None """ self.definition = self.client._perform_json("PUT", "/admin/blueprint/%s" % self.blueprint_id, body=self.definition)["blueprint"] class GovernAdminBlueprintVersionListItem(object): """ An item in a list of blueprint versions. Do not create this directly, use :meth:`~dataikuapi.govern.admin_blueprint_designer.GovernAdminBlueprint.list_versions` """ def __init__(self, client, blueprint_id, data): self.client = client self.blueprint_id = blueprint_id self._data = data def get_raw(self): """ Get the raw content of the blueprint version list item :return: the raw content of the blueprint version list item as a dict :rtype: dict """ return self._data def to_blueprint_version(self): """ Gets the :class:`~dataikuapi.govern.admin_blueprint_designer.GovernAdminBlueprintVersion` corresponding to this blueprint version object :return: the blueprint object :rtype: a :class:`~dataikuapi.govern.admin_blueprint_designer.GovernAdminBlueprintVersion` """ return GovernAdminBlueprintVersion(self.client, self.blueprint_id, self._data["blueprintVersion"]["id"]["versionId"]) class GovernAdminBlueprintVersion(object): """ A handle to interact with a blueprint version. Do not create this directly, use :meth:`~dataikuapi.govern.admin_blueprint_designer.GovernAdminBlueprint.get_version` """ def __init__(self, client, blueprint_id, version_id): self.client = client self.blueprint_id = blueprint_id self.version_id = version_id def get_definition(self): """ Get the definition of this blueprint version. To modify the definition, call :meth:`~dataikuapi.govern.admin_blueprint_designer.GovernAdminBlueprintVersionDefinition.save` on the returned object. :return: The definition of the blueprint version as an object. :rtype: :class:`~dataikuapi.govern.admin_blueprint_designer.GovernAdminBlueprintVersionDefinition`. """ version = self.client._perform_json("GET", "/admin/blueprint/%s/version/%s" % (self.blueprint_id, self.version_id)) return GovernAdminBlueprintVersionDefinition(self.client, self.blueprint_id, self.version_id, version["blueprintVersion"]) def get_trace(self): """ Get a handle of the blueprint version trace containing information about its lineage and its status. :return: the trace of the blueprint version. :rtype: :class:`~dataikuapi.govern.admin_blueprint_designer.GovernAdminBlueprintVersionTrace`. """ version = self.client._perform_json("GET", "/admin/blueprint/%s/version/%s" % (self.blueprint_id, self.version_id)) return GovernAdminBlueprintVersionTrace(self.client, self.blueprint_id, self.version_id, version["blueprintVersionTrace"]) def list_signoff_configurations(self): """ Get the blueprint sign-off configurations of this blueprint version. :return: The list of sign-off configurations :rtype: list of :class:`~dataikuapi.govern.admin_blueprint_designer.GovernAdminSignoffConfigurationListItem` """ configurations = self.client._perform_json("GET", "/admin/blueprint/%s/version/%s/signoffs" % (self.blueprint_id, self.version_id)) return [GovernAdminSignoffConfigurationListItem(self.client, self.blueprint_id, self.version_id, configuration) for configuration in configurations] def get_signoff_configuration(self, step_id): """ Get the sign-off configurations for a specific step :param str step_id: The step ID of the sign-off :return: The sign-off configuration as an object :rtype: a :class:`~dataikuapi.govern.admin_blueprint_designer.GovernAdminSignoffConfiguration` """ return GovernAdminSignoffConfiguration(self.client, self.blueprint_id, self.version_id, step_id) def create_signoff_configuration(self, step_id, signoff_configuration): """ Create a new sign-off for a specific step of the workflow and return a handle to interact with it. :param str step_id: The step ID of the workflow on which the sign-off will be added. :param dict signoff_configuration: The configuration of the sign-off :return: The newly created sign-off configuration as an object :rtype: :class:`~dataikuapi.govern.admin_blueprint_designer.GovernAdminSignoffConfiguration` """ self.client._perform_json("POST", "/admin/blueprint/%s/version/%s/workflow/step/%s/signoff" % (self.blueprint_id, self.version_id, step_id), body=signoff_configuration) return GovernAdminSignoffConfiguration(self.client, self.blueprint_id, self.version_id, step_id) def delete(self): """ Delete the blueprint version. To delete a blueprint, all related artifacts must be deleted beforehand. :return: None """ self.client._perform_empty("DELETE", "/admin/blueprint/%s/version/%s" % (self.blueprint_id, self.version_id)) class GovernAdminBlueprintVersionDefinition(object): """ The blueprint version definition. Do not create this directly, use :meth:`~dataikuapi.govern.admin_blueprint_designer.GovernAdminBlueprintVersion.get_definition` """ def __init__(self, client, blueprint_id, version_id, definition): self.client = client self.blueprint_id = blueprint_id self.version_id = version_id self.definition = definition def get_raw(self): """ Get raw definition of the blueprint version. :return: the raw definition of blueprint version, as a dict. Modifications made to the returned object are reflected when saving :rtype: dict """ return self.definition def save(self, danger_zone_accepted=None): """ Save this definition back to the blueprint version definition. :param boolean danger_zone_accepted: ignore the warning about existing artifacts. If there are existing artifacts using this blueprint version, modifying it may break them (ie. removing artifact field values). By default, the save call will fail in this case. If this parameter is set to true, the call will ignore the warning and be run anyway. :return: None """ params = {} if danger_zone_accepted is not None: params["dangerZoneAccepted"] = danger_zone_accepted self.definition = self.client._perform_json("PUT", "/admin/blueprint/%s/version/%s" % (self.blueprint_id, self.version_id), params=params, body=self.definition)["blueprintVersion"] class GovernAdminBlueprintVersionTrace(object): """ The trace of a blueprint version containing information about its lineage and its status. Do not create this directly, use :meth:`~dataikuapi.govern.admin_blueprint_designer.GovernAdminBlueprintVersion.get_trace` """ def __init__(self, client, blueprint_id, version_id, trace): self.client = client self.blueprint_id = blueprint_id self.version_id = version_id self.trace = trace def get_raw(self): """ Get raw trace of the blueprint version. :return: The raw trace of blueprint version, as a dict. :rtype: dict """ return self.trace @property def status(self): """ Get the status of the blueprint version among (DRAFT, ACTIVE, or ARCHIVED) :rtype: str """ return self.trace.get("status") @property def origin_version_id(self): """ Get the origin version ID of this blueprint version :rtype: str """ return self.trace.get("originVersionId") def set_status(self, status): """ Directly update the status of the blueprint version. :param str status: DRAFT, ACTIVE, or ARCHIVED :return: None """ version = self.client._perform_json("PUT", "/admin/blueprint/%s/version/%s/status" % (self.blueprint_id, self.version_id), body=status) self.trace = version["blueprintVersionTrace"] class GovernAdminSignoffConfigurationListItem(object): """ An item in a list of sign-off configurations. Do not create this directly, use :meth:`~dataikuapi.govern.admin_blueprint_designer.GovernAdminBlueprintVersion.list_signoff_configurations` """ def __init__(self, client, blueprint_id, version_id, data): self.client = client self.blueprint_id = blueprint_id self.version_id = version_id self._data = data def get_raw(self): """ Get the raw content of the sign-off configuration list item :return: the raw content of the sign-off configuration list item as a dict :rtype: dict """ return self._data def to_signoff_configuration(self): """ Gets the :class:`~dataikuapi.govern.admin_blueprint_designer.GovernAdminSignoffConfiguration` corresponding to this sign-off configuration object :return: the sign-off configuration object :rtype: a :class:`~dataikuapi.govern.admin_blueprint_designer.GovernAdminSignoffConfiguration` """ return GovernAdminSignoffConfiguration(self.client, self.blueprint_id, self.version_id, self._data["id"]["stepId"]) class GovernAdminSignoffConfiguration(object): """ A handle to interact with the sign-off configuration of a specific step of a workflow. Do not create this directly, use :meth:`~dataikuapi.govern.admin_blueprint_designer.GovernAdminBlueprintVersion.get_signoff_configuration` """ def __init__(self, client, blueprint_id, version_id, step_id): self.client = client self.blueprint_id = blueprint_id self.version_id = version_id self.step_id = step_id def get_definition(self): """ Get the definition of the configuration, to modify the configuration call :meth:`~dataikuapi.govern.admin_blueprint_designer.GovernAdminSignoffConfigurationDefinition.save` on the returned object. :return: The blueprint definition as an object. :rtype: :class:`~dataikuapi.govern.admin_blueprint_designer.GovernAdminSignoffConfigurationDefinition` """ definition = self.client._perform_json("GET", "/admin/blueprint/%s/version/%s/workflow/step/%s/signoff" % (self.blueprint_id, self.version_id, self.step_id)) return GovernAdminSignoffConfigurationDefinition(self.client, self.blueprint_id, self.version_id, self.step_id, definition) def delete(self): """ Delete the sign-off configuration. :return: None """ self.client._perform_empty("DELETE", "/admin/blueprint/%s/version/%s/workflow/step/%s/signoff" % (self.blueprint_id, self.version_id, self.step_id)) class GovernAdminSignoffConfigurationDefinition(object): """ The definition of sign-off configuration. Do not create this class directly, instead use :meth:`~dataikuapi.govern.admin_blueprint_designer.GovernAdminSignoffConfiguration.get_definition` """ def __init__(self, client, blueprint_id, version_id, step_id, definition): self.client = client self.blueprint_id = blueprint_id self.version_id = version_id self.step_id = step_id self.definition = definition def get_raw(self): """ Get raw definition of the sign-off configuration :return: the raw configuration of the sign-off, as a dict. Modifications made to the returned object are reflected when saving :rtype: dict """ return self.definition def save(self): """ Save this settings back to the sign-off configuration. :return: None """ self.definition = self.client._perform_json( "PUT", "/admin/blueprint/%s/version/%s/workflow/step/%s/signoff" % (self.blueprint_id, self.version_id, self.step_id), body=self.definition) dataiku-api-client-python-13.2.2/dataikuapi/govern/admin_custom_pages_handler.py000066400000000000000000000126101471211575100301130ustar00rootroot00000000000000class GovernAdminCustomPagesHandler(object): """ Handle to edit the custom pages Do not create this directly, use :meth:`~dataikuapi.govern_client.GovernClient.get_custom_pages_handler` """ def __init__(self, client): self.client = client def list_custom_pages(self): """ List custom pages :return: A list of custom pages :rtype: list of :class:`~dataikuapi.govern.admin_custom_pages_handler.GovernAdminCustomPageListItem` """ pages = self.client._perform_json("GET", "/admin/custom-pages") return [GovernAdminCustomPageListItem(self.client, page) for page in pages] def get_custom_page(self, custom_page_id): """ Get a custom page :param str custom_page_id: ID of the custom page to retrieve :return: A custom page as an object :rtype: :class:`~dataikuapi.govern.admin_custom_pages_handler.GovernAdminCustomPage` """ return GovernAdminCustomPage(self.client, custom_page_id) def create_custom_page(self, new_identifier, custom_page): """ Create a custom page :param str new_identifier: the new identifier for this custom page. Allowed characters are letters, digits, hyphen, and underscore. :param dict custom_page: the custom page definition. :return: the handle of the created custom page :rtype: :class:`~dataikuapi.govern.admin_custom_pages_handler.GovernAdminCustomPage` """ result = self.client._perform_json("POST", "/admin/custom-pages", params={"newIdentifier": new_identifier}, body=custom_page) return GovernAdminCustomPage(self.client, result["id"]) def get_custom_pages_order(self): """ Retrieves the order of display of the custom pages by their id :return: the order of the pages :rtype: list[string] """ order = self.client._perform_json("GET", "/admin/custom-pages/order") return order def save_custom_pages_order(self, custom_pages_order): """ Update the order of display of the custom pages. :param custom_pages_order: list[string] new custom pages order. Must contain ids of all the custom pages :return: updated custom pages order :rtype: list[string] """ order = self.client._perform_json("PUT", "/admin/custom-pages/order", body=custom_pages_order) return order class GovernAdminCustomPageListItem(object): """ An item in a list of custom pages. Do not create this directly, use :meth:`~dataikuapi.govern.admin_custom_pages_handler.GovernAdminCustomPagesHandler.list_custom_pages` """ def __init__(self, client, data): self.client = client self._data = data def get_raw(self): """ Get the raw content of the custom page list item :return: the raw content of the custom page list item as a dict :rtype: dict """ return self._data def to_custom_page(self): """ Gets the :class:`~dataikuapi.govern.admin_custom_pages_handler.GovernAdminCustomPage` corresponding to this custom page object :return: the custom page object :rtype: a :class:`~dataikuapi.govern.admin_custom_pages_handler.GovernAdminCustomPage` """ return GovernAdminCustomPage(self.client, self._data["id"]) class GovernAdminCustomPage(object): """ A handle to interact with a custom page as an administrator. Do not create this directly, use :meth:`~dataikuapi.govern.admin_custom_pages_handler.GovernAdminCustomPagesHandler.get_custom_page` """ def __init__(self, client, custom_page_id): self.client = client self.custom_page_id = custom_page_id def get_definition(self): """ Get the definition of the custom page, to modify the definition call :meth:`~dataikuapi.govern.admin_custom_pages_handler.GovernAdminCustomPageDefinition.save` on the returned object. :return: A custom page definition as an object :rtype: :class:`~dataikuapi.govern.admin_custom_pages_handler.GovernAdminCustomPageDefinition` """ definition = self.client._perform_json("GET", "/admin/custom-page/%s" % self.custom_page_id) return GovernAdminCustomPageDefinition(self.client, self.custom_page_id, definition) def delete(self): """ Delete the custom page :return: None """ self.client._perform_empty("DELETE", "/admin/custom-page/%s" % self.custom_page_id) class GovernAdminCustomPageDefinition(object): """ The definition of a custom page. Do not create this directly, use :meth:`~dataikuapi.govern.admin_custom_pages_handler.GovernAdminCustomPage.get_definition` """ def __init__(self, client, custom_page_id, definition): self.client = client self.custom_page_id = custom_page_id self.definition = definition def get_raw(self): """ Get the raw content of the custom page. This returns a reference to the custom page so changes made to the returned object will be reflected when saving. :rtype: dict """ return self.definition def save(self): """ Save this settings back to the custom page. :return: None """ self.definition = self.client._perform_json("PUT", "/admin/custom-page/%s" % self.custom_page_id, body=self.definition) dataiku-api-client-python-13.2.2/dataikuapi/govern/admin_roles_permissions_handler.py000066400000000000000000000363451471211575100312140ustar00rootroot00000000000000class GovernAdminRolesPermissionsHandler(object): """ Handle to edit the roles and permissions Do not create this directly, use :meth:`~dataikuapi.govern_client.GovernClient.get_roles_permissions_handler` """ def __init__(self, client): self.client = client def list_roles(self): """ List roles :return: a list of roles :rtype: list of :class:`~dataikuapi.govern.admin_roles_permissions_handler.GovernAdminRoleListItem` """ roles = self.client._perform_json("GET", "/admin/roles") return [GovernAdminRoleListItem(self.client, role) for role in roles] def get_role(self, role_id): """ Return a specific role on the Govern instance :param str role_id: Identifier of the role :rtype: :class:`~dataikuapi.govern.admin_roles_permissions_handler.GovernAdminRole` """ return GovernAdminRole(self.client, role_id) def create_role(self, new_identifier, role): """ Create a new role and returns the handle to interact with it. :param str new_identifier: Identifier of the new role. Allowed characters are letters, digits, hyphen, and underscore. :param dict role: The definition of the role. :rtype: :class:`~dataikuapi.govern.admin_roles_permissions_handler.GovernAdminRole` """ result = self.client._perform_json("POST", "/admin/roles/", params={"newIdentifier": new_identifier}, body=role) return GovernAdminRole(self.client, result["id"]) def list_role_assignments(self): """ List the blueprint role assignments :return: A list of role assignments for each blueprint :rtype: list of :class:`~dataikuapi.govern.admin_roles_permissions_handler.GovernAdminBlueprintRoleAssignmentsListItem` """ assignments_list = self.client._perform_json("GET", "/admin/blueprint-role-assignments") return [GovernAdminBlueprintRoleAssignmentsListItem(self.client, assignments) for assignments in assignments_list] def get_role_assignments(self, blueprint_id): """ Get the role assignments for a specific blueprint. Returns a handle to interact with it. :param str blueprint_id: id of the blueprint :return: A object representing the role assignments for this blueprint :rtype: :class:`~dataikuapi.govern.admin_roles_permissions_handler.GovernAdminBlueprintRoleAssignments` """ return GovernAdminBlueprintRoleAssignments(self.client, blueprint_id) def create_role_assignments(self, role_assignments): """ Create a role assignments on the Govern instance and returns the handle to interact with it. :param dict role_assignment: Blueprint permission as a dict :return: The newly created role assignment. :rtype: :class:`~dataikuapi.govern.admin_roles_permissions_handler.GovernAdminBlueprintRoleAssignments` """ result = self.client._perform_json("POST", "/admin/blueprint-role-assignments", body=role_assignments) return GovernAdminBlueprintRoleAssignments(self.client, result["blueprintId"]) def get_default_permissions_definition(self): """ Get the default permissions definition. :return: A permissions object :rtype: :class:`~dataikuapi.govern.admin_roles_permissions_handler.GovernAdminDefaultPermissionsDefinition` """ definition = self.client._perform_json("GET", "/admin/blueprint-permissions/default") return GovernAdminDefaultPermissionsDefinition(self.client, definition) def list_blueprint_permissions(self): """ List blueprint permissions :return: A list of blueprint permissions :rtype: list of :class:`~dataikuapi.govern.admin_roles_permissions_handler.GovernAdminBlueprintPermissionsListItem` """ permissions_list = self.client._perform_json("GET", "/admin/blueprint-permissions") return [GovernAdminBlueprintPermissionsListItem(self.client, permissions) for permissions in permissions_list] def get_blueprint_permissions(self, blueprint_id): """ Get the permissions for a specific blueprint. Returns a handle to interact with the permissions :param str blueprint_id: id of the blueprint for which you need the permissions :return: A permissions object for the specific blueprint :rtype: :class:`~dataikuapi.govern.admin_roles_permissions_handler.GovernAdminBlueprintPermissions` """ return GovernAdminBlueprintPermissions(self.client, blueprint_id) def create_blueprint_permissions(self, blueprint_permission): """ Create blueprint permissions and returns the handle to interact with it. :param dict blueprint_permission: Blueprint permission as a dict :return: the newly created permissions object :rtype: :class:`~dataikuapi.govern.admin_roles_permissions_handler.GovernAdminBlueprintPermissions` """ result = self.client._perform_json("POST", "/admin/blueprint-permissions", body=blueprint_permission) return GovernAdminBlueprintPermissions(self.client, result["blueprintId"]) class GovernAdminRoleListItem(object): """ An item in a list of roles. Do not create this directly, use :meth:`~dataikuapi.govern.admin_roles_permissions_handler.GovernAdminRolesPermissionsHandler.list_roles` """ def __init__(self, client, data): self.client = client self._data = data def get_raw(self): """ Get the raw content of the role list item :return: the raw content of the role list item as a dict :rtype: dict """ return self._data def to_role(self): """ Gets the :class:`~dataikuapi.govern.admin_roles_permissions_handler.GovernAdminRole` corresponding to this role object :return: the role object :rtype: a :class:`~dataikuapi.govern.admin_roles_permissions_handler.GovernAdminRole` """ return GovernAdminRole(self.client, self._data["id"]) class GovernAdminRole(object): """ A handle to interact with the roles of the instance as an admin. Do not create this directly, use :meth:`~dataikuapi.govern.admin_roles_permissions_handler.GovernAdminRolesPermissionsHandler.get_role` """ def __init__(self, client, role_id): self.client = client self.role_id = role_id def get_definition(self): """ Return the information of the role as an object :return: the information of the role. :rtype: :class:`~dataikuapi.govern.admin_roles_permissions_handler.GovernAdminRoleDefinition` """ definition = self.client._perform_json("GET", "/admin/role/%s" % self.role_id) return GovernAdminRoleDefinition(self.client, self.role_id, definition) def delete(self): """ Delete the role :return: None """ self.client._perform_empty("DELETE", "/admin/role/%s" % self.role_id) class GovernAdminRoleDefinition(object): """ The definition of a specific role. Do not create this directly, use :meth:`~dataikuapi.govern.admin_roles_permissions_handler.GovernAdminRole.get_definition` """ def __init__(self, client, role_id, definition): self.client = client self.role_id = role_id self.definition = definition def get_raw(self): """ Get raw information of the role. :return: the raw definition of role, as a dict. Modifications made to the returned object are reflected when saving :rtype: dict """ return self.definition def save(self): """ Save this information back to the role :return: None """ self.definition = self.client._perform_json("PUT", "/admin/role/%s" % self.role_id, body=self.definition) class GovernAdminBlueprintRoleAssignmentsListItem(object): """ An item in a list of blueprint role assignments. Do not create this directly, use :meth:`~dataikuapi.govern.admin_roles_permissions_handler.GovernAdminRolesPermissionsHandler.list_role_assignments` """ def __init__(self, client, data): self.client = client self._data = data def get_raw(self): """ Get the raw content of the blueprint role assignments list item :return: the raw content of the blueprint role assignments list item as a dict :rtype: dict """ return self._data def to_blueprint_role_assignments(self): """ Gets the :class:`~dataikuapi.govern.admin_roles_permissions_handler.GovernAdminBlueprintRoleAssignments` corresponding to this blueprint role assignments object :return: the blueprint role assignments object :rtype: a :class:`~dataikuapi.govern.admin_roles_permissions_handler.GovernAdminBlueprintRoleAssignments` """ return GovernAdminBlueprintRoleAssignments(self.client, self._data["blueprintId"]) class GovernAdminBlueprintRoleAssignments(object): """ A handle to interact with the blueprint role assignments for a specific blueprint Do not create this directly, use :meth:`~dataikuapi.govern.admin_roles_permissions_handler.GovernAdminRolesPermissionsHandler.get_role_assignments` """ def __init__(self, client, blueprint_id): self.client = client self.blueprint_id = blueprint_id def get_definition(self): """ Get the role assignments definition. Returns a handle to interact with it. :return: The role assignments for a specific blueprint. :rtype: :class:`~dataikuapi.govern.admin_roles_permissions_handler.GovernAdminBlueprintRoleAssignments` """ definition = self.client._perform_json("GET", "/admin/blueprint-role-assignments/%s" % self.blueprint_id) return GovernAdminBlueprintRoleAssignmentsDefinition(self.client, self.blueprint_id, definition) def delete(self): """ Delete the role assignments for a specific blueprint. :return: None """ self.client._perform_empty("DELETE", "/admin/blueprint-role-assignments/%s" % self.blueprint_id) class GovernAdminBlueprintRoleAssignmentsDefinition(object): """ The role assignments for a specific blueprint. Do not create this class directly, instead use :meth:`~dataikuapi.govern.admin_roles_permissions_handler.GovernAdminBlueprintRoleAssignments.get_definition` """ def __init__(self, client, blueprint_id, definition): self.client = client self.blueprint_id = blueprint_id self.definition = definition def get_raw(self): """ Get the raw content definition of the assignments for this blueprint. This returns a reference to the raw assignments, not a copy, so changes made to the returned object will be reflected when saving. :rtype: dict """ return self.definition def save(self): """ Save this role assignments. :return: None """ self.definition = self.client._perform_json("PUT", "/admin/blueprint-role-assignments/%s" % self.blueprint_id, body=self.definition) class GovernAdminBlueprintPermissionsListItem(object): """ An item in a list of blueprint permissions. Do not create this directly, use :meth:`~dataikuapi.govern.admin_roles_permissions_handler.GovernAdminRolesPermissionsHandler.list_blueprint_permissions` """ def __init__(self, client, data): self.client = client self._data = data def get_raw(self): """ Get the raw content of the blueprint permissions list item :return: the raw content of the blueprint permissions list item as a dict :rtype: dict """ return self._data def to_blueprint_permissions(self): """ Gets the :class:`~dataikuapi.govern.admin_roles_permissions_handler.GovernAdminBlueprintPermissions` corresponding to this blueprint permissions object :return: the blueprint permissions object :rtype: a :class:`~dataikuapi.govern.admin_roles_permissions_handler.GovernAdminBlueprintPermissions` """ return GovernAdminBlueprintPermissions(self.client, self._data["blueprintId"]) class GovernAdminBlueprintPermissions(object): """ A handle to interact with blueprint permissions for a specific blueprint Do not create this directly, use :meth:`~dataikuapi.govern.admin_roles_permissions_handler.GovernAdminRolesPermissionsHandler.get_blueprint_permissions` """ def __init__(self, client, blueprint_id): self.client = client self.blueprint_id = blueprint_id def get_definition(self): """ Get the blueprint permissions definition. Returns a handle to interact with it. :return: The permissions definition for a specific blueprint. :rtype: :class:`~dataikuapi.govern.admin_roles_permissions_handler.GovernAdminBlueprintPermissionsDefinition` """ definition = self.client._perform_json("GET", "/admin/blueprint-permissions/%s" % self.blueprint_id) return GovernAdminBlueprintPermissionsDefinition(self.client, self.blueprint_id, definition) def delete(self): """ Delete the permissions for this blueprint and use default permissions instead. :return: None """ self.client._perform_empty("DELETE", "/admin/blueprint-permissions/%s" % self.blueprint_id) class GovernAdminBlueprintPermissionsDefinition(object): """ The permissions for a specific blueprint. Do not create this class directly, instead use :meth:`~dataikuapi.govern.admin_roles_permissions_handler.GovernAdminBlueprintPermissions.get_definition` """ def __init__(self, client, blueprint_id, definition): self.client = client self.blueprint_id = blueprint_id self.definition = definition def get_raw(self): """ Get the raw content of the permissions for this blueprint. This returns a reference to the raw permissions, not a copy, so changes made to the returned object will be reflected when saving. :rtype: dict """ return self.definition def save(self): """ Save this permission back to the blueprint permission definition. :return: None """ self.definition = self.client._perform_json("PUT", "/admin/blueprint-permissions/%s" % self.blueprint_id, body=self.definition) class GovernAdminDefaultPermissionsDefinition(object): """ The default permissions of the instance Do not create this directly, use :meth:`~dataikuapi.govern.admin_roles_permissions_handler.GovernAdminRolesPermissionsHandler.get_default_permissions_definition` """ def __init__(self, client, definition): self.client = client self.definition = definition def get_raw(self): """ Get the raw content of the default permissions. This returns a reference to the raw permissions, not a copy, so changes made to the returned object will be reflected when saving. :rtype: dict """ return self.definition def save(self): """ Save the default permissions :return: None """ self.definition = self.client._perform_json("PUT", "/admin/blueprint-permissions/default", body=self.definition) dataiku-api-client-python-13.2.2/dataikuapi/govern/artifact.py000066400000000000000000000540051471211575100243560ustar00rootroot00000000000000from dataikuapi.govern.blueprint import GovernBlueprintVersion class GovernArtifact(object): """ A handle to interact with an artifact on the Govern instance. Do not create this directly, use :meth:`~dataikuapi.govern_client.GovernClient.get_artifact` """ def __init__(self, client, artifact_id): self.client = client self.artifact_id = artifact_id def get_definition(self): """ Retrieve the artifact definition and return it as an object. :return: the corresponding artifact definition object :rtype: :class:`~dataikuapi.govern.artifact.GovernArtifactDefinition` """ artifact = self.client._perform_json("GET", "/artifact/%s" % self.artifact_id) return GovernArtifactDefinition(self.client, self.artifact_id, artifact["artifact"]) def list_signoffs(self): """ List all the sign-offs from the different steps of the workflow for this current artifact. :return: the list of sign-offs :rtype: list of :class:`~dataikuapi.govern.artifact.GovernArtifactSignoffListItem` """ signoffs = self.client._perform_json("GET", "/artifact/%s/signoffs" % self.artifact_id) return [GovernArtifactSignoffListItem(self.client, self.artifact_id, signoff) for signoff in signoffs] def get_signoff(self, step_id): """ Get the sign-off for a specific step of the workflow for this current artifact. :param str step_id: id of the step to retrieve the sign-off from :return: the corresponding :class:`~dataikuapi.govern.artifact.GovernArtifactSignoff` """ return GovernArtifactSignoff(self.client, self.artifact_id, step_id) def delete(self): """ Delete the artifact :return: None """ self.client._perform_empty("DELETE", "/artifact/%s" % self.artifact_id) def create_signoff(self, step_id): """ Create a sign-off for the given stepId. The step must be ongoing and must not hold an existing sign-off. Note: the sign-offs of all steps are created automatically when the artifact is created based on the blueprint version sign-off configurations; thus this call is useful when the configuration has been added after the creation of the artifact. :return: the created :class:`~dataikuapi.govern.artifact.GovernArtifactSignoff` """ self.client._perform_json("POST", "/artifact/%s/workflow/step/%s/signoff/create" % (self.artifact_id, step_id)) return GovernArtifactSignoff(self.client, self.artifact_id, step_id) class GovernArtifactDefinition(object): """ The definition of an artifact. Do not create this class directly, instead use :meth:`~dataikuapi.govern.artifact.GovernArtifact.get_definition` """ def __init__(self, client, artifact_id, definition): self.client = client self.artifact_id = artifact_id self.definition = definition def get_blueprint_version(self): """ Retrieve the blueprint version handle of this artifact :return: the blueprint version handle :rtype: :class:`~dataikuapi.govern.blueprint.GovernBlueprintVersion` """ return GovernBlueprintVersion(self.client, self.definition["blueprintVersionId"]["blueprintId"], self.definition["blueprintVersionId"]["versionId"]) def get_raw(self): """ Get the raw content of the artifact. This returns a reference to the artifact so changes made to the returned object will be reflected when saving. :rtype: dict """ return self.definition def save(self): """ Save this settings back to the artifact. """ self.definition = self.client._perform_json("PUT", "/artifact/%s" % self.artifact_id, body=self.definition)["artifact"] class GovernArtifactSignoffListItem(object): """ An item in a list of sign-offs. Do not create this directly, use :meth:`~dataikuapi.govern.artifact.GovernArtifact.list_signoffs` """ def __init__(self, client, artifact_id, data): self.client = client self.artifact_id = artifact_id self._data = data def get_raw(self): """ Get the raw content of the sign-off list item :return: the raw content of the sign-off list item as a dict :rtype: dict """ return self._data def to_signoff(self): """ Gets the :class:`~dataikuapi.govern.artifact.GovernArtifactSignoff` corresponding to this sign-off object :return: the sign-off object :rtype: a :class:`~dataikuapi.govern.artifact.GovernArtifactSignoff` """ return GovernArtifactSignoff(self.client, self.artifact_id, self._data["signoffId"]["stepId"]) class GovernArtifactSignoff(object): """ Handle to interact with the sign-off of a specific workflow step. Do not create this directly, use :meth:`~dataikuapi.govern.artifact.GovernArtifact.get_signoff` """ def __init__(self, client, artifact_id, step_id): self.client = client self.artifact_id = artifact_id self.step_id = step_id def get_definition(self): """ Get the definition of the sign-off for this specific workflow step. :return: the sign-off definition :rtype: :class:`~dataikuapi.govern.artifact.GovernArtifactSignoffDefinition` """ definition = self.client._perform_json("GET", "/artifact/%s/workflow/step/%s/signoff" % (self.artifact_id, self.step_id)) return GovernArtifactSignoffDefinition(self.client, self.artifact_id, self.step_id, definition) def get_recurrence_configuration(self): """ Get the recurrence configuration of the sign-off for this specific workflow step. :return: the sign-off recurrence configuration :rtype: :class:`~dataikuapi.govern.artifact.GovernArtifactSignoffRecurrenceConfiguration` """ definition = self.client._perform_json("GET", "/artifact/%s/workflow/step/%s/signoff" % (self.artifact_id, self.step_id)) return GovernArtifactSignoffRecurrenceConfiguration(self.client, self.artifact_id, self.step_id, definition.get("configuration", {}).get("recurrenceConfiguration", {})) def get_details(self): """ Get the sign-off details for this specific workflow step. This contains a list of computed users included in feedback groups and in the approval. :return: sign-off details :rtype: :class:`~dataikuapi.govern.artifact.GovernArtifactSignoffDetails` """ details = self.client._perform_json("GET", "/artifact/%s/workflow/step/%s/signoff/details" % (self.artifact_id, self.step_id)) return GovernArtifactSignoffDetails(self.client, self.artifact_id, self.step_id, details) def update_status(self, signoff_status, users_to_notify=None, reload_conf_for_reset=False): """ Change the status of the sign-off, takes as input the target status, optionally a list of users to notify and a boolean to indicate if the sign-off configuration should be updated from the blueprint version. Only the users included in the groups of feedback and approval are able to give feedback or approval and can be notified, the complete list is available using: :meth:`~dataikuapi.govern.artifact.GovernArtifactSignoff.get_details`. For the feedback, the users will be notified as part of a chosen group of feedback and the group must be specified. :param str signoff_status: target feedback status to be chosen from: NOT_STARTED, WAITING_FOR_FEEDBACK, WAITING_FOR_APPROVAL, APPROVED, REJECTED, ABANDONED :param users_to_notify: (Optional) List of the user to notify as part of the status change (WAITING_FOR_FEEDBACK will involve the feedback groups, WAITING_FOR_APPROVAL will involve the final approval). The list should be a list of dict containing two keys "userLogin" and "groupId" for each user to notify. The "groupId" key is mandatory for feedback notification and forbidden for the final approval notification. All users that are not in the sign-off configuration will be ignored. :param boolean reload_conf_for_reset: (Optional, defaults to **False**) Usefull only when the target status is NOT_STARTED. If True the current sign-off configuration will be overwritten by the one coming from the blueprint version, all delegated users will be reset. If False the current sign-off configuration will remain the same, allowing all delegated users to be retained but any changes to the sign-off configuration in the blueprint version will not be reflected. :type users_to_notify: list of dict :return: None """ if users_to_notify is None: users_to_notify = [] self.client._perform_json("POST", "/artifact/%s/workflow/step/%s/signoff/update-status" % (self.artifact_id, self.step_id), body={"targetStatus": signoff_status, "usersToSendEmailTo": users_to_notify, "reloadConfForReset": reload_conf_for_reset}) def list_feedbacks(self): """ List all the feedbacks of this current sign-off. :return: the list of feedbacks :rtype: list of :class:`~dataikuapi.govern.artifact.GovernArtifactSignoffFeedbackListItem` """ definition = self.client._perform_json("GET", "/artifact/%s/workflow/step/%s/signoff" % (self.artifact_id, self.step_id)) return [GovernArtifactSignoffFeedbackListItem(self.client, self.artifact_id, self.step_id, feedback["id"], feedback) for feedback in definition.get("feedbackResponses", [])] def get_feedback(self, feedback_id): """ Get a specific feedback review of this sign-off. :param str feedback_id: ID of the feedback review to retrieve from the sign-off :return: the corresponding :class:`~dataikuapi.govern.artifact.GovernArtifactSignoffFeedback` """ return GovernArtifactSignoffFeedback(self.client, self.artifact_id, self.step_id, feedback_id) def add_feedback(self, group_id, feedback_status, comment=None): """ Add a feedback review for a specific feedback group of the sign-off. Takes as input a group_id (the feedback group id), a feedback status and an optional comment. :param str group_id: ID of the feedback group :param str feedback_status: feedback status to be chosen from: APPROVED, MINOR_ISSUE, MAJOR_ISSUE :param str comment: (Optional) feedback comment :rtype: a :class:`~dataikuapi.govern.artifact.GovernArtifactSignoffFeedback` """ body = {"status": feedback_status} if comment is not None: body["comment"] = comment feedback_data = self.client._perform_json("POST", "/artifact/%s/workflow/step/%s/signoff/feedback" % (self.artifact_id, self.step_id), params={"groupId": group_id}, body=body) return GovernArtifactSignoffFeedback(self.client, self.artifact_id, self.step_id, feedback_data["id"]) def delegate_feedback(self, group_id, users_container): """ Delegate the feedback to specific users for the sign-off. Takes as input a group_id (the feedback group that should have done the feedback originally), and an users container definition to delegate to. :param str group_id: ID of the feedback group :param dict users_container: a dict representing the users to delegate to. Use :meth:`~dataikuapi.govern.users_container.GovernUserUsersContainer.build` to build a users container definition for a single user. :return: None """ self.client._perform_json("POST", "/artifact/%s/workflow/step/%s/signoff/delegate-feedback" % (self.artifact_id, self.step_id), params={"groupId": group_id}, body=users_container) def get_approval(self): """ Get the current approval of this sign-off. :return: the corresponding :class:`~dataikuapi.govern.artifact.GovernArtifactSignoffApproval` """ return GovernArtifactSignoffApproval(self.client, self.artifact_id, self.step_id) def add_approval(self, approval_status, comment=None): """ Add the final approval of the sign-off. Takes as input an approval status and an optional comment. :param str approval_status: approval status to be chosen from: APPROVED, REJECTED, ABANDONED :param str comment: (Optional) approval comment :rtype: a :class:`~dataikuapi.govern.artifact.GovernArtifactSignoffApproval` """ body = {"status": approval_status} if comment is not None: body["comment"] = comment self.client._perform_json("POST", "/artifact/%s/workflow/step/%s/signoff/approval" % (self.artifact_id, self.step_id), body=body) return GovernArtifactSignoffApproval(self.client, self.artifact_id, self.step_id) def delegate_approval(self, users_container): """ Delegate the approval to specific users of the sign-off. Takes as input an users container definition to delegate to. :param str users_container: a dict representing the users to delegate to. Use :meth:`~dataikuapi.govern.users_container.GovernUserUsersContainer.build` to build a users container definition for a single user. :return: None """ self.client._perform_json("POST", "/artifact/%s/workflow/step/%s/signoff/delegate-approval" % (self.artifact_id, self.step_id), body=users_container) class GovernArtifactSignoffDefinition(object): """ The definition of a sign-off. Do not create this class directly, instead use :meth:`~dataikuapi.govern.artifact.GovernArtifactSignoff.get_definition` """ def __init__(self, client, artifact_id, step_id, definition): self.client = client self.artifact_id = artifact_id self.step_id = step_id self.definition = definition def get_raw(self): """ Get the raw content of the sign-off definition. :rtype: dict """ return self.definition class GovernArtifactSignoffRecurrenceConfiguration(object): """ The recurrence configuration of a sign-off. Do not create this class directly, instead use :meth:`~dataikuapi.govern.artifact.GovernArtifactSignoff.get_recurrence_configuration` """ def __init__(self, client, artifact_id, step_id, recurrence_configuration): self.client = client self.artifact_id = artifact_id self.step_id = step_id self.recurrence_configuration = recurrence_configuration def get_raw(self): """ Get the raw content of the sign-off recurrence configuration. This returns a reference that can be modified to be saved later. The returned dict can be empty if the recurrence configuration was not already configured. :rtype: dict """ return self.recurrence_configuration def save(self): """ Save the recurrence configuration back to the sign-off. The recurrence configuration must have the following properties: * ``activated`` (boolean) * ``days`` (int) * ``weeks`` (int) * ``months`` (int) * ``years`` (int) * ``reloadConf`` (boolean) :return: None """ signoff = self.client._perform_json("POST", "/artifact/%s/workflow/step/%s/signoff/schedule-reset" % (self.artifact_id, self.step_id), body=self.recurrence_configuration) self.recurrence_configuration = signoff.get("configuration", {}).get("recurrenceConfiguration", {}) class GovernArtifactSignoffDetails(object): """ The details of a sign-off. Do not create this class directly, instead use :meth:`~dataikuapi.govern.artifact.GovernArtifactSignoff.get_details` """ def __init__(self, client, artifact_id, step_id, details): self.client = client self.artifact_id = artifact_id self.step_id = step_id self.details = details def get_raw(self): """ Get the raw content of the sign-off details. :rtype: dict """ return self.details class GovernArtifactSignoffFeedbackListItem(object): """ An item in a list of feedback reviews. Do not create this directly, use :meth:`~dataikuapi.govern.artifact.GovernArtifactSignoff.list_feedbacks` """ def __init__(self, client, artifact_id, step_id, feedback_id, data): self.client = client self.artifact_id = artifact_id self.step_id = step_id self.feedback_id = feedback_id self._data = data def get_raw(self): """ Get the raw content of the feedback review. :rtype: dict """ return self._data def to_feedback(self): """ Gets the :class:`~dataikuapi.govern.artifact.GovernArtifactSignoffFeedback` corresponding to this feedback object :return: the feedback object :rtype: a :class:`~dataikuapi.govern.artifact.GovernArtifactSignoffFeedback` """ return GovernArtifactSignoffFeedback(self.client, self.artifact_id, self.step_id, self.feedback_id) class GovernArtifactSignoffFeedback(object): """ Handle to interact with a feedback. Do not create this directly, use :meth:`~dataikuapi.govern.artifact.GovernArtifactSignoff.get_feedback` """ def __init__(self, client, artifact_id, step_id, feedback_id): self.client = client self.artifact_id = artifact_id self.step_id = step_id self.feedback_id = feedback_id def get_definition(self): """ Get the feedback definition. :return: the feedback definition object :rtype: a :class:`~dataikuapi.govern.artifact.GovernArtifactSignoffFeedbackDefinition` """ definition = self.client._perform_json("GET", "/artifact/%s/workflow/step/%s/signoff" % (self.artifact_id, self.step_id)) feedback_data = next((feedback for feedback in definition.get("feedbackResponses", []) if feedback.get("id") == self.feedback_id), None) if feedback_data is None: raise ValueError("Unable to find a feedback with id '%s'" % (self.feedback_id)) return GovernArtifactSignoffFeedbackDefinition(self.client, self.artifact_id, self.step_id, self.feedback_id, feedback_data) def delete(self): """ Delete this feedback review :return: None """ self.client._perform_empty("DELETE", "/artifact/%s/workflow/step/%s/signoff/feedback/%s" % (self.artifact_id, self.step_id, self.feedback_id)) class GovernArtifactSignoffFeedbackDefinition(object): """ The definition of a feedback review. Do not create this directly, use :meth:`~dataikuapi.govern.artifact.GovernArtifactSignoffFeedback.get_definition` """ def __init__(self, client, artifact_id, step_id, feedback_id, definition): self.client = client self.artifact_id = artifact_id self.step_id = step_id self.feedback_id = feedback_id self.definition = definition def get_raw(self): """ Get the raw content of the feedback definition. This returns a reference to the feedback so changes made to the `status` and `comment` properties will be reflected when saving. :rtype: dict """ return self.definition def save(self): """ Save this feedback review back to the sign-off :return: None """ body = {"status": self.definition.get("status")} comment = self.definition.get("comment") if comment is not None: body["comment"] = comment self.definition = self.client._perform_json("PUT", "/artifact/%s/workflow/step/%s/signoff/feedback/%s" % (self.artifact_id, self.step_id, self.feedback_id), body=body) class GovernArtifactSignoffApproval(object): """ Handle to interact with an approval. Do not create this directly, use :meth:`~dataikuapi.govern.artifact.GovernArtifactSignoff.get_approval` """ def __init__(self, client, artifact_id, step_id): self.client = client self.artifact_id = artifact_id self.step_id = step_id def get_definition(self): """ Get the approval definition. :return: the approval definition object :rtype: a :class:`~dataikuapi.govern.artifact.GovernArtifactSignoffApprovalDefinition` """ definition = self.client._perform_json("GET", "/artifact/%s/workflow/step/%s/signoff" % (self.artifact_id, self.step_id)) if "approverResponse" not in definition or definition.get("approverResponse") is None: raise ValueError("Unable to find the approver response") return GovernArtifactSignoffApprovalDefinition(self.client, self.artifact_id, self.step_id, definition["approverResponse"]) def delete(self): """ Delete this approval. :return: None """ self.client._perform_empty("DELETE", "/artifact/%s/workflow/step/%s/signoff/approval" % (self.artifact_id, self.step_id)) class GovernArtifactSignoffApprovalDefinition(object): """ The definition of an approval. Do not create this directly, use :meth:`~dataikuapi.govern.artifact.GovernArtifactSignoffApproval.get_definition` """ def __init__(self, client, artifact_id, step_id, definition): self.client = client self.artifact_id = artifact_id self.step_id = step_id self.definition = definition def get_raw(self): """ Get the raw content of the approval. This returns a reference to the approval so changes made to the `status` and `comment` properties will be reflected when saving. :rtype: dict """ return self.definition def save(self): """ Save this approval back to the sign-off :return: None """ body = {"status": self.definition.get("status")} comment = self.definition.get("comment") if comment is not None: body["comment"] = comment self.definition = self.client._perform_json("PUT", "/artifact/%s/workflow/step/%s/signoff/approval" % (self.artifact_id, self.step_id), body=body) dataiku-api-client-python-13.2.2/dataikuapi/govern/artifact_search.py000066400000000000000000000411641471211575100257050ustar00rootroot00000000000000from dataikuapi.govern.artifact import GovernArtifact import copy class GovernArtifactSearchRequest(object): """ A search request object. Do not create this directly, use :meth:`~dataikuapi.govern_client.GovernClient.new_artifact_search_request` and then run the query using :meth:`~dataikuapi.govern.artifact_search.GovernArtifactSearchRequest.fetch_next_batch` """ def __init__(self, client, artifact_search_query): self.client = client self.search_query = artifact_search_query.build() self.last_artifact_id = None def fetch_next_batch(self, page_size=20): """ Run the search request fetching the next batch of results. Use page_size to specify the size of the result page. :param int page_size: (Optional) size of the result page, default value is set to 20. :return: The response of a single fetch of the search request :rtype: a :class:`~dataikuapi.govern.artifact_search.GovernArtifactSearchResponse` """ body = copy.deepcopy(self.search_query) body["artifactSearchPagination"] = { "pageSize": page_size, "lastArtifactId": self.last_artifact_id } response = self.client._perform_json("POST", "/artifacts/search", body=body) artifact_list = response.get("uiArtifacts", []) # update local last_artifact_id for next requests if artifact_list: self.last_artifact_id = artifact_list[-1]["artifact"]["id"] return GovernArtifactSearchResponse(self.client, response) class GovernArtifactSearchResponse(object): """ A search request response for a single batch. Do not create this directly, use :meth:`~dataikuapi.govern.artifact_search.GovernArtifactSearchRequest.fetch_next_batch`. """ def __init__(self, client, response): self.client = client self.response = response def get_raw(self): """ Get the raw content of the search response :return: the raw content of the search response as a dict :rtype: dict """ return self.response def get_response_hits(self): """ Get the search response hits (artifacts) :return: list of the search response hits (artifacts) :rtype: list of :class:`~dataikuapi.govern.artifact_search.GovernArtifactSearchResponseHit` """ return [GovernArtifactSearchResponseHit(self.client, hit) for hit in self.response.get("uiArtifacts", [])] class GovernArtifactSearchResponseHit(object): """ A search request response. Do not create this directly, use :meth:`~dataikuapi.govern.artifact_search.GovernArtifactSearchResponse.get_response_hits`. """ def __init__(self, client, hit): self.client = client self.hit = hit def get_raw(self): """ Get the raw content of the search response hit :return: the raw content of the search response hit as a dict :rtype: dict """ return self.hit def to_artifact(self): """ Gets the :class:`~dataikuapi.govern.artifact.GovernArtifact` corresponding to this search response hit :return: the custom page object :rtype: a :class:`~dataikuapi.govern.artifact.GovernArtifact` """ return GovernArtifact(self.client, self.hit["artifact"]["id"]) class GovernArtifactSearchQuery(object): """ A definition of an artifact query. Instantiate and interact with this object to customize the query. :param artifact_search_source: (Optional) The search source to target a subset of artifacts. For now, by default, the search will be executed on all artifacts using the :class:`~dataikuapi.govern.artifact_search.GovernArtifactSearchSourceAll` search source. :type artifact_search_source: :class:`~dataikuapi.govern.artifact_search.GovernArtifactSearchSource` :param artifact_filters: A list of filters to apply on the query. :type artifact_filters: list of :class:`~dataikuapi.govern.artifact_search.GovernArtifactFilter` :param artifact_search_sort: The sort configuration to apply on the query :type artifact_search_sort: :class:`~dataikuapi.govern.artifact_search.GovernArtifactSearchSort` """ def __init__(self, artifact_search_source=None, artifact_filters=None, artifact_search_sort=None): self.artifact_search_source = artifact_search_source if artifact_search_source is not None else GovernArtifactSearchSourceAll() self.artifact_filters = artifact_filters if artifact_filters is not None else [] self.artifact_search_sort = artifact_search_sort def set_artifact_search_source(self, artifact_search_source): """ Set a search source for this query :param artifact_search_source: (Optional) The search source to target a subset of artifacts. For now, by default, the search will be executed on all artifacts using the :class:`~dataikuapi.govern.artifact_search.GovernArtifactSearchSourceAll` search source. :type artifact_search_source: :class:`~dataikuapi.govern.artifact_search.GovernArtifactSearchSource` :return: None """ self.artifact_search_source = artifact_search_source def clear_artifact_filters(self): """ Remove the filters set for this query :return: None """ self.artifact_filters = [] def add_artifact_filter(self, artifact_filter): """ Add a new artifact filter to the filter list of the query. :param artifact_filter: A filter to add to the filter list. :type artifact_filter: :class:`~dataikuapi.govern.artifact_search.GovernArtifactFilter` :return: None """ self.artifact_filters.append(artifact_filter) def clear_artifact_search_sort(self): """ Remove the sort configuration of this query. :return: None """ self.artifact_search_sort = None def set_artifact_search_sort(self, artifact_search_sort): """ Set a new search sort configuration for this request. :param artifact_search_sort: The sort configuration to apply on the query :type artifact_search_sort: :class:`~dataikuapi.govern.artifact_search.GovernArtifactSearchSort` :return: None """ self.artifact_search_sort = artifact_search_sort def build(self): """ :return: the search query definition :rtype: (dict, dict) """ query = { "artifactSearchSource": self.artifact_search_source.build(), "artifactFilters": [artifact_filter.build() for artifact_filter in self.artifact_filters], } if self.artifact_search_sort is not None: query["artifactSearchSort"] = self.artifact_search_sort.build() return query ######################################################## # Search Source ######################################################## class GovernArtifactSearchSource(object): """ An abstract class to represent the different search source. Do not instantiate this class but one of its subclasses. """ def __init__(self, search_source_type): self.search_source_type = search_source_type def build(self): raise NotImplementedError("Cannot build an abstract artifact search source") class GovernArtifactSearchSourceAll(GovernArtifactSearchSource): """ A generic search source definition. """ def __init__(self): super(GovernArtifactSearchSourceAll, self).__init__(search_source_type="all") def build(self): """ :return: the search source definition as a dict :rtype: dict """ return {"type": self.search_source_type} ######################################################## # Search Sort ######################################################## class GovernArtifactSearchSort(object): """ An abstract class to represent the different search sort. Do not instantiate this class but one of its subclasses. """ def __init__(self, artifact_search_sort_type, direction): self.artifact_search_sort_type = artifact_search_sort_type self.direction = direction def build(self): raise NotImplementedError("Cannot build an abstract artifact search sort") class GovernArtifactSearchSortName(GovernArtifactSearchSort): """ An artifact sort definition based on their names. :param str direction: (Optional) The direction on which the artifacts will be sorted. Can be either "ASC" or "DESC" """ def __init__(self, direction="ASC"): super(GovernArtifactSearchSortName, self).__init__(artifact_search_sort_type="name", direction=direction) def build(self): """ :return: the search sort definition as a dict :rtype: dict """ return {"direction": self.direction, "column": {"type": self.artifact_search_sort_type}} class GovernArtifactSearchSortWorkflow(GovernArtifactSearchSort): """ An artifact sort defintion based on their workflow. :param str direction: (Optional) The direction on which the artifacts will be sorted. Can be either "ASC" or "DESC" """ def __init__(self, direction="ASC"): super(GovernArtifactSearchSortWorkflow, self).__init__(artifact_search_sort_type="workflow", direction=direction) def build(self): """ :return: the search sort definition as a dict :rtype: dict """ return {"direction": self.direction, "column": {"type": self.artifact_search_sort_type}} class GovernArtifactSearchSortField(GovernArtifactSearchSort): """ An artifact sort definition based on a list of fields. :param fields: (Optional) A list of fields on which the artifacts will be sorted. Use :meth:`~dataikuapi.govern.artifact_search.GovernArtifactSearchSortFieldDefinition.build` to build a field based sort definition. :type fields: list of dict :param str direction: (Optional) The direction on which the artifacts will be sorted. Can be either "ASC" or "DESC" """ def __init__(self, fields=None, direction="ASC"): super(GovernArtifactSearchSortField, self).__init__(artifact_search_sort_type="field", direction=direction) self.fields = fields if fields is not None else [] def build(self): """ :return: the search sort definition as a dict :rtype: dict """ return {"direction": self.direction, "column": {"type": self.artifact_search_sort_type, "fields": self.fields}} class GovernArtifactSearchSortFieldDefinition(object): """ A field sort definition builder to use in a search query in order to sort on a field of a blueprint. :param str blueprint_id: the Blueprint ID :param str field_id: the field ID """ def __init__(self, blueprint_id, field_id): self.blueprint_id = blueprint_id self.field_id = field_id def build(self): """ :return: the field search sort definition :rtype: dict """ return {"blueprintId": self.blueprint_id, "fieldId": self.field_id} ######################################################## # Search Filters ######################################################## class GovernArtifactFilter(object): """ An abstract class to represent artifact filters. Do not instance this class but one of its subclasses. """ def __init__(self, filter_type): self.filter_type = filter_type def build(self): raise NotImplementedError("Cannot build an abstract artifact filter") class GovernArtifactFilterBlueprints(GovernArtifactFilter): """ An artifact filter definition based on a list of specific blueprints. :param blueprint_ids: (Optional) the list of blueprint IDs to use to filter the artifacts :param blueprint_ids: lsit of str """ def __init__(self, blueprint_ids=None): super(GovernArtifactFilterBlueprints, self).__init__(filter_type="blueprints") self.blueprint_ids = blueprint_ids if blueprint_ids is not None else [] def build(self): """ :return: the artifact filter definition as a dict :rtype: dict """ return {"type": self.filter_type, "blueprintIds": self.blueprint_ids} class GovernArtifactFilterBlueprintVersions(GovernArtifactFilter): """ An artifact filter definition based on a list of specific blueprint versions. :param blueprint_version_ids: (Optional) the list of blueprint version IDs to use to filter the artifacts. A blueprint version ID definition is a dict composed of the blueprint ID and the version ID like the following dict: `{"blueprintId": "bp.my_blueprint", "versionId": "bv.my_version"}`. You can use :meth:`~dataikuapi.govern.blueprint.GovernBlueprintVersionId.build` to build a blueprint version ID definition directly. At the end, the `blueprint_version_ids` parameter expects a value looking like this: `[{"blueprintId": "bp.my_blueprint", "versionId": "bv.my_version"}, {"blueprintId": "bp.my_blueprint", "versionId": "bv.my_version2"}`. :type blueprint_version_ids: list of dict """ def __init__(self, blueprint_version_ids=None): super(GovernArtifactFilterBlueprintVersions, self).__init__(filter_type="blueprintVersions") self.blueprint_version_ids = blueprint_version_ids if blueprint_version_ids is not None else [] def build(self): """ :return: the artifact filter definition as a dict :rtype: dict """ return {"type": self.filter_type, "blueprintVersionIds": self.blueprint_version_ids} class GovernArtifactFilterArtifacts(GovernArtifactFilter): """ An artifact filter definition based on a list of specific artifacts. :param artifact_ids: (Optional) the list of artifacts IDs to use to filter the artifacts. :type artifact_ids: list of str """ def __init__(self, artifact_ids=None): super(GovernArtifactFilterArtifacts, self).__init__(filter_type="artifacts") self.artifact_ids = artifact_ids if artifact_ids is not None else [] def build(self): """ :return: the artifact filter definition as a dict :rtype: dict """ return {"type": self.filter_type, "artifactIds": self.artifact_ids} class GovernArtifactFilterFieldValue(GovernArtifactFilter): """ An artifact filter definition based on specific fields value. :param str condition_type: the condition type of the filter. Has to be chosen from EQUALS, CONTAINS, START_WITH, END_WITH. :param str condition: (Optional) The value on which the condition will be applied. :param str field_id: (Optional) The ID of the field on which the condition will be applied. If not specified the filter will apply on the name. :param boolean negate_condition: (Optional) A boolean to negate the condition. By default, the condition is not negated. :param str case_sensitive: (Optional) Can be used to activate case-sensitive filtering. By default, filters will not be case-sensitive. """ def __init__(self, condition_type, condition=None, field_id=None, negate_condition=None, case_sensitive=None): super(GovernArtifactFilterFieldValue, self).__init__(filter_type="field") self.condition_type = condition_type self.condition = condition self.field_id = field_id self.negate_condition = negate_condition self.case_sensitive = case_sensitive def build(self): """ :return: the artifact filter definition as a dict :rtype: dict """ field_filter = {"type": self.filter_type, "conditionType": self.condition_type} if self.condition is not None: field_filter["condition"] = self.condition if self.field_id is not None: field_filter["fieldId"] = self.field_id if self.negate_condition is not None: field_filter["negateCondition"] = self.negate_condition if self.case_sensitive is not None: field_filter["caseSensitive"] = self.case_sensitive return field_filter class GovernArtifactFilterArchivedStatus(GovernArtifactFilter): """ An artifact filter definition based on the archived status. :param boolean is_archived: the value for filtering. If is_archived is set to True, all artifacts including archived ones will be part of the search result """ def __init__(self, is_archived): super(GovernArtifactFilterArchivedStatus, self).__init__(filter_type="archived") self.is_archived = is_archived def build(self): """ :return: the artifact filter definition as a dict :rtype: dict """ return {"type": self.filter_type, "isArchived": self.is_archived} dataiku-api-client-python-13.2.2/dataikuapi/govern/blueprint.py000066400000000000000000000200041471211575100245550ustar00rootroot00000000000000class GovernBlueprintListItem(object): """ An item in a list of blueprints. Do not create this directly, use :meth:`~dataikuapi.govern_client.GovernClient.list_blueprints` """ def __init__(self, client, data): self.client = client self._data = data def get_raw(self): """ Get the raw content of the blueprint list item :return: the raw content of the blueprint list item as a dict :rtype: dict """ return self._data def to_blueprint(self): """ Gets the :class:`~dataikuapi.govern.blueprint.GovernBlueprint` corresponding to this blueprint object :return: the blueprint object :rtype: a :class:`~dataikuapi.govern.blueprint.GovernBlueprint` """ return GovernBlueprint(self.client, self._data["blueprint"]["id"]) class GovernBlueprint(object): """ A handle to read a blueprint on the Govern instance. If you wish to edit blueprints or the blueprint versions, use the blueprint designer object :class:`~dataikuapi.govern.admin_blueprint_designer.GovernAdminBlueprintDesigner`. Do not create this directly, use :meth:`~dataikuapi.govern_client.GovernClient.get_blueprint` """ def __init__(self, client, blueprint_id): self.client = client self.blueprint_id = blueprint_id def get_definition(self): """ Return the definition of the blueprint as an object. :return: The blueprint definition as an object. :rtype: :class:`~dataikuapi.govern.blueprint.GovernBlueprintDefinition` """ blueprint = self.client._perform_json("GET", "/blueprint/%s" % self.blueprint_id) return GovernBlueprintDefinition(self.client, self.blueprint_id, blueprint["blueprint"]) def list_versions(self): """ List versions of this blueprint. :return: the list of blueprint versions :rtype: list of :class:`~dataikuapi.govern.blueprint.GovernBlueprintVersionListItem` """ versions = self.client._perform_json("GET", "/blueprint/%s/versions" % self.blueprint_id) return [GovernBlueprintVersionListItem(self.client, self.blueprint_id, version) for version in versions] def get_version(self, version_id): """ Return a handle to interact with a blueprint version :param str version_id: ID of the version :rtype: :class:`~dataikuapi.govern.blueprint.GovernBlueprintVersion` """ return GovernBlueprintVersion(self.client, self.blueprint_id, version_id) class GovernBlueprintDefinition(object): """ The definition of a blueprint. Do not create this class directly, instead use :meth:`~dataikuapi.govern.blueprint.GovernBlueprint.get_definition` """ def __init__(self, client, blueprint_id, definition): self.client = client self.blueprint_id = blueprint_id self.definition = definition def get_raw(self): """ Get raw definition of a blueprint :return: The raw definition of blueprint, as a dict. :rtype: dict """ return self.definition class GovernBlueprintVersionListItem(object): """ An item in a list of blueprint versions. Do not create this directly, use :meth:`~dataikuapi.govern.blueprint.GovernBlueprint.list_versions` """ def __init__(self, client, blueprint_id, data): self.client = client self.blueprint_id = blueprint_id self._data = data def get_raw(self): """ Get the raw content of the blueprint version list item :return: the raw content of the blueprint version list item as a dict :rtype: dict """ return self._data def to_blueprint_version(self): """ Gets the :class:`~dataikuapi.govern.blueprint.GovernBlueprintVersion` corresponding to this blueprint version object :return: the blueprint object :rtype: a :class:`~dataikuapi.govern.blueprint.GovernBlueprintVersion` """ return GovernBlueprintVersion(self.client, self.blueprint_id, self._data["blueprintVersion"]["id"]["versionId"]) class GovernBlueprintVersion(object): """ A handle to interact with a blueprint version on the Govern instance. Do not create this directly, use :meth:`~dataikuapi.govern.blueprint.GovernBlueprint.get_version` """ def __init__(self, client, blueprint_id, version_id): self.client = client self.blueprint_id = blueprint_id self.version_id = version_id def get_blueprint(self): """ Retrieve the blueprint handle of this blueprint version. :return: the corresponding blueprint handle :rtype: :class:`~dataikuapi.govern.blueprint.GovernBlueprint` """ return GovernBlueprint(self.client, self.blueprint_id) def get_trace(self): """ Get the trace of this blueprint version (info about its status and origin blueprint version lineage). :return: The trace of this blueprint version. :rtype: :class:`~dataikuapi.govern.blueprint.GovernBlueprintVersionTrace` """ version = self.client._perform_json("GET", "/blueprint/%s/version/%s" % (self.blueprint_id, self.version_id)) return GovernBlueprintVersionTrace(self.client, self.blueprint_id, self.version_id, version["blueprintVersionTrace"]) def get_definition(self): """ Get the definition of this blueprint version. :return: The definition of the blueprint version as an object. :rtype: :class:`~dataikuapi.govern.blueprint.GovernBlueprintVersionDefinition` """ version = self.client._perform_json("GET", "/blueprint/%s/version/%s" % (self.blueprint_id, self.version_id)) return GovernBlueprintVersionDefinition(self.client, self.blueprint_id, self.version_id, version["blueprintVersion"]) class GovernBlueprintVersionTrace(object): """ The trace of a blueprint version containing information about its lineage and its status. Do not create this directly, use :meth:`~dataikuapi.govern.blueprint.GovernBlueprintVersion.get_trace` """ def __init__(self, client, blueprint_id, version_id, trace): self.client = client self.blueprint_id = blueprint_id self.version_id = version_id self.trace = trace def get_raw(self): """ Get raw trace of the blueprint version. :return: The raw trace of blueprint version, as a dict. :rtype: dict """ return self.trace @property def status(self): """ Get the status of the blueprint version among (DRAFT, ACTIVE, or ARCHIVED) :rtype: str """ return self.trace.get("status") @property def origin_version_id(self): """ Get the origin version ID of this blueprint version :rtype: str """ return self.trace.get("originVersionId") class GovernBlueprintVersionDefinition(object): """ The definition of a blueprint version. Do not create this directly, use :meth:`~dataikuapi.govern.blueprint.GovernBlueprintVersion.get_definition` """ def __init__(self, client, blueprint_id, version_id, definition): self.client = client self.blueprint_id = blueprint_id self.version_id = version_id self.definition = definition def get_raw(self): """ Get raw definition of the blueprint version. :return: The raw definition of blueprint version, as a dict. :rtype: dict """ return self.definition class GovernBlueprintVersionId(object): """ A Blueprint Version ID builder :param str blueprint_id: the Blueprint ID :param str version_id: the Version ID """ def __init__(self, blueprint_id, version_id): self.blueprint_id = blueprint_id self.version_id = version_id def build(self): """ :return: the built blueprint version ID definition :rtype: dict """ return {"blueprintId": self.blueprint_id, "versionId": self.version_id} dataiku-api-client-python-13.2.2/dataikuapi/govern/custom_page.py000066400000000000000000000041461471211575100250700ustar00rootroot00000000000000class GovernCustomPageListItem(object): """ An item in a list of custom pages. Do not create this directly, use :meth:`~dataikuapi.govern_client.GovernClient.list_custom_pages` """ def __init__(self, client, data): self.client = client self._data = data def get_raw(self): """ Get the raw content of the custom page list item :return: the raw content of the custom page list item as a dict :rtype: dict """ return self._data def to_custom_page(self): """ Gets the :class:`~dataikuapi.govern.custom_page.GovernCustomPage` corresponding to this custom page object :return: the custom page object :rtype: a :class:`~dataikuapi.govern.custom_page.GovernCustomPage` """ return GovernCustomPage(self.client, self._data["id"]) class GovernCustomPage(object): """ A handle to interact with a custom page. Do not create this directly, use :meth:`~dataikuapi.govern_client.GovernClient.get_custom_page` """ def __init__(self, client, custom_page_id): self.client = client self.custom_page_id = custom_page_id def get_definition(self): """ Get the definition of the custom page. :return: the corresponding custom page definition object :rtype: a :class:`~dataikuapi.govern.custom_page.GovernCustomPageDefinition` """ result = self.client._perform_json("GET", "/custom-page/%s" % self.custom_page_id) return GovernCustomPageDefinition(self.client, self.custom_page_id, result) class GovernCustomPageDefinition(object): """ The definition of a custom page. Do not create this directly, use :meth:`~dataikuapi.govern.custom_page.GovernCustomPage.get_definition` """ def __init__(self, client, custom_page_id, definition): self.client = client self.custom_page_id = custom_page_id self.definition = definition def get_raw(self): """ :return: the raw content of the custom page as a dict :rtype: dict """ return self.definition dataiku-api-client-python-13.2.2/dataikuapi/govern/future.py000066400000000000000000000120211471211575100240630ustar00rootroot00000000000000import time class GovernFuture(object): """ A future represents a long-running task on a Govern instance. It allows you to track the state of the task, retrieve its result when it is ready or abort it. Usage example: .. code-block:: python # In this example, 'client' is a GovernClient future = client.start_resync_users_from_supplier(['login']) # Waits until the user sync is complete sync_result = future.wait_for_result() .. note:: This class does not need to be instantiated directly. A :class:`dataikuapi.govern.future.GovernFuture` is usually returned by the API calls that are initiating long running-tasks. """ def __init__(self, client, job_id, state=None, result_wrapper=lambda result: result): self.client = client self.job_id = job_id self.state = state self.state_is_peek = True self.result_wrapper = result_wrapper @staticmethod def from_resp(client, resp, result_wrapper=lambda result: result): """ Creates a :class:`dataikuapi.govern.future.GovernFuture` from the response of an endpoint that initiated a long-running task. :param client: An api client to connect to the Govern backend :type client: :class:`dataikuapi.govern_client.GovernClient` :param resp: The response of the API call that initiated a long-running task. :type resp: dict :param result_wrapper: (optional) a function to apply to the result of the future, before it is returned by `get_result` or `wait_for_result` methods. :type result_wrapper: callable :return: :class:`dataikuapi.govern.future.GovernFuture` """ return GovernFuture(client, resp.get('jobId', None), state=resp, result_wrapper=result_wrapper) @classmethod def get_result_wait_if_needed(cls, client, ret): """ :meta private: """ if 'jobId' in ret: future = GovernFuture(client, ret["jobId"], ret) future.wait_for_result() return future.get_result() else: return ret['result'] def abort(self): """ Aborts the long-running task. """ self.client._perform_empty("DELETE", "/futures/%s" % self.job_id) def get_state(self): """ Queries the state of the future, and fetches the result if it's ready. :return: the state of the future, including the result if it is ready. :rtype: dict """ self.state = self.client._perform_json( "GET", "/futures/%s" % self.job_id, params={'peek': False}) self.state_is_peek = False return self.state def peek_state(self): """ Queries the state of the future, without fetching the result. :return: the state of the future :rtype: dict """ self.state = self.client._perform_json( "GET", "/futures/%s" % self.job_id, params={'peek': True}) self.state_is_peek = True return self.state def get_result(self): """ Returns the future's result if it's ready. .. note:: if a custom result_wrapper was provided, it will be applied on the result that will be returned. :return: the result of the future :rtype: object :raises Exception: if the result is not ready (i.e. the task is still running, or it has failed) """ if self.state is None or not self.state.get('hasResult', False) or self.state_is_peek: self.get_state() if self.state.get('hasResult', False): return self.result_wrapper(self.state.get('result', None)) else: raise Exception("Result not ready") def has_result(self): """ Checks whether the task is completed successfully and has a result. :return: True if the task has successfully returned a result, False otherwise :rtype: bool """ if self.state is None or not self.state.get('hasResult', False): self.get_state() return self.state.get('hasResult', False) def wait_for_result(self): """ Waits for the completion of the long-running task, and returns its result. .. note:: if a custom result_wrapper was provided, it will be applied on the result that will be returned. :return: the result of the future :rtype: object :raises Exception: if the task failed """ if self.state is not None and self.state.get('hasResult', False): # no future created in backend, result already in the state return self.result_wrapper(self.state.get('result', None)) if self.state is None or not self.state.get('hasResult', False) or self.state_is_peek: self.get_state() while not self.state.get('hasResult', False): time.sleep(5) self.get_state() if self.state.get('hasResult', False): return self.result_wrapper(self.state.get('result', None)) else: raise Exception("No result") dataiku-api-client-python-13.2.2/dataikuapi/govern/time_series.py000066400000000000000000000054451471211575100250750ustar00rootroot00000000000000class GovernTimeSeries(object): """ A handle to interact with a time series. Do not create this directly, use :meth:`~dataikuapi.govern_client.GovernClient.get_time_series` """ def __init__(self, client, time_series_id): self.client = client self.time_series_id = time_series_id def get_values(self, min_timestamp=None, max_timestamp=None): """ Get the values of the time series. Use the parameters `min_timestamp` and `max_timestamp` to define a time window. Only values within this window will be returned :param int min_timestamp: (Optional) The minimum timestamp of the time window as an epoch in milliseconds :param int max_timestamp: (Optional) The maximum timestamp of the time window as an epoch in milliseconds :return: a list data points of the time series as Python dict :rtype: list of dict """ params = {} if min_timestamp is not None: params["timestampMin"] = min_timestamp if max_timestamp is not None: params["timestampMax"] = max_timestamp datapoints = self.client._perform_json( "GET", "/time-series/%s" % self.time_series_id, params=params) return datapoints def push_values(self, datapoints, upsert=True): """ Push a list of values inside the time series. :param list datapoints: a list of Python dict - The list of datapoints as Python dict containing the following keys "timeSeriesId", "timestamp" (an epoch in milliseconds), and "value" (an object) :param boolean upsert: (Optional) If set to false, values for existing timestamps will not be overridden. Default value is True. :return: None """ for datapoint in datapoints: datapoint['timeSeriesId'] = self.time_series_id self.client._perform_json("PUT", "/time-series/%s" % self.time_series_id, body=datapoints, params={"upsert": upsert}) def delete(self, min_timestamp=None, max_timestamp=None): """ Delete the values of the time series. Use the parameters `min_timestamp` and `max_timestamp` to define a time window. Only values within this window will be deleted. :param int min_timestamp: (Optional) The minimum timestamp of the time window as an epoch in milliseconds :param int max_timestamp: (Optional) The maximum timestamp of the time window as an epoch in milliseconds :return: None """ params = {} if min_timestamp is not None: params["timestampMin"] = min_timestamp if max_timestamp is not None: params["timestampMax"] = max_timestamp self.client._perform_empty( "DELETE", "/time-series/%s" % self.time_series_id, params=params) dataiku-api-client-python-13.2.2/dataikuapi/govern/uploaded_file.py000066400000000000000000000021331471211575100253500ustar00rootroot00000000000000class GovernUploadedFile(object): """ A handle to interact with an uploaded file Do not create this directly, use :meth:`~dataikuapi.govern_client.GovernClient.get_uploaded_file` """ def __init__(self, client, uploaded_file_id): self.client = client self.uploaded_file_id = uploaded_file_id def get_description(self): """ Get the description of the uploaded file. :return: The description of the file as a python dict :rtype: dict """ return self.client._perform_json("GET", "/uploaded-file/%s" % self.uploaded_file_id) def download(self): """ Download the uploaded file :return: the uploaded file contents, as a stream - file-like object :rtype: a stream - file-like object """ return self.client._perform_raw("GET", "/uploaded-file/%s/download" % self.uploaded_file_id).raw def delete(self): """ Delete the file :return: None """ return self.client._perform_empty("DELETE", "/uploaded-file/%s" % self.uploaded_file_id) dataiku-api-client-python-13.2.2/dataikuapi/govern/users_container.py000066400000000000000000000014641471211575100257650ustar00rootroot00000000000000class GovernUsersContainer(object): """ An abstract class to represent a users container definition. Do not instance this class but one of its subclasses. """ def __init__(self, type): self.type = type def build(self): raise NotImplementedError("Cannot build an abstract users container") class GovernUserUsersContainer(GovernUsersContainer): """ A users container representing a single user :param str user_login: the user login """ def __init__(self, user_login): super(GovernUserUsersContainer, self).__init__(type="user") self.user_login = user_login def build(self): """ :return: the users container definition as a dict :rtype: dict """ return {"type": self.type, "login": self.user_login} dataiku-api-client-python-13.2.2/dataikuapi/govern_client.py000066400000000000000000000707361471211575100241300ustar00rootroot00000000000000import json from requests import Session, exceptions from requests.auth import HTTPBasicAuth from .iam.settings import DSSSSOSettings, DSSLDAPSettings, DSSAzureADSettings from dataikuapi.govern.future import GovernFuture from dataikuapi.govern.admin import GovernUser, GovernGroup, GovernOwnUser, GovernGlobalApiKey, GovernGeneralSettings, GovernUserActivity, \ GovernAuthorizationMatrix, GovernGlobalApiKeyListItem from dataikuapi.govern.admin_blueprint_designer import GovernAdminBlueprintDesigner from dataikuapi.govern.admin_custom_pages_handler import GovernAdminCustomPagesHandler from dataikuapi.govern.admin_roles_permissions_handler import GovernAdminRolesPermissionsHandler from dataikuapi.govern.artifact import GovernArtifact from dataikuapi.govern.artifact_search import GovernArtifactSearchRequest from dataikuapi.govern.blueprint import GovernBlueprintListItem, GovernBlueprint from dataikuapi.govern.custom_page import GovernCustomPageListItem, GovernCustomPage from dataikuapi.govern.time_series import GovernTimeSeries from dataikuapi.govern.uploaded_file import GovernUploadedFile from dataikuapi.utils import DataikuException class GovernClient(object): """Entry point for the Govern API client""" def __init__(self, host, api_key=None, internal_ticket=None, extra_headers=None, insecure_tls=False): """ Instantiate a new Govern API client on the given host with the given API key. API keys can be managed in Govern in the global settings. The API key will define which operations are allowed for the client. """ self.api_key = api_key self.internal_ticket = internal_ticket self.host = host self._session = Session() if insecure_tls: self._session.verify = False if self.api_key is not None: self._session.auth = HTTPBasicAuth(self.api_key, "") elif self.internal_ticket is not None: self._session.headers.update( {"X-DKU-APITicket": self.internal_ticket}) else: raise ValueError("API Key is required") if extra_headers is not None: self._session.headers.update(extra_headers) ######################################################## # Internal Request handling ######################################################## def _perform_http(self, method, path, params=None, body=None, stream=False, files=None, raw_body=None, headers=None): if body is not None: body = json.dumps(body) if raw_body is not None: body = raw_body try: http_res = self._session.request( method, "%s/dip/publicapi%s" % (self.host, path), params=params, data=body, files=files, stream=stream, headers=headers) http_res.raise_for_status() return http_res except exceptions.HTTPError: try: ex = http_res.json() except ValueError: ex = {"message": http_res.text} raise DataikuException("%s: %s" % (ex.get("errorType", "Unknown error"), ex.get("message", "No message"))) def _perform_empty(self, method, path, params=None, body=None, files=None, raw_body=None): self._perform_http(method, path, params=params, body=body, files=files, stream=False, raw_body=raw_body) def _perform_text(self, method, path, params=None, body=None, files=None, raw_body=None): return self._perform_http(method, path, params=params, body=body, files=files, stream=False, raw_body=raw_body).text def _perform_json(self, method, path, params=None, body=None, files=None, raw_body=None): return self._perform_http(method, path, params=params, body=body, files=files, stream=False, raw_body=raw_body).json() def _perform_raw(self, method, path, params=None, body=None, files=None, raw_body=None): return self._perform_http(method, path, params=params, body=body, files=files, stream=True, raw_body=raw_body) def _perform_json_upload(self, method, path, name, f): try: http_res = self._session.request( method, "%s/dip/publicapi%s" % (self.host, path), files={'file': (name, f, {'Expires': '0'})}) http_res.raise_for_status() return http_res except exceptions.HTTPError: ex = http_res.json() raise DataikuException("%s: %s" % (ex.get("errorType", "Unknown error"), ex.get("message", "No message"))) ######################################################## # Blueprint Designer ######################################################## def get_blueprint_designer(self): """ Return a handle to interact with the blueprint designer Note: this call requires an API key with Govern manager rights :rtype: A :class:`~dataikuapi.govern.admin_blueprint_designer.GovernAdminBlueprintDesigner` """ return GovernAdminBlueprintDesigner(self) ######################################################## # Roles and Permissions ######################################################## def get_roles_permissions_handler(self): """ Return a handler to manage the roles and permissions of the Govern instance Note: this call requires an API key with Govern manager rights :rtype: A :class:`~dataikuapi.govern.admin_roles_permissions_handler.GovernAdminRolesPermissionsHandler` """ return GovernAdminRolesPermissionsHandler(self) ######################################################## # Custom Pages handler ######################################################## def get_custom_pages_handler(self): """ Return a handler to manage custom pages Note: this call requires an API key with Govern manager rights :rtype: A :class:`~dataikuapi.govern.admin_custom_pages_handler.GovernAdminCustomPagesHandler` """ return GovernAdminCustomPagesHandler(self) ######################################################## # Blueprints ######################################################## def list_blueprints(self): """ List all the blueprints :return: a list of blueprints :rtype: list of :class:`~dataikuapi.govern.blueprint.GovernBlueprintListItem` """ blueprint_list = self._perform_json("GET", "/blueprints") return [GovernBlueprintListItem(self, blueprint) for blueprint in blueprint_list] def get_blueprint(self, blueprint_id): """ Get a handle to interact with a blueprint. If you want to edit it or one of its versions, use instead: :meth:`~dataikuapi.govern.admin_blueprint_designer.GovernAdminBlueprintDesigner.get_blueprint` :param str blueprint_id: id of the blueprint to retrieve :returns: The handle of the blueprint :rtype: :class:`~dataikuapi.govern.blueprint.GovernBlueprint` """ return GovernBlueprint(self, blueprint_id) ######################################################## # Artifacts ######################################################## def get_artifact(self, artifact_id): """ Return a handle to interact with an artifact. :param str artifact_id: id of the artifact to retrieve :return: the corresponding :class:`~dataikuapi.govern.artifact.GovernArtifact` """ return GovernArtifact(self, artifact_id) def create_artifact(self, artifact): """ Create an artifact :param dict artifact: the definition of the artifact as a dict :return: the created :class:`~dataikuapi.govern.artifact.GovernArtifact` """ result = self._perform_json("POST", "/artifacts", body=artifact) return GovernArtifact(self, result["artifactId"]) def new_artifact_search_request(self, artifact_search_query): """ Create a new artifact search request and return the object that will be used to launch the requests. :param artifact_search_query: The query that will be addressed during the search. :type artifact_search_query: :class:`~dataikuapi.govern.artifact_search.GovernArtifactSearchQuery` :return: The created artifact search request object :rtype: :class:`~dataikuapi.govern.artifact_search.GovernArtifactSearchRequest` """ return GovernArtifactSearchRequest(self, artifact_search_query) ######################################################## # Custom Pages ######################################################## def get_custom_page(self, custom_page_id): """ Retrieve a custom page. To edit a custom page use instead the custom page editor :meth:`~dataikuapi.govern.admin_custom_pages_handler.GovernAdminCustomPagesHandler.get_custom_page` :param str custom_page_id: id of the custom page to retrieve :return: the corresponding custom page object :rtype: a :class:`~dataikuapi.govern.custom_page.GovernCustomPage` """ return GovernCustomPage(self, custom_page_id) def list_custom_pages(self): """ List custom pages. :return: a list of custom pages :rtype: list of :class:`~dataikuapi.govern.custom_page.GovernCustomPageListItem` """ pages = self._perform_json("GET", "/custom-pages") return [GovernCustomPageListItem(self, page) for page in pages] ######################################################## # Time Series ######################################################## def create_time_series(self, datapoints=None): """ Create a new time series and push a list of values inside it. :param list datapoints: (Optional) a list of Python dict - The list of datapoints as Python dict containing the following keys "timeSeriesId", "timestamp" (an epoch in milliseconds), and "value" (an object) :return: the created time-series object :rtype: a :class:`~dataikuapi.govern.time_series.GovernTimeSeries` """ if datapoints is None: datapoints = [] result = self._perform_json("POST", "/time-series", body=datapoints) return GovernTimeSeries(self, result["id"]) def get_time_series(self, time_series_id): """ Return a handle to interact with the time series :param str time_series_id: ID of the time series :return: the corresponding time series object :rtype: a :class:`~dataikuapi.govern.time_series.GovernTimeSeries` """ return GovernTimeSeries(self, time_series_id) ######################################################## # Uploaded files ######################################################## def get_uploaded_file(self, uploaded_file_id): """ Return a handle to interact with an uploaded file :param str uploaded_file_id: ID of the uploaded file :return: the corresponding uploaded file object :rtype: a :class:`~dataikuapi.govern.uploaded_file.GovernUploadedFile` """ return GovernUploadedFile(self, uploaded_file_id) def upload_file(self, file_name, file): """ Upload a file on Govern. Return a handle to interact with this new uploaded file. :param str file_name: Name of the file :param stream file: file contents, as a stream - file-like object :return: the newly uploaded file object :rtype: a :class:`~dataikuapi.govern.uploaded_file.GovernUploadedFile` """ description = self._perform_json_upload("POST", "/uploaded-files", file_name, file).json() return GovernUploadedFile(self, description["id"]) ######################################################## # Users ######################################################## def list_users(self, as_objects=False): """ List all user setup on the Govern instance Note: this call requires an API key with admin rights :return: A list of users, as a list of :class:`~dataikuapi.govern.admin.GovernUser` if as_objects is True, else as a list of dicts :rtype: list of :class:`~dataikuapi.govern.admin.GovernUser` or list of dicts """ users = self._perform_json("GET", "/admin/users/") if as_objects: return [GovernUser(self, user["login"]) for user in users] else: return users def get_user(self, login): """ Get a handle to interact with a specific user :param str login: the login of the desired user :return: A :class:`~dataikuapi.govern.admin.GovernUser` user handle """ return GovernUser(self, login) def create_user(self, login, password, display_name='', source_type='LOCAL', groups=None, profile='DATA_SCIENTIST'): """ Create a user, and return a handle to interact with it Note: this call requires an API key with admin rights :param str login: the login of the new user :param str password: the password of the new user :param str display_name: the displayed name for the new user :param str source_type: the type of new user. Admissible values are 'LOCAL' or 'LDAP' :param list groups: the names of the groups the new user belongs to (defaults to `[]`) :param str profile: The profile for the new user. Typical values (depend on your license): FULL_DESIGNER, DATA_DESIGNER, AI_CONSUMER, ... :return: A :class:`~dataikuapi.govern.admin.GovernUser` user handle """ if groups is None: groups = [] self._perform_json("POST", '/admin/users/', body={ "login": login, "password": password, "displayName": display_name, "sourceType": source_type, "groups": groups, "userProfile": profile }) return GovernUser(self, login) def get_own_user(self): """ Get a handle to interact with the current user :return: A :class:`~dataikuapi.govern.admin.GovernOwnUser` user handle """ return GovernOwnUser(self) def list_users_activity(self, enabled_users_only=False): """ List all users activity Note: this call requires an API key with admin rights :return: A list of user activity logs, as a list of :class:`dataikuapi.govern.admin.GovernUserActivity` :rtype: list of :class:`dataikuapi.govern.admin.GovernUserActivity` """ params = { "enabledUsersOnly": enabled_users_only } all_activity = self._perform_json("GET", "/admin/users-activity", params=params) return [GovernUserActivity(self, user_activity["login"], user_activity) for user_activity in all_activity] def get_authorization_matrix(self): """ Get the authorization matrix for all enabled users and groups Note: this call requires an API key with admin rights :return: The authorization matrix :rtype: A :class:`dataikuapi.govern.admin.GovernAuthorizationMatrix` authorization matrix handle """ resp = self._perform_json("GET", "/admin/authorization-matrix") return GovernAuthorizationMatrix(resp) def start_resync_users_from_supplier(self, logins): """ Starts a resync of multiple users from an external supplier (LDAP, Azure AD or custom auth) :param list logins: list of logins to resync :return: a :class:`dataikuapi.govern.future.GovernFuture` representing the sync process :rtype: :class:`dataikuapi.govern.future.GovernFuture` """ future_resp = self._perform_json("POST", "/admin/users/actions/resync-multi", body=logins) return GovernFuture.from_resp(self, future_resp) def start_resync_all_users_from_supplier(self): """ Starts a resync of all users from an external supplier (LDAP, Azure AD or custom auth) :return: a :class:`dataikuapi.govern.future.GovernFuture` representing the sync process :rtype: :class:`dataikuapi.govern.future.GovernFuture` """ future_resp = self._perform_json("POST", "/admin/users/actions/resync-multi") return GovernFuture.from_resp(self, future_resp) def start_fetch_external_groups(self, user_source_type): """ Fetch groups from external source :param user_source_type: 'LDAP', 'AZURE_AD' or 'CUSTOM' :rtype: :class:`dataikuapi.govern.future.GovernFuture` :return: a GovernFuture containing a list of group names """ future_resp = self._perform_json("GET", "/admin/external-groups", params={'userSourceType': user_source_type}) return GovernFuture.from_resp(self, future_resp) def start_fetch_external_users(self, user_source_type, login=None, group_name=None): """ Fetch users from external source filtered by login or group name: - if login is provided, will search for a user with an exact match in the external source (e.g. before login remapping) - else, - if group_name is provided, will search for members of the group in the external source - else will search for all users :param user_source_type: 'LDAP', 'AZURE_AD' or 'CUSTOM' :param login: optional - the login of the user in the external source :param group_name: optional - the group name of the group in the external source :rtype: :class:`dataikuapi.govern.future.GovernFuture` :return: a GovernFuture containing a list of ExternalUser """ future_resp = self._perform_json("GET", "/admin/external-users", params={'userSourceType': user_source_type, 'login': login, 'groupName': group_name}) return GovernFuture.from_resp(self, future_resp) def start_provision_users(self, user_source_type, users): """ Provision users of given source type :param string user_source_type: 'LDAP', 'AZURE_AD' or 'CUSTOM' :param list users: list of user attributes coming form the external source :rtype: :class:`dataikuapi.govern.future.GovernFuture` """ future_resp = self._perform_json("POST", "/admin/users/actions/provision", body={'userSourceType': user_source_type, 'users': users}) return GovernFuture.from_resp(self, future_resp) ######################################################## # Groups ######################################################## def list_groups(self): """ List all groups setup on the Govern instance Note: this call requires an API key with admin rights :returns: A list of groups, as a list of dicts :rtype: list of dicts """ return self._perform_json("GET", "/admin/groups/") def get_group(self, name): """ Get a handle to interact with a specific group :param str name: the name of the desired group :returns: A :class:`~dataikuapi.govern.admin.GovernGroup` group handle """ return GovernGroup(self, name) def create_group(self, name, description=None, source_type='LOCAL'): """ Create a group, and return a handle to interact with it Note: this call requires an API key with admin rights :param str name: the name of the new group :param str description: (optional) a description of the new group :param str source_type: the type of the new group. Admissible values are 'LOCAL' and 'LDAP' :returns: A :class:`~dataikuapi.govern.admin.GovernGroup` group handle """ self._perform_text( "POST", "/admin/groups/", body={ "name": name, "description": description, "sourceType": source_type }) return GovernGroup(self, name) ######################################################## # Global API Keys ######################################################## def list_global_api_keys(self, as_type='listitems'): """ List all global API keys set up on the Govern instance .. note:: This call requires an API key with admin rights .. note:: If the secure API keys feature is enabled, the secret key of the listed API keys will not be present in the returned objects :param str as_type: How to return the global API keys. Possible values are "listitems" and "objects" :return: if as_type=listitems, each key as a :class:`dataikuapi.govern.admin.GovernGlobalApiKeyListItem`. if as_type=objects, each key is returned as a :class:`dataikuapi.govern.admin.GovernGlobalApiKey`. """ resp = self._perform_json( "GET", "/admin/global-api-keys/") if as_type == "listitems": return [GovernGlobalApiKeyListItem(self, item) for item in resp] elif as_type == 'objects': return [GovernGlobalApiKey(self, item["key"], item["id"]) for item in resp] else: raise ValueError("Unknown as_type") def get_global_api_key(self, key): """ Get a handle to interact with a specific Global API key .. deprecated:: 13.0.0 Use :meth:`GovernClient.get_global_api_key_by_id`. Calling this method with an invalid secret key will now result in an immediate error. :param str key: the secret key of the API key :returns: A :class:`dataikuapi.govern.admin.GovernGlobalApiKey` API key handle """ resp = self._perform_json( "GET", "/admin/globalAPIKeys/%s" % key) return GovernGlobalApiKey(self, key, resp['id']) def get_global_api_key_by_id(self, id_): """ Get a handle to interact with a specific Global API key :param str id_: the id the API key :returns: A :class:`dataikuapi.govern.admin.GovernGlobalApiKey` API key handle """ resp = self._perform_json( "GET", "/admin/global-api-keys/%s" % id_) return GovernGlobalApiKey(self, resp["key"], id_) def create_global_api_key(self, label=None, description=None, admin=False): """ Create a Global API key, and return a handle to interact with it .. note:: This call requires an API key with admin rights .. note:: The secret key of the created API key will always be present in the returned object, even if the secure API keys feature is enabled :param str label: the label of the new API key :param str description: the description of the new API key :param boolean admin: has the new API key admin rights (True or False) :returns: A :class:`dataikuapi.govern.admin.GovernGlobalApiKey` API key handle """ resp = self._perform_json( "POST", "/admin/global-api-keys/", body={ "label": label, "description": description, "globalPermissions": { "admin": admin } }) if resp is None: raise Exception('API key creation returned no data') if resp.get('messages', {}).get('error', False): raise Exception('API key creation failed : %s' % (json.dumps(resp.get('messages', {}).get('messages', {})))) if not resp.get('id', False): raise Exception('API key creation returned no key') return GovernGlobalApiKey(self, resp.get('key', ''), resp['id']) ######################################################## # Logs ######################################################## def list_logs(self): """ List all available log files on the Govern instance This call requires an API key with admin rights :returns: A list of log file names """ return self._perform_json("GET", "/admin/logs/") def get_log(self, name): """ Get the contents of a specific log file This call requires an API key with admin rights :param str name: the name of the desired log file (obtained with :meth:`list_logs`) :returns: The full content of the log file, as a string """ return self._perform_json("GET", "/admin/logs/%s" % name) def log_custom_audit(self, custom_type, custom_params=None): """ Log a custom entry to the audit trail :param str custom_type: value for customMsgType in audit trail item :param dict custom_params: value for customMsgParams in audit trail item (defaults to `{}`) """ if custom_params is None: custom_params = {} return self._perform_empty("POST", "/admin/audit/custom/%s" % custom_type, body=custom_params) ######################################################## # General settings ######################################################## def get_general_settings(self): """ Gets a handle to interact with the general settings. This call requires an API key with admin rights :returns: a :class:`~dataikuapi.govern.admin.GovernGeneralSettings` handle """ return GovernGeneralSettings(self) ######################################################## # Auth ######################################################## def get_auth_info(self): """ Returns various information about the user currently authenticated using this instance of the API client. This method returns a dict that may contain the following keys (may also contain others): * authIdentifier: login for a user, id for an API key * groups: list of group names (if context is a user) :returns: a dict :rtype: dict """ return self._perform_json("GET", "/auth/info") ######################################################## # Licensing ######################################################## def get_licensing_status(self): """ Returns a dictionary with information about licensing status of this Govern instance :rtype: dict """ return self._perform_json("GET", "/admin/licensing/status") def set_license(self, license): """ Sets a new licence for Govern :param str license: license (content of license file) :return: None """ self._perform_empty("POST", "/admin/licensing/license", body=json.loads(license)) ######################################################## # Global Instance Info ######################################################## def get_instance_info(self): """ Get global information about the Govern instance :returns: a :class:`GovernInstanceInfo` """ resp = self._perform_json("GET", "/instance-info") return GovernInstanceInfo(resp) ######################################################## # IAM ######################################################## def get_sso_settings(self): """ Get the Single Sign-On (SSO) settings :return: SSO settings :rtype: :class:`dataikuapi.iam.settings.SSOSettings` """ sso = self._perform_json("GET", "/admin/iam/sso-settings") return DSSSSOSettings(self, sso) def get_ldap_settings(self): """ Get the LDAP settings :return: LDAP settings :rtype: :class:`dataikuapi.iam.settings.LDAPSettings` """ ldap = self._perform_json("GET", "/admin/iam/ldap-settings") return DSSLDAPSettings(self, ldap) def get_azure_ad_settings(self): """ Get the Azure Active Directory (aka Microsoft Entra ID) settings :return: Azure AD settings :rtype: :class:`dataikuapi.iam.settings.AzureADSettings` """ ldap = self._perform_json("GET", "/admin/iam/azure-ad-settings") return DSSAzureADSettings(self, ldap) class GovernInstanceInfo(object): """Global information about the Govern instance""" def __init__(self, data): """Do not call this directly, use :meth:`GovernClient.get_instance_info`""" self._data = data @property def raw(self): """Returns all data as a Python dictionary""" return self._data @property def node_id(self): """Returns the node id (as defined in Cloud Stacks or in install.ini)""" return self._data["nodeId"] @property def node_name(self): """Returns the node name as it appears in the navigation bar""" return self._data["nodeName"] @property def node_type(self): """ Returns the node type :return: GOVERN """ return self._data["nodeType"] dataiku-api-client-python-13.2.2/dataikuapi/iam/000077500000000000000000000000001471211575100214515ustar00rootroot00000000000000dataiku-api-client-python-13.2.2/dataikuapi/iam/__init__.py000066400000000000000000000001121471211575100235540ustar00rootroot00000000000000from .settings import DSSSSOSettings, DSSLDAPSettings, DSSAzureADSettings dataiku-api-client-python-13.2.2/dataikuapi/iam/settings.py000066400000000000000000001432741471211575100236760ustar00rootroot00000000000000import sys if sys.version_info > (3, 4): from enum import Enum else: class Enum(object): pass class UserRemappingRule(object): def __init__(self, rule_from=None, rule_to=None): """ A class representing a User Remapping Rule :param str rule_from: Rule From :param str rule_to: Rule To """ self.rule_from = rule_from self.rule_to = rule_to class SSOProtocol(Enum): SAML = "SAML" SPNEGO = "SPNEGO" OPENID = "OPENID" class TokenEndpointAuthMethod(Enum): CLIENT_SECRET_BASIC = "CLIENT_SECRET_BASIC" CLIENT_SECRET_POST = "CLIENT_SECRET_POST" class OpenIDParams(dict): def __init__(self, data): """ A class holding OpenID parameters for SSO settings Do not create this directly. :param dict data: Initial OpenID parameters """ super(OpenIDParams, self).__init__(data) def set_client_id(self, client_id): """ Set the OpenID client ID. :param str client_id: OpenID client ID :return: self :rtype: OpenIDParams """ self["clientId"] = client_id return self def set_client_secret(self, client_secret): """ Set the OpenID client secret. :param str client_secret: OpenID client secret :return: self :rtype: OpenIDParams """ self["clientSecret"] = client_secret return self def set_scope(self, scope): """ Set the OpenID scope. :param str scope: OpenID scope :return: self :rtype: OpenIDParams """ self["scope"] = scope return self def set_issuer(self, issuer): """ Set the OpenID issuer. :param str issuer: OpenID issuer :return: self :rtype: OpenIDParams """ self["issuer"] = issuer return self def set_authorization_endpoint(self, authorization_endpoint): """ Set the OpenID authorization endpoint. :param str authorization_endpoint: OpenID authorization endpoint :return: self :rtype: OpenIDParams """ self["authorizationEndpoint"] = authorization_endpoint return self def set_token_endpoint(self, token_endpoint): """ Set the OpenID token endpoint. :param str token_endpoint: OpenID token endpoint :return: self :rtype: OpenIDParams """ self["tokenEndpoint"] = token_endpoint return self def set_jwks_uri(self, jwks_uri): """ Set the OpenID JWKS URI. :param str jwks_uri: OpenID JWKS URI :return: self :rtype: OpenIDParams """ self["jwksUri"] = jwks_uri return self def set_claim_key_identifier(self, claim_key_identifier): """ Set the OpenID claim key for identifier. :param str claim_key_identifier: OpenID claim key for identifier :return: self :rtype: OpenIDParams """ self["claimKeyIdentifier"] = claim_key_identifier return self def set_claim_key_display_name(self, claim_key_display_name): """ Set the OpenID claim key for display name. :param str claim_key_display_name: OpenID claim key for display name :return: self :rtype: OpenIDParams """ self["claimKeyDisplayName"] = claim_key_display_name return self def set_claim_key_email(self, claim_key_email): """ Set the OpenID claim key for email. :param str claim_key_email: OpenID claim key for email :return: self :rtype: OpenIDParams """ self["claimKeyEmail"] = claim_key_email return self def set_enable_groups(self, enable_groups): """ Set whether groups are enabled in OpenID. :param bool enable_groups: True to enable groups, False otherwise :return: self :rtype: OpenIDParams """ self["enableGroups"] = enable_groups return self def set_claim_key_groups(self, claim_key_groups): """ Set the OpenID claim key for groups. :param str claim_key_groups: OpenID claim key for groups :return: self :rtype: OpenIDParams """ self["claimKeyGroups"] = claim_key_groups return self def set_use_global_proxy(self, use_global_proxy): """ Set whether to use the global proxy in OpenID. :param bool use_global_proxy: True to use the global proxy, False otherwise :return: self :rtype: OpenIDParams """ self["useGlobalProxy"] = use_global_proxy return self def set_use_pkce(self, use_pkce): """ Set whether to use PKCE in OpenID. :param bool use_pkce: True to use PKCE, False otherwise :return: self :rtype: OpenIDParams """ self["usePKCE"] = use_pkce return self def set_token_endpoint_auth_method(self, token_endpoint_auth_method): """ Set the token endpoint authentication method. :param TokenEndpointAuthMethod token_endpoint_auth_method: Token endpoint authentication method :return: self :rtype: OpenIDParams """ if not isinstance(token_endpoint_auth_method, TokenEndpointAuthMethod): raise ValueError("Invalid value for token_endpoint_auth_method. Must be one of: CLIENT_SECRET_BASIC, CLIENT_SECRET_POST") self["tokenEndpointAuthMethod"] = token_endpoint_auth_method.value return self def get_client_id(self): """ Get the OpenID client ID. :return: OpenID client ID :rtype: str """ return self.get("clientId") def get_client_secret(self): """ Get the OpenID client secret. :return: OpenID client secret :rtype: str """ return self.get("clientSecret") def get_scope(self): """ Get the OpenID scope. :return: OpenID scope :rtype: str """ return self.get("scope") def get_issuer(self): """ Get the OpenID issuer. :return: OpenID issuer :rtype: str """ return self.get("issuer") def get_authorization_endpoint(self): """ Get the OpenID authorization endpoint. :return: OpenID authorization endpoint :rtype: str """ return self.get("authorizationEndpoint") def get_token_endpoint(self): """ Get the OpenID token endpoint. :return: OpenID token endpoint :rtype: str """ return self.get("tokenEndpoint") def get_jwks_uri(self): """ Get the OpenID JWKS URI. :return: OpenID JWKS URI :rtype: str """ return self.get("jwksUri") def get_claim_key_identifier(self): """ Get the OpenID claim key for identifier. :return: OpenID claim key for identifier :rtype: str """ return self.get("claimKeyIdentifier") def get_claim_key_display_name(self): """ Get the OpenID claim key for display name. :return: OpenID claim key for display name :rtype: str """ return self.get("claimKeyDisplayName") def get_claim_key_email(self): """ Get the OpenID claim key for email. :return: OpenID claim key for email :rtype: str """ return self.get("claimKeyEmail") def get_enable_groups(self): """ Get whether groups are enabled in OpenID. :return: True if groups are enabled, False otherwise :rtype: bool """ return self.get("enableGroups") def get_claim_key_groups(self): """ Get the OpenID claim key for groups. :return: OpenID claim key for groups :rtype: str """ return self.get("claimKeyGroups") def get_use_global_proxy(self): """ Get whether to use the global proxy in OpenID. :return: True if using the global proxy, False otherwise :rtype: bool """ return self.get("useGlobalProxy") def get_use_pkce(self): """ Get whether to use PKCE in OpenID. :return: True if using PKCE, False otherwise :rtype: bool """ return self.get("usePKCE") def get_token_endpoint_auth_method(self): """ Get the token endpoint authentication method. :return: Token endpoint authentication method :rtype: TokenEndpointAuthMethod """ return TokenEndpointAuthMethod(self.get("tokenEndpointAuthMethod")) class SAMLSPParams(dict): def __init__(self, data): """ A class holding SAML Service Provider parameters for SSO settings Do not create this directly. :param dict data: Initial SAML SP parameters """ super(SAMLSPParams, self).__init__(data) def set_sign_requests(self, sign_requests): """ Set whether to sign requests in SAML. :param bool sign_requests: True to sign requests, False otherwise :return: self :rtype: SAMLSPParams """ self["signRequests"] = sign_requests return self def set_display_name_attribute(self, display_name_attribute): """ Set the display name attribute in SAML. :param str display_name_attribute: Display name attribute :return: self :rtype: SAMLSPParams """ self["displayNameAttribute"] = display_name_attribute return self def set_email_attribute(self, email_attribute): """ Set the email attribute in SAML. :param str email_attribute: Email attribute :return: self :rtype: SAMLSPParams """ self["emailAttribute"] = email_attribute return self def set_enable_groups(self, enable_groups): """ Set whether groups are enabled in SAML. :param bool enable_groups: True to enable groups, False otherwise :return: self :rtype: SAMLSPParams """ self["enableGroups"] = enable_groups return self def set_groups_attribute(self, groups_attribute): """ Set the groups attribute in SAML. :param str groups_attribute: Groups attribute :return: self :rtype: SAMLSPParams """ self["groupsAttribute"] = groups_attribute return self def get_sign_requests(self): """ Get whether to sign requests in SAML. :return: True if signing requests, False otherwise :rtype: bool """ return self.get("signRequests") def get_display_name_attribute(self): """ Get the display name attribute in SAML. :return: Display name attribute :rtype: str """ return self.get("displayNameAttribute") def get_email_attribute(self): """ Get the email attribute in SAML. :return: Email attribute :rtype: str """ return self.get("emailAttribute") def get_enable_groups(self): """ Get whether groups are enabled in SAML. :return: True if groups are enabled, False otherwise :rtype: bool """ return self.get("enableGroups") def get_groups_attribute(self): """ Get the groups attribute in SAML. :return: Groups attribute :rtype: str """ return self.get("groupsAttribute") class SPNEGOMode(Enum): PREAUTH_KEYTAB = "PREAUTH_KEYTAB" CUSTOM_LOGIN_CONF = "CUSTOM_LOGIN_CONF" class SSOSettings: def __init__(self, client, path, sso_settings): """ A class holding SSO settings information Do not create this directly. :param client: The client associated with the SSO settings :type client: Client :param path: The endpoints path prefix :param dict sso_settings: The initial SSO settings """ self.client = client self.path = path self.sso_settings = sso_settings self.openid_params_instance = OpenIDParams(self.sso_settings.get("openIDParams", {})) self.saml_sp_params_instance = SAMLSPParams(self.sso_settings.get("samlSPParams", {})) def save(self): """ Saves back the settings to the project """ self.sso_settings["openIDParams"] = dict(self.openid_params_instance) self.sso_settings["samlSPParams"] = dict(self.saml_sp_params_instance) self.client._perform_empty( "PUT", self.path + "/iam/sso-settings", body=self.sso_settings ) def set_protocol(self, protocol): """ Set the SSO protocol. :param SSOProtocol protocol: SSO protocol :return: self :rtype: SSOSettings """ if not isinstance(protocol, SSOProtocol): raise ValueError("Invalid value for protocol. Must be one of: SAML, SPNEGO, OPENID") self.sso_settings["protocol"] = protocol.value return self def get_protocol(self): """ Get the SSO protocol. :return: SSO protocol :rtype: SSOProtocol """ return SSOProtocol(self.sso_settings["protocol"]) def set_remapping_rules(self, remapping_rules): """ Set the remapping rules. :param list remapping_rules: List of UserRemappingRule instances :return: self :rtype: SSOSettings """ # Convert UserRemappingRule instances to dictionaries remapping_rules_data = [{"ruleFrom": rule.rule_from, "ruleTo": rule.rule_to} for rule in remapping_rules] self.sso_settings["remappingRules"] = remapping_rules_data return self def get_remapping_rules(self): """ Get the remapping rules. :return: List of UserRemappingRule instances :rtype: list """ return [UserRemappingRule(rule["ruleFrom"], rule["ruleTo"]) for rule in self.sso_settings.get("remappingRules", [])] def set_saml_sp_params(self, saml_sp_params): """ Set the SAML Service Provider parameters. :param saml_sp_params: Instance of SAMLSPParams :type saml_sp_params: SAMLSPParams :return: self :rtype: SSOSettings """ self.sso_settings["samlSPParams"] = saml_sp_params self.saml_sp_params_instance = SAMLSPParams(self.sso_settings.get("samlSPParams", {})) return self def get_saml_sp_params(self): """ Get the SAML Service Provider parameters. :return: Instance of SAMLSPParams :rtype: SAMLSPParams """ return self.saml_sp_params_instance def set_openid_params(self, openid_params): """ Set the OpenID parameters. :param openid_params: Instance of OpenIDParams :type openid_params: OpenIDParams :return: self :rtype: SSOSettings """ self.sso_settings["openIDParams"] = openid_params self.openid_params_instance = OpenIDParams(self.sso_settings.get("openIDParams", {})) return self def get_openid_params(self): """ Get the OpenID parameters. :return: Instance of OpenIDParams :rtype: OpenIDParams """ return self.openid_params_instance def set_spnego_mode(self, spnego_mode): """ Set the SPNEGO mode. :param SPNEGOMode spnego_mode: SPNEGO mode :return: self :rtype: SSOSettings """ if not isinstance(spnego_mode, SPNEGOMode): raise ValueError("Invalid value for spnego_mode. Must be one of: PREAUTH_KEYTAB, CUSTOM_LOGIN_CONF") self.sso_settings["spnegoMode"] = spnego_mode.value return self def get_spnego_mode(self): """ Get the SPNEGO mode. :return: SPNEGO mode :rtype: SPNEGOMode """ return SPNEGOMode(self.sso_settings["spnegoMode"]) def set_sso_enabled(self, enabled): """ Set whether SSO is enabled. :param bool enabled: True if SSO is enabled, False otherwise :return: self :rtype: SSOSettings """ self.sso_settings["enabled"] = enabled return self def get_sso_enabled(self): """ Get whether SSO is enabled. :return: True if SSO is enabled, False otherwise :rtype: bool """ return self.sso_settings.get("enabled", False) def set_auto_provision_users_at_login_time(self, auto_provision): """ Set whether auto-provisioning of users is enabled at login time. :param bool auto_provision: True to enable auto-provisioning, False otherwise :return: self :rtype: SSOSettings """ self.sso_settings["autoProvisionUsersAtLoginTime"] = auto_provision return self def get_auto_provision_users_at_login_time(self): """ Get whether auto-provisioning of users is enabled at login time. :return: True if auto-provisioning is enabled, False otherwise :rtype: bool """ return self.sso_settings.get("autoProvisionUsersAtLoginTime", False) def set_auto_sync_users_at_login_time(self, auto_sync): """ Set whether auto-syncing of users is enabled at login time. :param bool auto_sync: True to enable auto-syncing, False otherwise :return: self :rtype: SSOSettings """ self.sso_settings["autoSyncUsersAtLoginTime"] = auto_sync return self def get_auto_sync_users_at_login_time(self): """ Get whether auto-syncing of users is enabled at login time. :return: True if auto-syncing is enabled, False otherwise :rtype: bool """ return self.sso_settings.get("autoSyncUsersAtLoginTime", False) def set_default_user_profile(self, default_profile): """ Set the default user profile. :param str default_profile: Default user profile :return: self :rtype: SSOSettings """ self.sso_settings["defaultUserProfile"] = default_profile return self def get_default_user_profile(self): """ Get the default user profile. :return: Default user profile :rtype: str """ return self.sso_settings.get("defaultUserProfile", "") def set_group_profiles(self, group_profiles): """ Set the group profiles. :param list group_profiles: List of group profiles :return: self :rtype: SSOSettings """ self.sso_settings["groupProfiles"] = group_profiles return self def get_group_profiles(self): """ Get the group profiles. :return: List of group profiles :rtype: list """ return self.sso_settings.get("groupProfiles", []) def set_authorized_groups(self, authorized_groups): """ Set the authorized groups. :param list[str] authorized_groups: Authorized groups :return: self :rtype: SSOSettings """ # for retro-compatibility when authorized_group was a comma-separated list in a str if isinstance(authorized_groups, str): authorized_groups = authorized_groups.split(',') self.sso_settings["authorizedGroups"] = authorized_groups return self def get_authorized_groups(self): """ Get the authorized groups. :return: Authorized groups :rtype: list[str] """ return self.sso_settings.get("authorizedGroups", []) class LDAPSettings: def __init__(self, client, path, ldap_settings): """ A class holding LDAP settings information Do not create this directly. :param client: The client associated with the LDAP settings :type client: Client :param path: The endpoints path prefix :param dict ldap_settings: The initial LDAP settings """ self.client = client self.path = path self.ldap_settings = ldap_settings def save(self): """ Saves back the settings """ self.client._perform_empty( "PUT", self.path + "/iam/ldap-settings", body=self.ldap_settings ) def set_url(self, url): """ Set the LDAP server URL. :param str url: The LDAP server URL :return: self :rtype: LDAPSettings """ self.ldap_settings["url"] = url return self def get_url(self): """ Get the LDAP server URL. :return: The LDAP server URL :rtype: str """ return self.ldap_settings.get("url", "") def set_use_tls(self, use_tls): """ Set whether to use TLS in LDAP. :param bool use_tls: True to use TLS, False otherwise :return: self :rtype: LDAPSettings """ self.ldap_settings["useTls"] = use_tls return self def get_use_tls(self): """ Get whether to use TLS in LDAP. :return: True if using TLS, False otherwise :rtype: bool """ return self.ldap_settings.get("useTls", False) def set_bind_dn(self, bind_dn): """ Set the LDAP bind DN. :param str bind_dn: The LDAP bind DN :return: self :rtype: LDAPSettings """ self.ldap_settings["bindDN"] = bind_dn return self def get_bind_dn(self): """ Get the LDAP bind DN. :return: The LDAP bind DN :rtype: str """ return self.ldap_settings.get("bindDN", "") def set_bind_password(self, bind_password): """ Set the LDAP bind password. :param str bind_password: The LDAP bind password :return: self :rtype: LDAPSettings """ self.ldap_settings["bindPassword"] = bind_password return self def get_bind_password(self): """ Get the LDAP bind password. :return: The LDAP bind password :rtype: str """ return self.ldap_settings.get("bindPassword", "") def set_user_filter(self, user_filter): """ Set the LDAP user filter. :param str user_filter: LDAP user filter :return: self :rtype: LDAPSettings """ self.ldap_settings["userFilter"] = user_filter return self def get_user_filter(self): """ Get the LDAP user filter. :return: LDAP user filter :rtype: str """ return self.ldap_settings.get("userFilter", "") def set_display_name_attribute(self, display_name_attribute): """ Set the LDAP display name attribute. :param str display_name_attribute: The LDAP display name attribute :return: self :rtype: LDAPSettings """ self.ldap_settings["displayNameAttribute"] = display_name_attribute return self def get_display_name_attribute(self): """ Get the LDAP display name attribute. :return: The LDAP display name attribute :rtype: str """ return self.ldap_settings.get("displayNameAttribute", "") def set_email_attribute(self, email_attribute): """ Set the LDAP email attribute. :param str email_attribute: The LDAP email attribute :return: self :rtype: LDAPSettings """ self.ldap_settings["emailAttribute"] = email_attribute return self def get_email_attribute(self): """ Get the LDAP email attribute. :return: The LDAP email attribute :rtype: str """ return self.ldap_settings.get("emailAttribute", "") def set_enable_groups(self, enable_groups): """ Set whether to enable groups in LDAP. :param bool enable_groups: True to enable groups, False otherwise :return: self :rtype: LDAPSettings """ self.ldap_settings["enableGroups"] = enable_groups return self def get_enable_groups(self): """ Get whether groups are enabled in LDAP. :return: True if groups are enabled, False otherwise :rtype: bool """ return self.ldap_settings.get("enableGroups", False) def set_group_filter(self, group_filter): """ Set the LDAP group filter. :param str group_filter: LDAP group filter :return: self :rtype: LDAPSettings """ self.ldap_settings["groupFilter"] = group_filter return self def get_group_filter(self): """ Get the LDAP group filter. :return: LDAP group filter :rtype: str """ return self.ldap_settings.get("groupFilter", "") def set_group_name_attribute(self, group_name_attribute): """ Set the LDAP group name attribute. :param str group_name_attribute: The LDAP group name attribute :return: self :rtype: LDAPSettings """ self.ldap_settings["groupNameAttribute"] = group_name_attribute return self def get_group_name_attribute(self): """ Get the LDAP group name attribute. :return: The LDAP group name attribute :rtype: str """ return self.ldap_settings.get("groupNameAttribute", "") def set_authorized_groups(self, authorized_groups): """ Set the authorized groups in LDAP. :param list[str] authorized_groups: Authorized groups :return: self :rtype: LDAPSettings """ # for retro-compatibility when authorized_group was a comma-separated list in a str if isinstance(authorized_groups, str): authorized_groups = authorized_groups.split(',') self.ldap_settings["authorizedGroups"] = authorized_groups return self def get_authorized_groups(self): """ Get the authorized groups in LDAP. :return: Authorized groups :rtype: list[str] """ return self.ldap_settings.get("authorizedGroups", []) def set_enabled(self, enabled): """ Set whether LDAP is enabled. :param bool enabled: True if LDAP is enabled, False otherwise :return: self :rtype: LDAPSettings """ self.ldap_settings["enabled"] = enabled return self def get_enabled(self): """ Get whether LDAP is enabled. :return: True if LDAP is enabled, False otherwise :rtype: bool """ return self.ldap_settings.get("enabled", False) def set_auto_import_users(self, auto_import_users): """ Set whether to auto-import users in LDAP. :param bool auto_import_users: True to auto-import users, False otherwise :return: self :rtype: LDAPSettings """ self.ldap_settings["autoImportUsers"] = auto_import_users return self def get_auto_import_users(self): """ Get whether auto-importing users is enabled in LDAP. :return: True if auto-importing is enabled, False otherwise :rtype: bool """ return self.ldap_settings.get("autoImportUsers", False) def set_auto_sync_users_at_login_time(self, auto_sync_users_at_login_time): """ Set whether auto-syncing of users is enabled at login time in LDAP. :param bool auto_sync_users_at_login_time: True to enable auto-syncing, False otherwise :return: self :rtype: LDAPSettings """ self.ldap_settings["autoSyncUsersAtLoginTime"] = auto_sync_users_at_login_time return self def get_auto_sync_users_at_login_time(self): """ Get whether auto-syncing of users is enabled at login time in LDAP. :return: True if auto-syncing is enabled, False otherwise :rtype: bool """ return self.ldap_settings.get("autoSyncUsersAtLoginTime", False) def set_allow_on_demand_users_sync(self, allow_on_demand_users_sync): """ Set whether to allow on-demand users syncing in LDAP. :param bool allow_on_demand_users_sync: True to allow syncing, False otherwise :return: self :rtype: LDAPSettings """ self.ldap_settings["allowOnDemandUsersSync"] = allow_on_demand_users_sync return self def get_allow_on_demand_users_sync(self): """ Get whether on-demand users syncing is allowed in LDAP. :return: True if syncing is allowed, False otherwise :rtype: bool """ return self.ldap_settings.get("allowOnDemandUsersSync", False) def set_default_user_profile(self, default_user_profile): """ Set the default user profile in LDAP. :param str default_user_profile: Default user profile :return: self :rtype: LDAPSettings """ self.ldap_settings["defaultUserProfile"] = default_user_profile return self def get_default_user_profile(self): """ Get the default user profile in LDAP. :return: Default user profile :rtype: str """ return self.ldap_settings.get("defaultUserProfile", "") def set_group_profiles(self, group_profiles): """ Set the group profiles in LDAP. :param list group_profiles: List of group profiles :return: self :rtype: LDAPSettings """ self.ldap_settings["groupProfiles"] = group_profiles return self def get_group_profiles(self): """ Get the group profiles in LDAP. :return: List of group profiles :rtype: list """ return self.ldap_settings.get("groupProfiles", []) def set_authentication_enabled(self, authentication_enabled): """ Set whether authentication is enabled in LDAP. :param bool authentication_enabled: True if authentication is enabled, False otherwise :return: self :rtype: LDAPSettings """ self.ldap_settings["authenticationEnabled"] = authentication_enabled return self def get_authentication_enabled(self): """ Get whether authentication is enabled in LDAP. :return: True if authentication is enabled, False otherwise :rtype: bool """ return self.ldap_settings.get("authenticationEnabled", False) def set_all_groups_filter(self, all_groups_filter): """ Set the LDAP filter for retrieving all groups. :param str all_groups_filter: The LDAP filter for all groups :return: self :rtype: LDAPSettings """ self.ldap_settings["allGroupsFilter"] = all_groups_filter return self def get_all_groups_filter(self): """ Get the LDAP filter for retrieving all groups. :return: The LDAP filter for all groups :rtype: str """ return self.ldap_settings.get("allGroupsFilter", "") def set_all_users_filter(self, all_users_filter): """ Set the LDAP filter for retrieving all users. :param str all_users_filter: The LDAP filter for all users :return: self :rtype: LDAPSettings """ self.ldap_settings["allUsersFilter"] = all_users_filter return self def get_all_users_filter(self): """ Get the LDAP filter for retrieving all users. :return: The LDAP filter for all users :rtype: str """ return self.ldap_settings.get("allUsersFilter", "") def set_username_attribute(self, username_attribute): """ Set the LDAP username attribute. :param str username_attribute: The LDAP username attribute :return: self :rtype: LDAPSettings """ self.ldap_settings["usernameAttribute"] = username_attribute return self def get_username_attribute(self): """ Get the LDAP username attribute. :return: The LDAP username attribute :rtype: str """ return self.ldap_settings.get("usernameAttribute", "") def set_group_membership_attribute(self, group_membership_attribute): """ Set the LDAP group membership attribute. :param str group_membership_attribute: The LDAP group membership attribute :return: self :rtype: LDAPSettings """ self.ldap_settings["groupMembershipAttribute"] = group_membership_attribute return self def get_group_membership_attribute(self): """ Get the LDAP group membership attribute. :return: The LDAP group membership attribute :rtype: str """ return self.ldap_settings.get("groupMembershipAttribute", "") def set_allow_on_demand_users_provisioning(self, allow_on_demand_users_provisioning): """ Set whether on-demand users provisioning is allowed in LDAP. :param bool allow_on_demand_users_provisioning: True to allow provisioning, False otherwise :return: self :rtype: LDAPSettings """ self.ldap_settings["allowOnDemandUsersProvisioning"] = allow_on_demand_users_provisioning return self def get_allow_on_demand_users_provisioning(self): """ Get whether on-demand users provisioning is allowed in LDAP. :return: True if provisioning is allowed, False otherwise :rtype: bool """ return self.ldap_settings.get("allowOnDemandUsersProvisioning", False) def set_group_membership_user_attribute(self, group_membership_user_attribute): """ Set the LDAP group membership user attribute. :param str group_membership_user_attribute: The LDAP group membership user attribute :return: self :rtype: LDAPSettings """ self.ldap_settings["groupMembershipUserAttribute"] = group_membership_user_attribute return self def get_group_membership_user_attribute(self): """ Get the LDAP group membership user attribute. :return: The LDAP group membership user attribute :rtype: str """ return self.ldap_settings.get("groupMembershipUserAttribute", "") def set_remapping_rules(self, remapping_rules): """ Set the remapping rules in LDAP. :param list remapping_rules: List of remapping rules :return: self :rtype: LDAPSettings """ self.ldap_settings["remappingRules"] = remapping_rules return self def get_remapping_rules(self): """ Get the remapping rules in LDAP. :return: List of remapping rules :rtype: list """ return self.ldap_settings.get("remappingRules", []) class AzureADSettings: def __init__(self, client, path, azuread_settings): """ A class holding Azure AD settings information Do not create this directly. :param client: The client associated with the Azure AD settings :type client: Client :param path: The endpoints path prefix :param dict azuread_settings: The initial Azure AD settings """ self.client = client self.path = path self.azuread_settings = azuread_settings def save(self): """ Saves back the settings """ self.client._perform_empty( "PUT", self.path + "/iam/azure-ad-settings", body=self.azuread_settings ) def set_use_global_proxy(self, use_global_proxy): """ Set whether to use a global proxy for Azure AD. :param bool use_global_proxy: True to use a global proxy, False otherwise :return: self :rtype: AzureADSettings """ self.azuread_settings["useGlobalProxy"] = use_global_proxy return self def get_use_global_proxy(self): """ Get whether to use a global proxy for Azure AD. :return: True if using a global proxy, False otherwise :rtype: bool """ return self.azuread_settings.get("useGlobalProxy", False) def set_credential_method(self, credential_method): """ Set the credential method for Azure AD. :param str credential_method: The credential method (possible values: "OAUTH2_SECRET" or "OAUTH2_CERT") :return: self :rtype: AzureADSettings """ self.azuread_settings["credentialMethod"] = credential_method return self def get_credential_method(self): """ Get the credential method for Azure AD. :return: The credential method :rtype: str """ return self.azuread_settings.get("credentialMethod", "") def set_credentials_client_id(self, credentials_client_id): """ Set the client ID for Azure AD credentials. :param str credentials_client_id: The client ID :return: self :rtype: AzureADSettings """ self.azuread_settings["credentialsClientId"] = credentials_client_id return self def get_credentials_client_id(self): """ Get the client ID for Azure AD credentials. :return: The client ID :rtype: str """ return self.azuread_settings.get("credentialsClientId", "") def set_credentials_client_secret(self, credentials_client_secret): """ Set the client secret for Azure AD credentials. :param str credentials_client_secret: The client secret :return: self :rtype: AzureADSettings """ self.azuread_settings["credentialsClientSecret"] = credentials_client_secret return self def get_credentials_client_secret(self): """ Get the client secret for Azure AD credentials. :return: The client secret :rtype: str """ return self.azuread_settings.get("credentialsClientSecret", "") def set_credentials_tenant_id(self, credentials_tenant_id): """ Set the tenant ID for Azure AD credentials. :param str credentials_tenant_id: The tenant ID :return: self :rtype: AzureADSettings """ self.azuread_settings["credentialsTenantId"] = credentials_tenant_id return self def get_credentials_tenant_id(self): """ Get the tenant ID for Azure AD credentials. :return: The tenant ID :rtype: str """ return self.azuread_settings.get("credentialsTenantId", "") def set_user_query_filter(self, user_query_filter): """ Set the user query filter for Azure AD. :param str user_query_filter: The user query filter :return: self :rtype: AzureADSettings """ self.azuread_settings["userQueryFilter"] = user_query_filter return self def get_user_query_filter(self): """ Get the user query filter for Azure AD. :return: The user query filter :rtype: str """ return self.azuread_settings.get("userQueryFilter", "") def set_group_query_filter(self, group_query_filter): """ Set the group query filter for Azure AD. :param str group_query_filter: The group query filter :return: self :rtype: AzureADSettings """ self.azuread_settings["groupQueryFilter"] = group_query_filter return self def get_group_query_filter(self): """ Get the group query filter for Azure AD. :return: The group query filter :rtype: str """ return self.azuread_settings.get("groupQueryFilter", "") def set_groups_limit(self, groups_limit): """ Set the limit for the number of groups in Azure AD. :param int groups_limit: The groups limit :return: self :rtype: AzureADSettings """ self.azuread_settings["groupsLimit"] = groups_limit return self def get_groups_limit(self): """ Get the limit for the number of groups in Azure AD. :return: The groups limit :rtype: int """ return self.azuread_settings.get("groupsLimit", 0) def set_enabled(self, enabled): """ Set whether Azure AD integration is enabled. :param bool enabled: True if Azure AD integration is enabled, False otherwise :return: self :rtype: AzureADSettings """ self.azuread_settings["enabled"] = enabled return self def get_enabled(self): """ Get whether Azure AD integration is enabled. :return: True if Azure AD integration is enabled, False otherwise :rtype: bool """ return self.azuread_settings.get("enabled", False) def set_auto_provision_users_at_login_time(self, auto_provision_users_at_login_time): """ Set whether auto-provisioning of users is enabled at login time in Azure AD. :param bool auto_provision_users_at_login_time: True to enable auto-provisioning, False otherwise :return: self :rtype: AzureADSettings """ self.azuread_settings["autoProvisionUsersAtLoginTime"] = auto_provision_users_at_login_time return self def get_auto_provision_users_at_login_time(self): """ Get whether auto-provisioning of users is enabled at login time in Azure AD. :return: True if auto-provisioning is enabled, False otherwise :rtype: bool """ return self.azuread_settings.get("autoProvisionUsersAtLoginTime", False) def set_auto_sync_users_at_login_time(self, auto_sync_users_at_login_time): """ Set whether auto-syncing of users is enabled at login time in Azure AD. :param bool auto_sync_users_at_login_time: True to enable auto-syncing, False otherwise :return: self :rtype: AzureADSettings """ self.azuread_settings["autoSyncUsersAtLoginTime"] = auto_sync_users_at_login_time return self def get_auto_sync_users_at_login_time(self): """ Get whether auto-syncing of users is enabled at login time in Azure AD. :return: True if auto-syncing is enabled, False otherwise :rtype: bool """ return self.azuread_settings.get("autoSyncUsersAtLoginTime", False) def set_allow_on_demand_users_sync(self, allow_on_demand_users_sync): """ Set whether to allow on-demand users syncing in Azure AD. :param bool allow_on_demand_users_sync: True to allow syncing, False otherwise :return: self :rtype: AzureADSettings """ self.azuread_settings["allowOnDemandUsersSync"] = allow_on_demand_users_sync return self def get_allow_on_demand_users_sync(self): """ Get whether on-demand users syncing is allowed in Azure AD. :return: True if syncing is allowed, False otherwise :rtype: bool """ return self.azuread_settings.get("allowOnDemandUsersSync", False) def set_default_user_profile(self, default_user_profile): """ Set the default user profile in Azure AD. :param str default_user_profile: Default user profile :return: self :rtype: AzureADSettings """ self.azuread_settings["defaultUserProfile"] = default_user_profile return self def get_default_user_profile(self): """ Get the default user profile in Azure AD. :return: Default user profile :rtype: str """ return self.azuread_settings.get("defaultUserProfile", "") def set_group_profiles(self, group_profiles): """ Set the group profiles in Azure AD. :param list group_profiles: List of group profiles :return: self :rtype: AzureADSettings """ self.azuread_settings["groupProfiles"] = group_profiles return self def get_group_profiles(self): """ Get the group profiles in Azure AD. :return: List of group profiles :rtype: list """ return self.azuread_settings.get("groupProfiles", []) def set_authorized_groups(self, authorized_groups): """ Set the authorized groups in Azure AD. :param list[str] authorized_groups: Authorized groups :return: self :rtype: AzureADSettings """ # for retro-compatibility when authorized_group was a comma-separated list in a str if isinstance(authorized_groups, str): authorized_groups = authorized_groups.split(',') self.azuread_settings["authorizedGroups"] = authorized_groups return self def get_authorized_groups(self): """ Get the authorized groups in Azure AD. :return: Authorized groups :rtype: list[str] """ return self.azuread_settings.get("authorizedGroups", []) def set_allow_on_demand_users_provisioning(self, allow_on_demand_users_provisioning): """ Set whether to allow on-demand users provisioning in Azure AD. :param bool allow_on_demand_users_provisioning: True to allow provisioning, False otherwise :return: self :rtype: AzureADSettings """ self.azuread_settings["allowOnDemandUsersProvisioning"] = allow_on_demand_users_provisioning return self def get_allow_on_demand_users_provisioning(self): """ Get whether on-demand users provisioning is allowed in Azure AD. :return: True if provisioning is allowed, False otherwise :rtype: bool """ return self.azuread_settings.get("allowOnDemandUsersProvisioning", False) def set_login_attribute_ref(self, login_attribute_ref): """ Set the login attribute reference in Azure AD. :param str login_attribute_ref: The login attribute reference :return: self :rtype: AzureADSettings """ self.azuread_settings["loginAttributeRef"] = login_attribute_ref return self def get_login_attribute_ref(self): """ Get the login attribute reference in Azure AD. :return: The login attribute reference :rtype: str """ return self.azuread_settings.get("loginAttributeRef", "") def set_remapping_rules(self, remapping_rules): """ Set the remapping rules in Azure AD. :param list remapping_rules: List of remapping rules :return: self :rtype: AzureADSettings """ self.azuread_settings["remappingRules"] = remapping_rules return self def get_remapping_rules(self): """ Get the remapping rules in Azure AD. :return: List of remapping rules :rtype: list """ return self.azuread_settings.get("remappingRules", []) class FMSSOSettings(SSOSettings): def __init__(self, client, sso_settings): """ A class holding FM SSO settings information, inheriting from SSOSettings Do not create this directly. :param client: The client associated with the FM SSO settings :type client: Client :param dict sso_settings: The initial FM SSO settings """ # Call the constructor of the parent class (SSOSettings) path = "/tenants/%s" % str(client.get_tenant_id()) super().__init__(client, path, sso_settings) class DSSSSOSettings(SSOSettings): def __init__(self, client, sso_settings): """ A class holding DSS SSO settings information, inheriting from SSOSettings Do not create this directly. :param client: The client associated with the DSS SSO settings :type client: Client :param dict sso_settings: The initial DSS SSO settings """ # Call the constructor of the parent class (SSOSettings) super().__init__(client, "/admin/", sso_settings) class FMLDAPSettings(LDAPSettings): def __init__(self, client, ldap_settings): """ A class holding FM LDAP settings information, inheriting from LDAPSettings Do not create this directly. :param client: The client associated with the FM LDAP settings :type client: Client :param dict ldap_settings: The initial FM LDAP settings """ # Call the constructor of the parent class (LDAPSettings) path = "/tenants/%s" % str(client.get_tenant_id()) super().__init__(client, path, ldap_settings) class DSSLDAPSettings(LDAPSettings): def __init__(self, client, ldap_settings): """ A class holding DSS LDAP settings information, inheriting from LDAPSettings Do not create this directly. :param client: The client associated with the DSS LDAP settings :type client: Client :param dict ldap_settings: The initial DSS LDAP settings """ # Call the constructor of the parent class (LDAPSettings) super().__init__(client, "/admin/", ldap_settings) class FMAzureADSettings(AzureADSettings): def __init__(self, client, azure_ad_settings): """ A class holding FM Azure AD settings information, inheriting from AzureADSettings Do not create this directly. :param client: The client associated with the FM Azure AD settings :type client: Client :param dict azure_ad_settings: The initial FM Azure AD settings """ # Call the constructor of the parent class (AzureADSettings) path = "/tenants/%s" % str(client.get_tenant_id()) super().__init__(client, path, azure_ad_settings) class DSSAzureADSettings(AzureADSettings): def __init__(self, client, azure_ad_settings): """ A class holding DSS Azure AD settings information, inheriting from AzureADSettings Do not create this directly. :param client: The client associated with the DSS Azure AD settings :type client: Client :param dict azure_ad_settings: The initial DSS Azure AD settings """ # Call the constructor of the parent class (AzureADSettings) super().__init__(client, "/admin/", azure_ad_settings)dataiku-api-client-python-13.2.2/dataikuapi/utils.py000066400000000000000000000115731471211575100224240ustar00rootroot00000000000000import csv, sys from dateutil import parser as date_iso_parser from contextlib import closing import os import zipfile import itertools import sys from datetime import datetime if sys.version_info > (3,0): import codecs dku_basestring_type = str dku_zip_longest = itertools.zip_longest else: dku_basestring_type = basestring dku_zip_longest = itertools.izip_longest class DataikuException(Exception): """Exception launched by the Dataiku API clients when an error occurs""" class DataikuUTF8CSVReader(object): """ A CSV reader which will iterate over lines in the CSV file "f", which is encoded in UTF-8. """ def __init__(self, f, **kwds): self.reader = csv.reader(f, **kwds) def next(self): row = self.reader.next() return [unicode(s, "utf-8") for s in row] def __iter__(self): return self def none_if_throws(f): def aux(*args, **kargs): try: return f(*args, **kargs) except: return None return aux class DataikuValueCaster(object): def __init__(self, schema): self.casters = self._get_value_casters(schema) def _get_value_casters(self, schema): def decode(x): if sys.version_info > (3,0): return x else: return unicode(x, "utf8") def parse_iso_date(s): if s == "": return None else: return date_iso_parser.parse(s) def str_to_bool(s): if s is None: return False return s.lower() == "true" CASTERS = { "tinyint" : int, "smallint" : int, "int": int, "bigint": int, "float": float, "double": float, "date": parse_iso_date, "boolean": str_to_bool, } return [CASTERS.get(col["type"], decode) for col in schema] def cast_values(self, values): return [none_if_throws(caster)(val) for (caster, val) in dku_zip_longest(self.casters, values)] class DataikuStreamedHttpUTF8CSVReader(object): """ A CSV reader with a schema To verify the stream after all rows have been yielded, you must pass **ALL** the optional arguments: read_session_id, client, project_key, dataset_name """ def __init__(self, schema, csv_stream, read_session_id=None, client=None, project_key=None, dataset_name=None): self.schema = schema self.csv_stream = csv_stream # To verify a dataset streaming session self.read_session_id = read_session_id self.project_key = project_key self.dataset_name = dataset_name self.client = client def iter_rows(self): schema = self.schema value_caster = DataikuValueCaster(schema) with closing(self.csv_stream) as r: if sys.version_info > (3,0): raw_generator = codecs.iterdecode(r.raw, 'utf-8') else: raw_generator = r.raw for uncasted_tuple in csv.reader(raw_generator, delimiter='\t', quotechar='"', doublequote=True): yield value_caster.cast_values(uncasted_tuple) if self.read_session_id: # exception will be raised if there's an error while streaming self.client._perform_empty( "GET", "/projects/%s/datasets/%s/finish-streaming/" % (self.project_key, self.dataset_name), params = { "readSessionId" : self.read_session_id }) class CallableStr(str): def __init__(self, val): self.val = val def __call__(self): return self.val def _make_zipfile(output_filename, source_dir): relroot = os.path.abspath(os.path.join(source_dir)) with zipfile.ZipFile(output_filename, "w", zipfile.ZIP_DEFLATED) as zipfp: for root, dirs, files in os.walk(source_dir): for file in files: filename = os.path.join(root, file) if os.path.isfile(filename): arcname = os.path.join(os.path.relpath(root, relroot), file) zipfp.write(filename, arcname) return output_filename def _write_response_content_to_file(response, path): with open(path, 'wb') as f: for chunk in response.iter_content(chunk_size=10000): if chunk: f.write(chunk) f.flush() if sys.version_info >= (3,3): _local_timezone = datetime.now().astimezone().tzinfo else: _local_timezone = None def _timestamp_ms_to_zoned_datetime(timestamp_ms): if timestamp_ms and timestamp_ms > 0: return datetime.fromtimestamp(timestamp_ms / 1000, tz=_local_timezone) else: return Nonedataiku-api-client-python-13.2.2/meta/000077500000000000000000000000001471211575100175155ustar00rootroot00000000000000dataiku-api-client-python-13.2.2/meta/main.yml000066400000000000000000000003211471211575100211600ustar00rootroot00000000000000--- galaxy_info: author: Dataiku description: Dataiku API Client company: Dataiku license: Apache 2.0 min_ansible_version: 2.5 galaxy_tags: ["dataiku","dss","data-science-studio"] dependencies: [] dataiku-api-client-python-13.2.2/module_utils/000077500000000000000000000000001471211575100212745ustar00rootroot00000000000000dataiku-api-client-python-13.2.2/module_utils/dataikuapi000077700000000000000000000000001471211575100256002../dataikuapiustar00rootroot00000000000000dataiku-api-client-python-13.2.2/requirements.txt000066400000000000000000000000331471211575100220470ustar00rootroot00000000000000requests<3 python-dateutil dataiku-api-client-python-13.2.2/setup.py000066400000000000000000000021611471211575100203010ustar00rootroot00000000000000#!/usr/bin/env python from setuptools import setup VERSION = "13.2.2" long_description = (open('README').read() + '\n\n' + open('HISTORY.txt').read()) setup( name='dataiku-api-client', version=VERSION, license="Apache Software License", packages=["dataikuapi", "dataikuapi.dss", "dataikuapi.apinode_admin", "dataikuapi.fm", "dataikuapi.iam", "dataikuapi.govern", "dataikuapi.dss_plugin_mlflow"], description="Python API client for Dataiku APIs", long_description=long_description, author="Dataiku", author_email="support@dataiku.com", url="https://www.dataiku.com", download_url="https://github.com/dataiku/dataiku-api-client-python/tarball/" + VERSION, classifiers=[ 'Development Status :: 5 - Production/Stable', 'Intended Audience :: Developers', 'License :: OSI Approved :: Apache Software License', 'Topic :: Software Development :: Libraries', 'Programming Language :: Python', 'Operating System :: OS Independent' ], install_requires=[ "requests<3", "python-dateutil" ] )