pax_global_header00006660000000000000000000000064144046473640014526gustar00rootroot0000000000000052 comment=b648cddcb705bd74084bace3f27e514870d6b19a dataiku-api-client-python-11.4.0/000077500000000000000000000000001440464736400165755ustar00rootroot00000000000000dataiku-api-client-python-11.4.0/.gitignore000066400000000000000000000000321440464736400205600ustar00rootroot00000000000000*.pyc .idea *.iml egg-infodataiku-api-client-python-11.4.0/.travis.yml000066400000000000000000000001521440464736400207040ustar00rootroot00000000000000language: python python: - "2.7" - "3.4" install: "pip install -r requirements.txt" script: nosetests dataiku-api-client-python-11.4.0/HISTORY.txt000066400000000000000000000020501440464736400204740ustar00rootroot00000000000000Changelog ========== 11.1.0 (2022-10-04) ------------------- * Initial release for DSS 11.1.0 11.0.0 (2022-07-12) ------------------- * Initial release for DSS 11.0.0 9.0.0 (2021-05-07) ------------------ * Initial release for DSS 9.0 8.0.0 (2020-08-14) ------------------- * Initial release for DSS 8.0 7.0.0 (2020-07-21) ------------------ * Initial release for DSS 7.0 5.1.0 (2019-03-01) ------------------ * Initial release for DSS 5.1 5.0.0 (2018-07-26) ------------------ * Initial release for DSS 5.0 4.3.0 (2018-06-01) ------------------ * Initial release for DSS 4.3 4.2.1 (2018-04-30) ------------------- * Initial release for DSS 4.2 4.1.0 (2017-11-10) ------------------- * Initial release for DSS 4.1 4.0.0 (2017-01-10) ------------------ * Initial release for DSS 4.0 3.1.4 (2017-01-03) ------------------- * Pypi packaging fixes 3.1.3 (2017-01-03) ------------------- * Fix user creation 3.1.2 (2016-08-23) ------------------- * Add proper Pypi tags 3.1.0 (2016-08-22) ------------------- * Initial release for DSS 3.1 dataiku-api-client-python-11.4.0/LICENSE.txt000066400000000000000000000010551440464736400204210ustar00rootroot00000000000000Copyright 2015-2021 Dataiku SAS Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. dataiku-api-client-python-11.4.0/MANIFEST.in000066400000000000000000000000721440464736400203320ustar00rootroot00000000000000include README include *.txt recursive-include dataikuapi dataiku-api-client-python-11.4.0/README000066400000000000000000000002171440464736400174550ustar00rootroot00000000000000API client for Dataiku Data Science Studio For more information, see: https://doc.dataiku.com/dss/latest/python-api/rest-api-client/index.htmldataiku-api-client-python-11.4.0/dataikuapi/000077500000000000000000000000001440464736400207115ustar00rootroot00000000000000dataiku-api-client-python-11.4.0/dataikuapi/__init__.py000066400000000000000000000012321440464736400230200ustar00rootroot00000000000000from .dssclient import DSSClient from .fmclient import FMClientAWS, FMClientAzure, FMClientGCP from .govern_client import GovernClient from .apinode_client import APINodeClient from .apinode_admin_client import APINodeAdminClient from .dss.recipe import GroupingRecipeCreator, JoinRecipeCreator, StackRecipeCreator, WindowRecipeCreator, SyncRecipeCreator, SamplingRecipeCreator, SQLQueryRecipeCreator, CodeRecipeCreator, SplitRecipeCreator, SortRecipeCreator, TopNRecipeCreator, DistinctRecipeCreator, DownloadRecipeCreator, PredictionScoringRecipeCreator, ClusteringScoringRecipeCreator from .dss.admin import DSSUserImpersonationRule, DSSGroupImpersonationRule dataiku-api-client-python-11.4.0/dataikuapi/apinode_admin/000077500000000000000000000000001440464736400235005ustar00rootroot00000000000000dataiku-api-client-python-11.4.0/dataikuapi/apinode_admin/__init__.py000066400000000000000000000000001440464736400255770ustar00rootroot00000000000000dataiku-api-client-python-11.4.0/dataikuapi/apinode_admin/auth.py000066400000000000000000000013131440464736400250110ustar00rootroot00000000000000 class APINodeAuth(object): """ A handle to interact with authentication settings on API node """ def __init__(self, client): self.client = client def list_keys(self): """Lists the Admin API keys""" return self.client._perform_json("GET", "keys") def add_key(self, label=None, description=None, created_by=None): """Add an Admin API key. Returns the key details""" key = { "label" : label, "description" : description, "createdBy" : created_by } return self.client._perform_json("POST", "keys", body=key) def delete_key(self, key): self.client._perform_empty("DELETE", "keys/%s" % key)dataiku-api-client-python-11.4.0/dataikuapi/apinode_admin/service.py000066400000000000000000000041751440464736400255210ustar00rootroot00000000000000from os import path as osp class APINodeService(object): """ A handle to interact with the settings of an API node service """ def __init__(self, client, service_id): self.client = client self.service_id = service_id def delete(self): """Deletes the API node service""" self.client._perform_empty("DELETE", "services/%s" % self.service_id) ######################################################## # On-disk generations management ######################################################## def list_generations(self): return self.client._perform_json("GET", "services/%s/generations" % self.service_id)["generations"] def import_generation_from_archive(self, file_path): self.client._perform_empty("POST", "services/%s/generations/actions/importFromArchive" % (self.service_id), params = { "filePath" : osp.abspath(file_path) }) def preload_generation(self, generation): self.client._perform_empty("POST", "services/%s/generations/%s/preload" % (self.service_id, generation)) ######################################################## # Switch / mapping management ######################################################## def disable(self): """Disable the service.""" self.client._perform_empty("POST", "services/%s/actions/disable" % self.service_id) def enable(self): self.client._perform_empty("POST", "services/%s/actions/enable" % self.service_id) def set_generations_mapping(self, mapping): """Setting a generations mapping automatically enables the service""" self.client._perform_empty("POST", "services/%s/actions/setMapping" % self.service_id, body = mapping) def switch_to_newest(self): return self.client._perform_json("POST", "services/%s/actions/switchToNewest" % self.service_id) def switch_to_single_generation(self, generation): self.client._perform_empty("POST", "services/%s/actions/switchTo/%s" % (self.service_id, generation)) dataiku-api-client-python-11.4.0/dataikuapi/apinode_admin_client.py000066400000000000000000000026271440464736400254170ustar00rootroot00000000000000from .apinode_admin.service import APINodeService from .apinode_admin.auth import APINodeAuth from .utils import DataikuException from .base_client import DSSBaseClient class APINodeAdminClient(DSSBaseClient): """Entry point for the DSS APINode admin client""" def __init__(self, uri, api_key): """ Instantiate a new DSS API client on the given base uri with the given API key. """ DSSBaseClient.__init__(self, "%s/%s" % (uri, "admin/api"), api_key) ######################################################## # Services generations ######################################################## def create_service(self, service_id): self._perform_empty("POST", "services/", body = { "serviceId" : service_id }) def list_services(self): return self._perform_json("GET", "services") def service(self, service_id): """ Gets a handle to interact with a service """ return APINodeService(self, service_id) def auth(self): """Returns a handle to interact with authentication""" return APINodeAuth(self) def get_metrics(self): return self._perform_json("GET", "metrics") def import_code_env_in_cache(self, file_dir, language): self._perform_empty("POST", "cached-code-envs", params={ "fileDir": file_dir, "language": language }) dataiku-api-client-python-11.4.0/dataikuapi/apinode_client.py000066400000000000000000000246001440464736400242420ustar00rootroot00000000000000from .utils import DataikuException from .base_client import DSSBaseClient class APINodeClient(DSSBaseClient): """Entry point for the DSS API Node client This is an API client for the user-facing API of DSS API Node server (user facing API) """ def __init__(self, uri, service_id, api_key=None, bearer_token=None): """ Instantiate a new DSS API client on the given base URI with the given API key. :param str uri: Base URI of the DSS API node server (http://host:port/ or https://host:port/) :param str service_id: Identifier of the service to query :param str api_key: Optional, API key for the service. Only required if the service has its authorization setup to API keys :param str bearer_token: Optional, The bearer token. Only required if the service has its authorization setup to OAuth2/JWT """ DSSBaseClient.__init__(self, "%s/%s" % (uri, "public/api/v1/%s" % service_id), api_key=api_key, bearer_token=bearer_token) def predict_record(self, endpoint_id, features, forced_generation=None, dispatch_key=None, context=None, with_explanations=None, explanation_method=None, n_explanations=None, n_explanations_mc_steps=None): """ Predicts a single record on a DSS API node endpoint (standard or custom prediction) :param str endpoint_id: Identifier of the endpoint to query :param features: Python dictionary of features of the record :param forced_generation: See documentation about multi-version prediction :param dispatch_key: See documentation about multi-version prediction :param context: Optional, Python dictionary of additional context information. The context information is logged, but not directly used. :param with_explanations: Optional, whether individual explanations should be computed for each record. The prediction endpoint must be compatible. If None, will use the value configured in the endpoint. :param explanation_method: Optional, method to compute explanations. Valid values are 'SHAPLEY' or 'ICE'. If None, will use the value configured in the endpoint. :param n_explanations: Optional, number of explanations to output per prediction. If None, will use the value configured in the endpoint. :param n_explanations_mc_steps: Optional, precision parameter for SHAPLEY method, higher means more precise but slower (between 25 and 1000). If None, will use the value configured in the endpoint. :return: a Python dict of the API answer. The answer contains a "result" key (itself a dict) """ obj = { "features": features, "explanations": { "enabled": with_explanations, "method": explanation_method, "nExplanations": n_explanations, "nMonteCarloSteps": n_explanations_mc_steps } } if context is not None: obj["context"] = context if forced_generation is not None: obj["dispatch"] = {"forcedGeneration" : forced_generation } elif dispatch_key is not None: obj["dispatch"] = {"dispatchKey" : dispatch_key } return self._perform_json("POST", "%s/predict" % endpoint_id, body = obj) def predict_records(self, endpoint_id, records, forced_generation=None, dispatch_key=None, with_explanations=None, explanation_method=None, n_explanations=None, n_explanations_mc_steps=None): """ Predicts a batch of records on a DSS API node endpoint (standard or custom prediction) :param str endpoint_id: Identifier of the endpoint to query :param records: Python list of records. Each record must be a Python dict. Each record must contain a "features" dict (see predict_record) and optionally a "context" dict. :param forced_generation: See documentation about multi-version prediction :param dispatch_key: See documentation about multi-version prediction :param with_explanations: Optional, whether individual explanations should be computed for each record. The prediction endpoint must be compatible. If None, will use the value configured in the endpoint. :param explanation_method: Optional, method to compute explanations. Valid values are 'SHAPLEY' or 'ICE'. If None, will use the value configured in the endpoint. :param n_explanations: Optional, number of explanations to output per prediction. If None, will use the value configured in the endpoint. :param n_explanations_mc_steps: Optional, precision parameter for SHAPLEY method, higher means more precise but slower (between 25 and 1000). If None, will use the value configured in the endpoint. :return: a Python dict of the API answer. The answer contains a "results" key (which is an array of result objects) """ for record in records: if not "features" in record: raise ValueError("Each record must contain a 'features' dict") obj = { "items": records, "explanations": { "enabled": with_explanations, "method": explanation_method, "nExplanations": n_explanations, "nMonteCarloSteps": n_explanations_mc_steps } } if forced_generation is not None: obj["dispatch"] = {"forcedGeneration" : forced_generation } elif dispatch_key is not None: obj["dispatch"] = {"dispatchKey" : dispatch_key } return self._perform_json("POST", "%s/predict-multi" % endpoint_id, body = obj) def forecast(self, endpoint_id, records, forced_generation=None, dispatch_key=None): """ Forecast using a time series forecasting model on a DSS API node endpoint :param str endpoint_id: Identifier of the endpoint to query :param records: List of time series data records to be used as an input for the time series forecasting model. Each record should be a dict where keys are feature names, and values feature values. Example: records = [ {'date': '2015-01-04T00:00:00.000Z', 'timeseries_id': 'A', 'target': 10.0}, {'date': '2015-01-04T00:00:00.000Z', 'timeseries_id': 'B', 'target': 4.5}, {'date': '2015-01-05T00:00:00.000Z', 'timeseries_id': 'A', 'target': 2.0}, ..., {'date': '2015-03-20T00:00:00.000Z', 'timeseries_id': 'B', 'target': 1.3} ] :param forced_generation: See documentation about multi-version prediction :param dispatch_key: See documentation about multi-version prediction :return: a Python dict of the API answer. The answer contains a "results" key (which is an array of result objects, corresponding to the forecast records) Example: {'results': [ {'forecast': 12.57, 'ignored': False, 'quantiles': [0.0001, 0.5, 0.9999], 'quantilesValues': [3.0, 16.0, 16.0], 'time': '2015-03-21T00:00:00.000000Z', 'timeseriesIdentifier': {'timeseries_id': 'A'}}, {'forecast': 15.57, 'ignored': False, 'quantiles': [0.0001, 0.5, 0.9999], 'quantilesValues': [3.0, 18.0, 19.0], 'time': '2015-03-21T00:00:00.000000Z', 'timeseriesIdentifier': {'timeseries_id': 'B'}}, ...], ...} """ obj = {"items": records} if forced_generation is not None: obj["dispatch"] = {"forcedGeneration": forced_generation} elif dispatch_key is not None: obj["dispatch"] = {"dispatchKey": dispatch_key} return self._perform_json("POST", "{}/forecast".format(endpoint_id), body=obj) def sql_query(self, endpoint_id, parameters): """ Queries a "SQL query" endpoint on a DSS API node :param str endpoint_id: Identifier of the endpoint to query :param parameters: Python dictionary of the named parameters for the SQL query endpoint :return: a Python dict of the API answer. The answer is the a dict with a columns field and a rows field (list of rows as list of strings) """ return self._perform_json("POST", "%s/query" % endpoint_id, body = parameters) def lookup_record(self, endpoint_id, record, context=None): """ Lookup a single record on a DSS API node endpoint of "dataset lookup" type :param str endpoint_id: Identifier of the endpoint to query :param record: Python dictionary of features of the record :param context: Optional, Python dictionary of additional context information. The context information is logged, but not directly used. :return: a Python dict of the API answer. The answer contains a "data" key (itself a dict) """ obj = { "data" :record } if context is not None: obj["context"] = context return self._perform_json("POST", "%s/lookup" % endpoint_id, body = obj).get("results", [])[0] def lookup_records(self, endpoint_id, records): """ Lookups a batch of records on a DSS API node endpoint of "dataset lookup" type :param str endpoint_id: Identifier of the endpoint to query :param records: Python list of records. Each record must be a Python dict, containing at least one entry called "data": a dict containing the input columns :return: a Python dict of the API answer. The answer contains a "results" key, which is an array of result objects. Each result contains a "data" dict which is the output """ for record in records: if not "data" in record: raise ValueError("Each record must contain a 'data' dict") obj = { "items" : records } return self._perform_json("POST", "%s/lookup-multi" % endpoint_id, body = obj) def run_function(self, endpoint_id, **kwargs): """ Calls a "Run function" endpoint on a DSS API node :param str endpoint_id: Identifier of the endpoint to query :param kwargs: Arguments of the function :return: The function result """ obj = {} for (k,v) in kwargs.items(): obj[k] = v return self._perform_json("POST", "%s/run" % endpoint_id, body = obj) dataiku-api-client-python-11.4.0/dataikuapi/base_client.py000066400000000000000000000036661440464736400235460ustar00rootroot00000000000000import json from requests import Session, exceptions from requests import exceptions from requests.auth import HTTPBasicAuth from .utils import DataikuException class DSSBaseClient(object): def __init__(self, base_uri, api_key=None, internal_ticket=None, bearer_token=None): self.api_key = api_key self.bearer_token = bearer_token self.base_uri = base_uri self._session = Session() ######################################################## # Internal Request handling ######################################################## def _perform_http(self, method, path, params=None, body=None, stream=False): if body: body = json.dumps(body) headers = None auth = None if self.api_key: auth = HTTPBasicAuth(self.api_key, "") elif self.bearer_token: headers = {"Authorization": "Bearer " + self.bearer_token} try: http_res = self._session.request( method, "%s/%s" % (self.base_uri, path), params=params, data=body, headers=headers, auth=auth, stream = stream) http_res.raise_for_status() return http_res except exceptions.HTTPError: ex = http_res.json() raise DataikuException("%s: %s" % (ex.get("errorType", "Unknown error"), ex.get("message", "No message"))) def _perform_empty(self, method, path, params=None, body=None): self._perform_http(method, path, params, body, False) def _perform_text(self, method, path, params=None, body=None): return self._perform_http(method, path, params, body, False).text def _perform_json(self, method, path, params=None, body=None): return self._perform_http(method, path, params, body, False).json() def _perform_raw(self, method, path, params=None, body=None): return self._perform_http(method, path, params, body, True) dataiku-api-client-python-11.4.0/dataikuapi/dss/000077500000000000000000000000001440464736400215025ustar00rootroot00000000000000dataiku-api-client-python-11.4.0/dataikuapi/dss/__init__.py000066400000000000000000000000001440464736400236010ustar00rootroot00000000000000dataiku-api-client-python-11.4.0/dataikuapi/dss/admin.py000066400000000000000000003145021440464736400231510ustar00rootroot00000000000000from .future import DSSFuture import json, warnings from datetime import datetime class DSSConnectionListItem(dict): """ An item in a list of connections. .. important:: Do not instantiate directly, use :meth:`dataikuapi.DSSClient.list_connections` instead. """ def __init__(self, client, data): super(DSSConnectionListItem, self).__init__(data) self.client = client def to_connection(self): """ Gets a handle corresponding to this item :rtype: :class:`DSSConnection` """ return DSSConnection(self.client, self["name"]) @property def name(self): """ Get the identifier of the connection. :rtype: string """ return self["id"] @property def type(self): """ Get the type of the connection. :return: a DSS connection type, like PostgreSQL, EC2, Azure, ... :rtype: string """ return self["label"] class DSSConnectionInfo(dict): """ A class holding read-only information about a connection. .. important:: Do not instantiate directly, use :meth:`DSSConnection.get_info` instead. The main use case of this class is to retrieve the decrypted credentials for a connection, if allowed by the connection permissions. Depending on the connection kind, the credential may be available using :meth:`get_basic_credential` or :meth:`get_aws_credential`. """ def __init__(self, data): super(DSSConnectionInfo, self).__init__(data) def get_type(self): """ Get the type of the connection :return: a connection type, for example Azure, Snowflake, GCS, ... :rtype: string """ return self["type"] def get_params(self): """ Get the parameters of the connection, as a dict :return: the parameters, as a dict. Each connection type has different sets of fields. :rtype: dict """ return self["params"] def get_basic_credential(self): """ Get the basic credential (user/password pair) for this connection, if available :return: the credential, as a dict containing "user" and "password" :rtype: dict """ if not "resolvedBasicCredential" in self: raise ValueError("No basic credential available") return self["resolvedBasicCredential"] def get_aws_credential(self): """ Get the AWS credential for this connection, if available. The AWS credential can either be a keypair or a STS token triplet. :return: the credential, as a dict containing "accessKey", "secretKey", and "sessionToken" (only in the case of STS token) :rtype: dict """ if not "resolvedAWSCredential" in self: raise ValueError("No AWS credential available") return self["resolvedAWSCredential"] class DSSConnection(object): """ A connection on the DSS instance. .. important:: Do not instantiate directly, use :meth:`dataikuapi.DSSClient.get_connection` instead. """ def __init__(self, client, name): self.client = client self.name = name ######################################################## # Location info ######################################################## def get_location_info(self): """ Get information about this connection. .. caution:: Deprecated, use :meth:`~get_info()` """ warnings.warn("DSSConnection.get_location_info is deprecated, please use get_info", DeprecationWarning) return self.get_info() def get_info(self, contextual_project_key=None): """ Get information about this connection. .. note:: This call requires permissions to read connection details :param string contextual_project_key: (optional) project key to use to resolve variables :return: an object containing connection information :rtype: :class:`DSSConnectionInfo` """ additional_params = { "contextualProjectKey": contextual_project_key } if contextual_project_key is not None else None return DSSConnectionInfo(self.client._perform_json( "GET", "/connections/%s/info" % self.name, params=additional_params)) ######################################################## # Connection deletion ######################################################## def delete(self): """ Delete the connection """ return self.client._perform_empty( "DELETE", "/admin/connections/%s" % self.name) def get_settings(self): """ Get the settings of the connection. You must use :meth:`~DSSConnectionSettings.save()` on the returned object to make your changes effective on the connection. Usage example .. code-block:: python # make details of a connection accessible to some groups connection = client.get_connection("my_connection_name") settings = connection.settings() settings.set_readability(False, "group1", "group2") settings.save() :return: the settings of the connection :rtype: :class:`DSSConnectionSettings` """ settings = self.client._perform_json( "GET", "/admin/connections/%s" % self.name) return DSSConnectionSettings(self, settings) def get_definition(self): """ Get the connection's raw definition. .. caution:: Deprecated, use :meth:`get_settings()` instead. The exact structure of the returned dict is not documented and depends on the connection type. Create connections using the DSS UI and call :meth:`get_definition` to see the fields that are in it. .. note:: This method returns a dict with passwords and secrets in their encrypted form. If you need credentials, consider using :meth:`get_info()` and :meth:`dataikuapi.dss.admin.DSSConnectionInfo.get_basic_credential()`. :return: a connection definition, as a dict. See :meth:`DSSConnectionSettings.get_raw()` :rtype: dict """ return self.client._perform_json( "GET", "/admin/connections/%s" % self.name) def set_definition(self, definition): """ Set the connection's definition. .. caution:: Deprecated, use :meth:`get_settings()` then :meth:`DSSConnectionSettings.save()` instead. .. important:: You should only :meth:`set_definition` using an object that you obtained through :meth:`get_definition`, not create a new dict. Usage example .. code-block:: python # make details of a connection accessible to some groups connection = client.get_connection("my_connection_name") definition = connection.get_definition() definition['detailsReadability']['readableBy'] = 'ALLOWED' definition['detailsReadability']['allowedGroups'] = ['group1', 'group2'] connection.set_definition(definition) :param dict definition: the definition for the connection, as a dict. """ return self.client._perform_json( "PUT", "/admin/connections/%s" % self.name, body = definition) ######################################################## # Security ######################################################## def sync_root_acls(self): """ Resync root permissions on this connection path. This is only useful for HDFS connections when DSS has User Isolation activated with "DSS-managed HDFS ACL" :return: a handle to the task of resynchronizing the permissions :rtype: :class:`~dataikuapi.dss.future.DSSFuture` """ future_response = self.client._perform_json( "POST", "/admin/connections/%s/sync" % self.name, body = {'root':True}) return DSSFuture(self.client, future_response.get('jobId', None), future_response) def sync_datasets_acls(self): """ Resync permissions on datasets in this connection path. This is only useful for HDFS connections when DSS has User Isolation activated with "DSS-managed HDFS ACL" :return: a handle to the task of resynchronizing the permissions :rtype: :class:`~dataikuapi.dss.future.DSSFuture` """ future_response = self.client._perform_json( "POST", "/admin/connections/%s/sync" % self.name, body = {'root':True}) return DSSFuture(self.client, future_response.get('jobId', None), future_response) class DSSConnectionSettings(object): """ Settings of a DSS connection. .. important:: Do not instantiate directly, use :meth:`DSSConnection.get_settings` instead. Use :meth:`save` to save your changes """ def __init__(self, connection, settings): self.connection = connection self.settings = settings def get_raw(self): """ Get the raw settings of the connection. :return: a connection definition, as a dict. Notable fields are: * **type** : type of the connection (for example PostgreSQL, Azure, ...) * **params** : dict of the parameters specific to the connection type * **allowWrite** : if False, DSS will not perform write operations on the connection * **allowManagedDatasets** : whether DSS will allow creating managed datasets on the connection * **allowManagedFolders** : whether DSS will allow creating managed folders on the connection * **credentialsMode** : whether the credentials of the connection are per-user ("PER_USER") or global for all users ("GLOBAL") * **usableBy** : defines which DSS users can use the connection. Possible values: ALL, ALLOWED * **allowedGroups** : if **usableBy** is "ALLOWED", a list of group names * **detailsReadability** : definition of which DSS users can access the details of the connection, in particular the credentials in it * **readableBy** : defines which users can access the details. Possible values: NONE, ALL, ALLOWED. * **allowedGroups** : if **readableBy** is "ALLOWED", a list of group names * **indexingSettings** : for SQL-like connection, what to index * **maxActivities** : maximum number of concurrent activities in all jobs of the instance that use the connection * **useGlobalProxy** : if a proxy is defined in the DSS general settings, whether to use it or not :rtype: dict """ return self.settings @property def type(self): """ Get the type of the connection. :return: a DSS connection type, like PostgreSQL, EC2, Azure, ... :rtype: string """ return self.settings['type'] @property def allow_managed_datasets(self): """ Whether managed datasets can use the connection. :rtype: boolean """ return self.settings['allowManagedDatasets'] @allow_managed_datasets.setter def allow_managed_datasets(self, new_value): self.settings["allowManagedDatasets"] = new_value @property def allow_managed_folders(self): """ Whether managed datasets can use the connection. :rtype: boolean """ return self.settings['allowManagedFolders'] @allow_managed_folders.setter def allow_managed_folders(self, new_value): self.settings["allowManagedFolders"] = new_value @property def allow_write(self): """ Whether data can be written to this connection. If not, the connection is read-only from DSS point of view. :rtype: boolean """ return self.settings['allowWrite'] @allow_write.setter def allow_write(self, new_value): self.settings["allowWrite"] = new_value @property def details_readability(self): """ Get the access control to connection details. :return: an handle on the access control definition. :rtype: :class:`DSSConnectionDetailsReadability` """ return DSSConnectionDetailsReadability(self.settings["detailsReadability"]) @property def usable_by(self): """ Get the mode of access control. This controls usage of the connection, that is, reading and/or writing data from/to the connection. :return: one ALL (anybody) or ALLOWED (ie. only users from groups in :meth:`usable_by_allowed_groups()`) :rtype: string """ return self.settings["usableBy"] @property def usable_by_allowed_groups(self): """ Get the groups allowed to use the connection Only applies if :meth:`usable_by()` is ALLOWED. :return: a list of group names :rtype: list[string] """ return self.settings["allowedGroups"] def set_usability(self, all, *groups): """ Set who can use the connection. :param boolean all: if True, anybody can use the connection :param *string groups: a list of groups that can use the connection """ if all: self.settings["usableBy"] = 'ALL' else: self.settings["usableBy"] = 'ALLOWED' self.settings["allowedGroups"] = groups def save(self): """ Save the changes to the connection's settings """ self.connection.client._perform_json( "PUT", "/admin/connections/%s" % self.connection.name, body = self.settings) class DSSConnectionDetailsReadability(object): """ Handle on settings for access to connection details. Connection details mostly cover credentials, and giving access to the credentials is necessary to some workloads. Typically, having Spark processes access data directly implies giving credentials to these Spark processes, which in turn implies that the user can access the connection's details. """ def __init__(self, data): self._data = data @property def readable_by(self): """ Get the mode of access control. :return: one of NONE (nobody), ALL (anybody) or ALLOWED (ie. only users from groups in :meth:`allowed_groups()`) :rtype: string """ return self._data["readableBy"] @property def allowed_groups(self): """ Get the groups allowed to access connection details. Only applies if :meth:`readable_by()` is ALLOWED. :return: a list of group names :rtype: list[string] """ return self._data["allowedGroups"] def set_readability(self, all, *groups): """ Set who can get details from the connection. To make the details readable by nobody, pass all=False and no group. :param boolean all: if True, anybody can use the connection :param *string groups: a list of groups that can use the connection """ if all: self._data["readableBy"] = 'ALL' elif groups is None or len(groups) == 0: self._data["readableBy"] = 'NONE' else: self._data["readableBy"] = 'ALLOWED' self._data["allowedGroups"] = groups class DSSUser(object): """ A handle for a user on the DSS instance. .. important:: Do not instantiate directly, use :meth:`dataikuapi.DSSClient.get_user` instead. """ def __init__(self, client, login): self.client = client self.login = login def delete(self): """ Deletes the user """ return self.client._perform_empty( "DELETE", "/admin/users/%s" % self.login) def get_settings(self): """ Get the settings of the user. You must use :meth:`~DSSUserSettings.save()` on the returned object to make your changes effective on the user. Usage example .. code-block:: python # disable some user user = client.get_user('the_user_login') settings = user.get_settings() settings.enabled = False settings.save() :return: the settings of the user :rtype: :class:`DSSUserSettings` """ raw = self.client._perform_json("GET", "/admin/users/%s" % self.login) return DSSUserSettings(self.client, self.login, raw) def get_activity(self): """ Gets the activity of the user. :return: the user's activity :rtype: :class:`DSSUserActivity` """ activity = self.client._perform_json("GET", "/admin/users/%s/activity" % self.login) return DSSUserActivity(self.client, self.login, activity) ######################################################## # Legacy ######################################################## def get_definition(self): """ Get the definition of the user .. caution:: Deprecated, use :meth:`get_settings` instead :return: the user's definition, as a dict. Notable fields are * **login** : identifier of the user, can't be modified * **enabled** : whether the user can log into DSS * **email** : email of the user * **displayName** : name of the user in the UI * **groups** : list of group names this user belongs to * **sourceType** : how the user logs in. Possible values: LOCAL, LOCAL_NO_AUTH, LDAP, SAAS, PAM * **userProfile** : name of the license profile the user belongs to * **credentials** : dict of connection name or plugin preset name to credentials * **secrets** : list of secrets, as name/value pairs * **adminProperties** and **userProperties** : dicts of arbitrary properties * **activeWebSocketSesssions** : number of tabs currently open by the user (informative only, not modifiable) :rtype: dict """ warnings.warn("DSSUser.get_definition is deprecated, please use get_settings", DeprecationWarning) return self.client._perform_json("GET", "/admin/users/%s" % self.login) def set_definition(self, definition): """ Set the user's definition. .. caution:: Deprecated, use :meth:`dataikuapi.dss.admin.DSSUserSettings.save()` instead .. important:: You should only use :meth:`set_definition` with an object that you obtained through :meth:`get_definition`, not create a new dict. .. note:: This call requires an API key with admin rights The fields that may be changed in a user definition are: * email * displayName * enabled * groups * userProfile * password (not returned by :meth:`get_definition()` but can be set) * userProperties * adminProperties * secrets * credentials :param dict definition: the definition for the user, as a dict """ warnings.warn("DSSUser.set_definition is deprecated, please use get_settings", DeprecationWarning) return self.client._perform_json("PUT", "/admin/users/%s" % self.login, body = definition) def get_client_as(self): """ Get an API client that has the permissions of this user. This allows administrators to impersonate actions on behalf of other users, in order to perform actions on their behalf. :return: a client through which calls will be run as the user :rtype: :class:`dataikuapi.DSSClient` """ from dataikuapi.dssclient import DSSClient if self.client.api_key is not None: return DSSClient(self.client.host, self.client.api_key, extra_headers={"X-DKU-ProxyUser": self.login}) elif self.client.internal_ticket is not None: return DSSClient(self.client.host, internal_ticket = self.client.internal_ticket, extra_headers={"X-DKU-ProxyUser": self.login}) else: raise ValueError("Don't know how to proxy this client") class DSSOwnUser(object): """ A handle to interact with your own user .. important:: Do not instantiate directly, use :meth:`dataikuapi.DSSClient.get_own_user` instead. """ def __init__(self, client): self.client = client def get_settings(self): """ Get your own settings You must use :meth:`~DSSOwnUserSettings.save()` on the returned object to make your changes effective on the user. :rtype: :class:`DSSOwnUserSettings` """ raw = self.client._perform_json("GET", "/current-user") return DSSOwnUserSettings(self.client, raw) class DSSUserSettingsBase(object): """ Settings for a DSS user. .. important:: Do not instantiate directly, use :meth:`DSSUser.get_settings` or :meth:`DSSOwnUser.get_settings` instead. """ def __init__(self, settings): self.settings = settings def get_raw(self): """ Get the the raw settings of the user Modifications made to the returned object are reflected when saving. :return: the dict of the settings (not a copy). Notable fields are: * **login** : identifier of the user, can't be modified * **enabled** : whether the user can log into DSS * **email** : email of the user * **displayName** : name of the user in the UI * **groups** : list of group names this user belongs to * **sourceType** : how the user logs in. One of: LOCAL, LOCAL_NO_AUTH, LDAP, SAAS, PAM * **userProfile** : name of the license profile the user belongs to * **credentials** : dict of connection name or plugin preset name to credentials * **secrets** : list of secrets, as name/value pairs * **adminProperties** and **userProperties** : dicts of arbitrary properties * **activeWebSocketSesssions** : number of tabs currently open by the user (informative only, not modifiable) :rtype: dict """ return self.settings def add_secret(self, name, value): """ Add a user secret. If there was already a secret with the same name, it is replaced :param string name: name of the secret :param string value: name of the value """ self.remove_secret(name) return self.settings["secrets"].append({"name": name, "value": value, "secret": True}) def remove_secret(self, name): """ Remove a user secret based on its name If no secret of the given name exists, the method does nothing. :param string name: name of the secret """ self.settings["secrets"] = [x for x in self.settings["secrets"] if x["name"] != name] @property def user_properties(self): """ Get the user properties for this user. .. important:: Do not set this property, modify the dict in place User properties can be seen and modified by the user themselves. A contrario admin properties are for administrators' eyes only. :rtype: dict """ return self.settings["userProperties"] def set_basic_connection_credential(self, connection, login, password): """ Set per-user-credentials for a connection that takes a user/password pair. :param string connection: name of the connection :param string login: login of the credentials :param string password: password of the credentials """ self.settings["credentials"][connection] = { "type": "BASIC", "user": login, "password": password } def remove_connection_credential(self,connection): """ Remove per-user-credentials for a connection If no credentials for the givent connection exists, this method does nothing :param string connection: name of the connection """ if connection in self.settings["credentials"]: del self.settings["credentials"][connection] def set_basic_plugin_credential(self, plugin_id, param_set_id, preset_id, param_name, login, password): """ Set per-user-credentials for a plugin preset that takes a user/password pair :param string plugin_id: identifier of the plugin :param string param_set_id: identifier of the parameter set to which the preset belongs :param string preset_id: identifier of the preset :param string param_name: name of the credentials parameter in the preset :param string login: login of the credentials :param string password: password of the credentials """ name = json.dumps(["PLUGIN", plugin_id, param_set_id, preset_id, param_name])[1:-1] self.settings["credentials"][name] = { "type": "BASIC", "user": login, "password": password } def set_oauth2_plugin_credential(self, plugin_id, param_set_id, preset_id, param_name, refresh_token): """ Set per-user-credentials for a plugin preset that takes a OAuth refresh token :param string plugin_id: identifier of the plugin :param string param_set_id: identifier of the parameter set to which the preset belongs :param string preset_id: identifier of the preset :param string param_name: name of the credentials parameter in the preset :param string refresh_token: value of the refresh token """ name = json.dumps(["PLUGIN", plugin_id, param_set_id, preset_id, param_name])[1:-1] self.settings["credentials"][name] = { "type": "OAUTH_REFRESH_TOKEN", "refreshToken": refresh_token } def remove_plugin_credential(self, plugin_id, param_set_id, preset_id, param_name): """ Remove per-user-credentials for a plugin preset :param string plugin_id: identifier of the plugin :param string param_set_id: identifier of the parameter set to which the preset belongs :param string preset_id: identifier of the preset :param string param_name: name of the credentials parameter in the preset """ name = json.dumps(["PLUGIN", plugin_id, param_set_id, preset_id, param_name])[1:-1] if name in self.settings["credentials"]: del self.settings["credentials"][name] class DSSUserSettings(DSSUserSettingsBase): """ Settings for a DSS user. .. important:: Do not instantiate directly, use :meth:`DSSUser.get_settings` instead. """ def __init__(self, client, login, settings): super(DSSUserSettings, self).__init__(settings) self.client = client self.login = login @property def admin_properties(self): """ Get the admin properties for this user. .. important:: Do not set this property, modify the dict in place Admin properties can be seen and modified only by administrators, not by the user themselves. :rtype: dict """ return self.settings["adminProperties"] @property def enabled(self): """ Whether this user is enabled. :rtype: boolean """ return self.settings["enabled"] @enabled.setter def enabled(self, new_value): self.settings["enabled"] = new_value @property def creation_date(self): """ Get the timestamp of when the user was created :return: the creation date :rtype: :class:`datetime.datetime` or None """ timestamp = self.settings["creationDate"] if "creationDate" in self.settings else None return datetime.fromtimestamp(timestamp / 1000) if timestamp else None def save(self): """ Saves the settings """ self.client._perform_json("PUT", "/admin/users/%s" % self.login, body = self.settings) class DSSOwnUserSettings(DSSUserSettingsBase): """ Settings for the current DSS user. .. important:: Do not instantiate directly, use :meth:`DSSOwnUser.get_settings()` instead. """ def __init__(self, client, settings): super(DSSOwnUserSettings, self).__init__(settings) self.client = client def save(self): """ Saves the settings """ self.client._perform_empty("PUT", "/current-user", body = self.settings) class DSSUserActivity(object): """ Activity for a DSS user. .. important:: Do not instantiate directly, use :meth:`DSSUser.get_activity` or :meth:`dataikuapi.DSSClient.list_users_activity()` instead. """ def __init__(self, client, login, activity): self.client = client self.login = login self.activity = activity def get_raw(self): """ Get the raw activity of the user as a dict. :return: the raw activity. Fields are * **login** : the login of the user for this activity * **lastSuccessfulLogin** : timestamp in milliseconds of the last time the user logged into DSS * **lastFailedLogin** : timestamp in milliseconds of the last time DSS recorded a login failure for this user * **lastSessionActivity** : timestamp in milliseconds of the last time the user opened a tab :rtype: dict """ return self.activity @property def last_successful_login(self): """ Get the last successful login of the user Returns None if there was no successful login for this user. :return: the last successful login :rtype: :class:`datetime.datetime` or None """ timestamp = self.activity["lastSuccessfulLogin"] return datetime.fromtimestamp(timestamp / 1000) if timestamp > 0 else None @property def last_failed_login(self): """ Get the last failed login of the user Returns None if there were no failed login for this user. :return: the last failed login :rtype: :class:`datetime.datetime` or None """ timestamp = self.activity["lastFailedLogin"] return datetime.fromtimestamp(timestamp / 1000) if timestamp > 0 else None @property def last_session_activity(self): """ Get the last session activity of the user The last session activity is the last time the user opened a new DSS tab or refreshed his session. Returns None if there is no session activity yet. :return: the last session activity :rtype: :class:`datetime.datetime` or None """ timestamp = self.activity["lastSessionActivity"] return datetime.fromtimestamp(timestamp / 1000) if timestamp > 0 else None class DSSAuthorizationMatrix(object): """ The authorization matrix of all groups and enabled users of the DSS instance. .. important:: Do not instantiate directly, use :meth:`dataikuapi.DSSClient.get_authorization_matrix` instead. """ def __init__(self, authorization_matrix): self.authorization_matrix = authorization_matrix @property def raw(self): """ Get the raw authorization matrix as a dict :return: the authorization matrix. There are 2 parts in the matrix, each as a top-level field and with similar structures, the **perUser** and **perGroup**: * **users** (resp. **groups**) : list of user (resp. group) names * **mayXXXX** (with different permissions as "XXXX") : list of booleans of the same length as **users** (resp. **groups**) indicating where the corresponding user has the permission * **projectsGrants** : list of project permissions, each as a dict of: * **projectKey** and **projectName** : identifiers of the project * **grants** : list of dict of the same length as **users** (resp. **groups**) indicating which grants the corresponding user has on the project :rtype: dict """ return self.authorization_matrix class DSSGroup(object): """ A group on the DSS instance. .. important:: Do not instantiate directly, use :meth:`dataikuapi.DSSClient.get_group` instead. """ def __init__(self, client, name): self.client = client self.name = name ######################################################## # Group deletion ######################################################## def delete(self): """ Deletes the group """ return self.client._perform_empty( "DELETE", "/admin/groups/%s" % self.name) def get_definition(self): """ Get the group's definition (name, description, admin abilities, type, ldap name mapping) :return: the group's definition. Top-level fields are: * **name** : name of the group * **sourceType** : type of group. Possible values: LOCAL, LDAP * **description** : description of the group * **canObtainAPITicketFromCookiesForGroupsRegex** : users in the group can impersonate users from groups whose name match this regex * **admin** : whether users in the group have administrative rights in DSS * **mayXXXX** : whether users in the group has the "XXXX" permission :rtype: dict """ return self.client._perform_json( "GET", "/admin/groups/%s" % self.name) def set_definition(self, definition): """ Set the group's definition. .. important:: You should only use :meth:`set_definition` with an object that you obtained through :meth:`get_definition`, not create a new dict. :param dict definition: the definition for the group, as a dict """ return self.client._perform_json( "PUT", "/admin/groups/%s" % self.name, body = definition) class DSSGeneralSettings(object): """ The general settings of the DSS instance. .. important:: Do not instantiate directly, use :meth:`dataikuapi.DSSClient.get_general_settings` instead. """ def __init__(self, client): self.client = client self.settings = self.client._perform_json("GET", "/admin/general-settings") ######################################################## # Update settings on instance ######################################################## def save(self): """ Save the changes that were made to the settings on the DSS instance .. note:: This call requires an API key with admin rights """ return self.client._perform_empty("PUT", "/admin/general-settings", body = self.settings) ######################################################## # Value accessors ######################################################## def get_raw(self): """ Get the settings as a dictionary :return: the settings :rtype: dict """ return self.settings def add_impersonation_rule(self, rule, is_user_rule=True): """ Add a rule to the impersonation settings :param object rule: an impersonation rule, either a :class:`dataikuapi.dss.admin.DSSUserImpersonationRule` or a :class:`dataikuapi.dss.admin.DSSGroupImpersonationRule`, or a plain dict :param boolean is_user_rule: when the rule parameter is a dict, whether the rule is for users or groups """ rule_raw = rule if isinstance(rule, DSSUserImpersonationRule): rule_raw = rule.raw is_user_rule = True elif isinstance(rule, DSSGroupImpersonationRule): rule_raw = rule.raw is_user_rule = False impersonation = self.settings['impersonation'] if is_user_rule: impersonation['userRules'].append(rule_raw) else: impersonation['groupRules'].append(rule_raw) def get_impersonation_rules(self, dss_user=None, dss_group=None, unix_user=None, hadoop_user=None, project_key=None, scope=None, rule_type=None, is_user=None, rule_from=None): """ Retrieve the user or group impersonation rules that match the parameters :param string dss_user: a DSS user name :param string dss_group: a DSS group name :param string rule_from: a regex (which will be applied to user or group names) :param string unix_user: a name to match the target UNIX user :param string hadoop_user: a name to match the target Hadoop user :param string project_key: a project key :param string scope: project-scoped ('PROJECT') or global ('GLOBAL') :param string type: the rule user or group matching method ('IDENTITY', 'SINGLE_MAPPING', 'REGEXP_RULE') :param boolean is_user: True if only user-level rules should be considered, False for only group-level rules, None to consider both """ user_matches = self.settings['impersonation']['userRules'] if is_user == None or is_user == True else [] if dss_user is not None: user_matches = [m for m in user_matches if dss_user == m.get('dssUser', None)] if rule_from is not None: user_matches = [m for m in user_matches if rule_from == m.get('ruleFrom', None)] if unix_user is not None: user_matches = [m for m in user_matches if unix_user == m.get('targetUnix', None)] if hadoop_user is not None: user_matches = [m for m in user_matches if hadoop_user == m.get('targetHadoop', None)] if project_key is not None: user_matches = [m for m in user_matches if project_key == m.get('projectKey', None)] if rule_type is not None: user_matches = [m for m in user_matches if rule_type == m.get('type', None)] if scope is not None: user_matches = [m for m in user_matches if scope == m.get('scope', None)] group_matches = self.settings['impersonation']['groupRules'] if is_user == None or is_user == False else [] if dss_group is not None: group_matches = [m for m in group_matches if dss_group == m.get('dssGroup', None)] if rule_from is not None: group_matches = [m for m in group_matches if rule_from == m.get('ruleFrom', None)] if unix_user is not None: group_matches = [m for m in group_matches if unix_user == m.get('targetUnix', None)] if hadoop_user is not None: group_matches = [m for m in group_matches if hadoop_user == m.get('targetHadoop', None)] if rule_type is not None: group_matches = [m for m in group_matches if rule_type == m.get('type', None)] all_matches = [] for m in user_matches: all_matches.append(DSSUserImpersonationRule(m)) for m in group_matches: all_matches.append(DSSGroupImpersonationRule(m)) return all_matches def remove_impersonation_rules(self, dss_user=None, dss_group=None, unix_user=None, hadoop_user=None, project_key=None, scope=None, rule_type=None, is_user=None, rule_from=None): """ Remove the user or group impersonation rules that matches the parameters from the settings :param string dss_user: a DSS user name :param string dss_group: a DSS group name :param string rule_from: a regex (which will be applied to user or group names) :param string unix_user: a name to match the target UNIX user :param string hadoop_user: a name to match the target Hadoop user :param string project_key: a project key :param string scope: project-scoped ('PROJECT') or global ('GLOBAL') :param string type: the rule user or group matching method ('IDENTITY', 'SINGLE_MAPPING', 'REGEXP_RULE') :param boolean is_user: True if only user-level rules should be considered, False for only group-level rules, None to consider both """ for m in self.get_impersonation_rules(dss_user, dss_group, unix_user, hadoop_user, project_key, scope, rule_type, is_user, rule_from): if isinstance(m, DSSUserImpersonationRule): self.settings['impersonation']['userRules'].remove(m.raw) elif isinstance(m, DSSGroupImpersonationRule): self.settings['impersonation']['groupRules'].remove(m.raw) ######################################################## # Admin actions ######################################################## def push_container_exec_base_images(self): """ Push the container exec base images to their repository """ resp = self.client._perform_json("POST", "/admin/container-exec/actions/push-base-images") if resp is None: raise Exception('Container exec base image push returned no data') if resp.get('messages', {}).get('error', False): raise Exception('Container exec base image push failed : %s' % (json.dumps(resp.get('messages', {}).get('messages', {})))) return resp class DSSUserImpersonationRule(object): """ An user-level rule items for the impersonation settings """ def __init__(self, raw=None): self.raw = raw if raw is not None else {'scope':'GLOBAL','type':'IDENTITY'} def scope_global(self): """ Make the rule apply to all projects """ self.raw['scope'] = 'GLOBAL' return self def scope_project(self, project_key): """ Make the rule apply to a given project :param string project_key: the project this rule applies to """ self.raw['scope'] = 'PROJECT' self.raw['projectKey'] = project_key return self def user_identity(self): """ Make the rule map each DSS user to a UNIX user of the same name """ self.raw['type'] = 'IDENTITY' return self def user_single(self, dss_user, unix_user, hadoop_user=None): """ Make the rule map a given DSS user to a given UNIX user :param string dss_user: a DSS user :param string unix_user: a UNIX user :param string hadoop_user: a hadoop user (optional, defaults to unix_user) """ self.raw['type'] = 'SINGLE_MAPPING' self.raw['dssUser'] = dss_user self.raw['targetUnix'] = unix_user self.raw['targetHadoop'] = hadoop_user return self def user_regexp(self, regexp, unix_user, hadoop_user=None): """ Make the rule map a DSS users matching a given regular expression to a given UNIX user :param string regexp: a regular expression to match DSS user names :param string unix_user: a UNIX user :param string hadoop_user: a hadoop user (optional, defaults to unix_user) """ self.raw['type'] = 'REGEXP_RULE' self.raw['ruleFrom'] = regexp self.raw['targetUnix'] = unix_user self.raw['targetHadoop'] = hadoop_user return self class DSSGroupImpersonationRule(object): """ A group-level rule items for the impersonation settings """ def __init__(self, raw=None): self.raw = raw if raw is not None else {'type':'IDENTITY'} def group_identity(self): """ Make the rule map each DSS user to a UNIX user of the same name """ self.raw['type'] = 'IDENTITY' return self def group_single(self, dss_group, unix_user, hadoop_user=None): """ Make the rule map a given DSS user to a given UNIX user :param string dss_group: a DSS group :param string unix_user: a UNIX user :param string hadoop_user: a hadoop user (optional, defaults to unix_user) """ self.raw['type'] = 'SINGLE_MAPPING' self.raw['dssGroup'] = dss_group self.raw['targetUnix'] = unix_user self.raw['targetHadoop'] = hadoop_user return self def group_regexp(self, regexp, unix_user, hadoop_user=None): """ Make the rule map a DSS users matching a given regular expression to a given UNIX user :param string regexp: a regular expression to match DSS groups :param string unix_user: a UNIX user :param string hadoop_user: a hadoop user (optional, defaults to unix_user) """ self.raw['type'] = 'REGEXP_RULE' self.raw['ruleFrom'] = regexp self.raw['targetUnix'] = unix_user self.raw['targetHadoop'] = hadoop_user return self class DSSCodeEnv(object): """ A code env on the DSS instance. .. important:: Do not instantiate directly, use :meth:`dataikuapi.DSSClient.get_code_env` instead. """ def __init__(self, client, env_lang, env_name): self.client = client self.env_lang = env_lang self.env_name = env_name ######################################################## # Env deletion ######################################################## def delete(self): """ Delete the code env .. note:: This call requires an API key with admin rights """ resp = self.client._perform_json( "DELETE", "/admin/code-envs/%s/%s" % (self.env_lang, self.env_name)) if resp is None: raise Exception('Env deletion returned no data') if resp.get('messages', {}).get('error', False): raise Exception('Env deletion failed : %s' % (json.dumps(resp.get('messages', {}).get('messages', {})))) return resp ######################################################## # Code env description ######################################################## def get_definition(self): """ Get the code env's definition .. caution:: Deprecated, use :meth:`get_settings` instead .. note:: This call requires an API key with admin rights :return: the code env definition :rtype: dict """ return self.client._perform_json( "GET", "/admin/code-envs/%s/%s" % (self.env_lang, self.env_name)) def set_definition(self, env): """ Set the code env's definition. The definition should come from a call to :meth:`get_definition` .. caution:: Deprecated, use :meth:`get_settings` then :meth:`DSSDesignCodeEnvSettings.save()` or :meth:`DSSAutomationCodeEnvSettings.save()` instead Fields that can be updated in design node: * env.permissions, env.usableByAll, env.desc.owner * env.specCondaEnvironment, env.specPackageList, env.externalCondaEnvName, env.desc.installCorePackages, env.desc.corePackagesSet, env.desc.installJupyterSupport, env.desc.yarnPythonBin, env.desc.yarnRBin env.desc.envSettings, env.desc.allContainerConfs, env.desc.containerConfs, env.desc.allSparkKubernetesConfs, env.desc.sparkKubernetesConfs Fields that can be updated in automation node (where {version} is the updated version): * env.permissions, env.usableByAll, env.owner, env.envSettings * env.{version}.specCondaEnvironment, env.{version}.specPackageList, env.{version}.externalCondaEnvName, env.{version}.desc.installCorePackages, env.{version}.corePackagesSet, env.{version}.desc.installJupyterSupport env.{version}.desc.yarnPythonBin, env.{version}.desc.yarnRBin, env.{version}.desc.allContainerConfs, env.{version}.desc.containerConfs, env.{version}.desc.allSparkKubernetesConfs, env.{version}.{version}.desc.sparkKubernetesConfs .. note:: This call requires an API key with admin rights .. important:: You should only :meth:`set_definition` using an object that you obtained through :meth:`get_definition`, not create a new dict. :param dict data: a code env definition :return: the updated code env definition :rtype: dict """ return self.client._perform_json( "PUT", "/admin/code-envs/%s/%s" % (self.env_lang, self.env_name), body=env) def get_version_for_project(self, project_key): """ Resolve the code env version for a given project .. note:: Version will only be non-empty for versioned code envs actually used by the project :param string project_key: project to get the version for :return: the code env version full reference for the version of the code env that the project use. Fields are * **lang** : language of the code env (PYTHON or R) * **name** : name of the code env * **version** : identifier of the version * **projectKey** : project key * **bundleId** : identifier of the active bundle in the project :rtype: dict """ return self.client._perform_json( "GET", "/admin/code-envs/%s/%s/%s/version" % (self.env_lang, self.env_name, project_key)) def get_settings(self): """ Get the settings of this code env. .. important:: You must use :meth:`DSSCodeEnvSettings.save()` on the returned object to make your changes effective on the code env. .. code-block:: python # Example: setting the required packagd codeenv = client.get_code_env("PYTHON", "code_env_name") settings = codeenv.get_settings() settings.set_required_packages("dash==2.0.0", "bokeh<2.0") settings.save() # then proceed to update_packages() :rtype: :class:`DSSDesignCodeEnvSettings` or :class:`DSSAutomationCodeEnvSettings` """ data = self.client._perform_json( "GET", "/admin/code-envs/%s/%s" % (self.env_lang, self.env_name)) # you can't just use deploymentMode to check if it's an automation code # env, because some modes are common to both types of nodes. So we rely # on a non-null field that only the automation code envs have if data.get("versions", None) is not None: return DSSAutomationCodeEnvSettings(self, data) else: return DSSDesignCodeEnvSettings(self, data) ######################################################## # Code env actions ######################################################## def set_jupyter_support(self, active): """ Update the code env jupyter support .. note:: This call requires an API key with admin rights :param boolean active: True to activate jupyter support, False to deactivate """ resp = self.client._perform_json( "POST", "/admin/code-envs/%s/%s/jupyter" % (self.env_lang, self.env_name), params = {'active':active}) if resp is None: raise Exception('Env update returned no data') if resp.get('messages', {}).get('error', False): raise Exception('Env update failed : %s' % (json.dumps(resp.get('messages', {}).get('messages', {})))) return resp def update_packages(self, force_rebuild_env=False): """ Update the code env packages so that it matches its spec .. note:: This call requires an API key with admin rights :param boolean force_rebuild_env: whether to rebuild the code env from scratch :return: list of messages collected during the operation. Fields are: * **anyMessage** : whether there is at least 1 message * **success**, **warning**, **error** and **fatal** : whether there is at least one message of the corresponding category * **messages** : list of messages. Each message is a dict, with at least **severity** and **message** sufields. :rtype: dict """ resp = self.client._perform_json( "POST", "/admin/code-envs/%s/%s/packages" % (self.env_lang, self.env_name), params={"forceRebuildEnv": force_rebuild_env}) if resp is None: raise Exception('Env update returned no data') if resp.get('messages', {}).get('error', False): raise Exception('Env update failed : %s' % (json.dumps(resp.get('messages', {}).get('messages', {})))) return resp def update_images(self, env_version=None): """ Rebuild the docker image of the code env .. note:: This call requires an API key with admin rights :param string env_version: (optional) version of the code env. Applies only to versioned code envs. :return: list of messages collected during the operation. Fields are: * **anyMessage** : whether there is at least 1 message * **success**, **warning**, **error** and **fatal** : whether there is at least one message of the corresponding category * **messages** : list of messages. Each message is a dict, with at least **severity** and **message** sufields. :rtype: dict """ resp = self.client._perform_json( "POST", "/admin/code-envs/%s/%s/images" % (self.env_lang, self.env_name), params={"envVersion": env_version}) if resp is None: raise Exception('Env image build returned no data') if resp.get('messages', {}).get('error', False): raise Exception('Env image build failed : %s' % (json.dumps(resp.get('messages', {}).get('messages', {})))) return resp def list_usages(self): """ List usages of the code env in the instance :return: a list of objects where the code env is used. Each usage has fields: * **envLang** and **envName** : identifiers of the code env * **envUsage** : type of usage. Possible values: PROJECT, RECIPE, NOTEBOOK, PLUGIN, SCENARIO, SCENARIO_STEP, SCENARIO_TRIGGER, DATASET_METRIC, DATASET_CHECK, DATASET, WEBAPP, REPORT, API_SERVICE_ENDPOINT, SAVED_MODEL, MODEL, CODE_STUDIO_TEMPLATE * **projectKey** and **objectId** : identifier of the object where the code env is used * **accessible** : if False, the **projectKey** and **objectId** are obfuscated and point to an object of a project that the user can't access :rtype: list[dict] """ return self.client._perform_json( "GET", "/admin/code-envs/%s/%s/usages" % (self.env_lang, self.env_name)) def list_logs(self): """ List logs of the code env in the instance :return: a list of log descriptions. Each log description has fields: * **name** : name of the log file * **totalSize** : size in bytes of the log * **lastModified** : timestamp in milliseconds of the last change to the log * **tail** : structure holding the tail of the log file :rtype: list[dict] """ return self.client._perform_json( "GET", "/admin/code-envs/%s/%s/logs" % (self.env_lang, self.env_name)) def get_log(self, log_name): """ Get the logs of the code env :param string log_name: name of the log to fetch :return: the raw log :rtype: string """ return self.client._perform_text( "GET", "/admin/code-envs/%s/%s/logs/%s" % (self.env_lang, self.env_name, log_name)) class DSSCodeEnvSettings(object): """ Base settings class for a DSS code env. .. important:: Do not instantiate directly, use :meth:`DSSCodeEnv.get_settings` instead. Use :meth:`save` to save your changes """ def __init__(self, codeenv, settings): self.codeenv = codeenv self.settings = settings @property def env_lang(self): """ Get the language of the code env :return: a language (possible values: PYTHON, R) :rtype: string """ return self.codeenv.env_lang @property def env_name(self): """ Get the name of the code env :rtype: string """ return self.codeenv.env_name def save(self): """ Save the changes to the code env's settings """ self.codeenv.client._perform_json( "PUT", "/admin/code-envs/%s/%s" % (self.env_lang, self.env_name), body=self.settings) class DSSCodeEnvPackageListBearer(object): def get_required_packages(self, as_list=False): """ Get the list of required packages, as a single string :param boolean as_list: if True, return the spec as a list of lines; if False, return as a single multiline string :return: a list of packages specifications :rtype: list[string] or string """ x = self.settings.get("specPackageList", "") return x.split('\n') if as_list else x def set_required_packages(self, *packages): """ Set the list of required packages :param list[string] packages: a list of packages specifications """ self.settings["specPackageList"] = '\n'.join(packages) def get_required_conda_spec(self, as_list=False): """ Get the list of required conda packages, as a single string :param boolean as_list: if True, return the spec as a list of lines; if False, return as a single multiline string :return: a list of packages specifications :rtype: list[string] or string """ x = self.settings.get("specCondaEnvironment", "") return x.split('\n') if as_list else x def set_required_conda_spec(self, *spec): """ Set the list of required conda packages :param list[string] spec: a list of packages specifications """ self.settings["specCondaEnvironment"] = '\n'.join(packages) class DSSCodeEnvContainerConfsBearer(object): def get_built_for_all_container_confs(self): """ Whether the code env creates an image for each container config :rtype: boolean """ return self.settings.get("allContainerConfs", False) def get_built_container_confs(self): """ Get the list of container configs for which the code env builds an image (if not all) :return: a list of container configuration names :rtype: list[string] """ return self.settings.get("containerConfs", []) def set_built_container_confs(self, *configs, **kwargs): """ Set the list of container configs for which the code env builds an image :param boolean all: if True, an image is built for each config :param list[string] configs: list of configuration names to build images for """ all = kwargs.get("all", False) self.settings['allContainerConfs'] = all if not all: self.settings['containerConfs'] = configs def built_for_all_spark_kubernetes_confs(self): """ Whether the code env creates an image for each managed Spark over Kubernetes config """ return self.settings.get("allSparkKubernetesConfs", False) def get_built_spark_kubernetes_confs(self): """ Get the list of managed Spark over Kubernetes configs for which the code env builds an image (if not all) :return: a list of spark configuration names :rtype: list[string] """ return self.settings.get("sparkKubernetesConfs", []) def set_built_spark_kubernetes_confs(self, *configs, **kwargs): """ Set the list of managed Spark over Kubernetes configs for which the code env builds an image :param boolean all: if True, an image is built for each config :param list[string] configs: list of configuration names to build images for """ all = kwargs.get("all", False) self.settings['allSparkKubernetesConfs'] = all if not all: self.settings['sparkKubernetesConfs'] = configs class DSSDesignCodeEnvSettings(DSSCodeEnvSettings, DSSCodeEnvPackageListBearer, DSSCodeEnvContainerConfsBearer): """ Base settings class for a DSS code env on a design node. .. important:: Do not instantiate directly, use :meth:`DSSCodeEnv.get_settings` instead. Use :meth:`save` to save your changes """ def __init__(self, codeenv, settings): super(DSSDesignCodeEnvSettings, self).__init__(codeenv, settings) def get_raw(self): """ Get the raw code env settings The structure depends on the type of code env. Notable fields are: * **envLang** and **envName** : identifiers of the code env * **desc** : definition of the code env, persisted on disk * **deploymentMode** : type of code env. Possible values: DSS_INTERNAL, DESIGN_MANAGED, DESIGN_NON_MANAGED, PLUGIN_MANAGED, PLUGIN_NON_MANAGED, EXTERNAL_CONDA_NAMED * **conda** : if True, the code env is created using Conda. If False, using virtualenv (for Python) or by linking to the system R (for R) * **externalCondaEnvName** : for EXTERNAL_CONDA_NAMED code envs, the name of the associated conda env * **envSettings** : settings for the building of the code env * **inheritGlobalSettings** : if True, values come from the instance general settings * **condaInstallExtraOptions** : extra command line options to pass to `conda install` * **condaCreateExtraOptions** : extra command line options to pass to `conda create` * **pipInstallExtraOptions** : extra command line options to pass to `pip install` * **virtualenvCreateExtraOptions** : extra command line options to pass to `virtualenv` * **cranMirrorURL** : URL of CRAN mirror to use to pull package * **allContainerConfs** : if True, build container images for all container configs on code env updates. If False, build images only for configs in **containerConfs** * **containerConfs** : list of container config names * **allSparkKubernetesConfs** : if True, build container images for all spark configs on code env updates. If False, build images only for configs in **sparkKubernetesConfs** * **sparkKubernetesConfs** : list of spark config names * **rebuildDependentCodeStudioTemplates** : which code studio templates to rebuild on code env updates. Possible values are ASK (open modal to ask for user input), ALL, NONE * **owner** : login of the owner of the code env * **usableByAll** : if True, all users can use the code env. If false, **permissions** apply * **permissions** : list of permissions items. Each item has a group name and booleans for each permission * **yarnPythonBin** or **yarnRBin** : path to Python (resp. R) on the cluster nodes, for use in Spark jobs running on Yarn * **pythonInterpreter** : type of Python used. Possible values: PYTHON27, PYTHON34, PYTHON35, PYTHON36, PYTHON37, PYTHON38, PYTHON39, PYTHON310, PYTHON311, CUSTOM * **customInterpreter** : if **pythonInterpreter** is "CUSTOM", the path to the Python binary * **corePackagesSet** : which set of core packages to instal in the code env. Possible values: LEGACY_PANDAS023, PANDAS10, PANDAS11, PANDAS12, PANDAS13 * **installJupyterSupport** : if True, the packages necessary for using the code env in notebooks are installed * **dockerImageResources** : behavior w.r.t. code env resources. Possible values: INIT (run initialization script), COPY (copy resources), NONE * several fields from **desc** are also copied to the top-level, notably **deploymentMode** and the fields around permission handling. * **canUpdateCodeEnv** and **canManageUsersCodeEnv** : (read-only) indicate whether the current user can update the code env or manage its permissions * **resourcesInitScript** : for Python code env, the contents resource script * **info** : (read-only) for Python code env, a dict with a **pythonVersion** field * **specPackageList** and **specCondaEnvironment** : list of packages requested by the user, as strings * **actualPackageList** and **actualCondaEnvironment** : (read-only) actual packages in the code env, as strings * **mandatoryPackageList** and **mandatoryCondaEnvironment** : (read-only) base packages added automatically by DSS on update, as strings :return: code env settings :rtype: dict """ return self.settings class DSSAutomationCodeEnvSettings(DSSCodeEnvSettings, DSSCodeEnvContainerConfsBearer): """ Base settings class for a DSS code env on an automation node. .. important:: Do not instantiate directly, use :meth:`DSSCodeEnv.get_settings` instead. Use :meth:`save` to save your changes """ def __init__(self, codeenv, settings): super(DSSAutomationCodeEnvSettings, self).__init__(codeenv, settings) def get_raw(self): """ Get the raw code env settings The structure depends on the type of code env. Notable fields are: * **envLang** and **envName** : identifiers of the code env * **deploymentMode** : type of code env. Possible values: DSS_INTERNAL, PLUGIN_MANAGED, PLUGIN_NON_MANAGED, AUTOMATION_VERSIONED, AUTOMATION_SINGLE, AUTOMATION_NON_MANAGED_PATH, EXTERNAL_CONDA_NAMED * **allContainerConfs** : if True, build container images for all container configs on code env updates. If False, build images only for configs in **containerConfs** * **containerConfs** : list of container config names * **allSparkKubernetesConfs** : if True, build container images for all spark configs on code env updates. If False, build images only for configs in **sparkKubernetesConfs** * **sparkKubernetesConfs** : list of spark config names * **rebuildDependentCodeStudioTemplates** : which code studio templates to rebuild on code env updates. Possible values are ASK (open modal to ask for user input), ALL, NONE * **owner** : login of the owner of the code env * **usableByAll** : if True, all users can use the code env. If false, **permissions** apply * **permissions** : list of permissions items. Each item has a group name and booleans for each permission * **envSettings** : settings for the building of the code env * **overrideImportedEnvSettings** : if False, values come from the instance general settings * **condaInstallExtraOptions** : extra command line options to pass to `conda install` * **condaCreateExtraOptions** : extra command line options to pass to `conda create` * **pipInstallExtraOptions** : extra command line options to pass to `pip install` * **virtualenvCreateExtraOptions** : extra command line options to pass to `virtualenv` * **cranMirrorURL** : URL of CRAN mirror to use to pull package * **canUpdateCodeEnv** and **canManageUsersCodeEnv** : (read-only) indicate whether the current user can update the code env or manage its permissions * **currentVersion** : when **deploymentMode** is "AUTOMATION_SINGLE", the single version. Use :meth:`get_version()` to access * **versions** : when **deploymentMode** is "AUTOMATION_VERSIONED", a list of code env versions. Use :meth:`get_version()` to access * **noVersion** : when **deploymentMode** is neither "AUTOMATION_SINGLE" nor "AUTOMATION_VERSIONED", the spec of the code env. Use :meth:`get_version()` to access :return: code env settings :rtype: dict """ return self.settings def get_version(self, version_id=None): """ Get a specific code env version (for versioned envs) or the single version :param string version_id: for versioned code env, identifier of the desired version :return: the settings of a code env version :rtype: :class:`DSSAutomationCodeEnvVersionSettings` """ deployment_mode = self.settings.get("deploymentMode", None) if deployment_mode in ['AUTOMATION_SINGLE']: return DSSAutomationCodeEnvVersionSettings(self.codeenv, self.settings.get('currentVersion', {})) elif deployment_mode in ['AUTOMATION_VERSIONED']: versions = self.settings.get("versions", []) version_ids = [v.get('versionId') for v in versions] if version_id is None: raise Exception("A version id is required in a versioned code env. Existing ids: %s" % ', '.join(version_ids)) for version in versions: if version_id == version.get("versionId"): return DSSAutomationCodeEnvVersionSettings(self.codeenv, version) raise Exception("Version %s not found in : %s" % (version_id, ', '.join(version_ids))) elif deployment_mode in ['PLUGIN_NON_MANAGED', 'PLUGIN_MANAGED', 'AUTOMATION_NON_MANAGED_PATH', 'EXTERNAL_CONDA_NAMED']: return DSSAutomationCodeEnvVersionSettings(self.codeenv, self.settings.get('noVersion', {})) else: raise Exception("Unexpected deployment mode %s for an automation node code env. Alter the settings directly with get_raw()", deployment_mode) class DSSAutomationCodeEnvVersionSettings(DSSCodeEnvPackageListBearer): """ Base settings class for a DSS code env version on an automation node. .. important:: Do not instantiate directly, use :meth:`DSSAutomationCodeEnvSettings.get_version` instead. Use :meth:`save` on the :class:`DSSAutomationCodeEnvSettings` to save your changes """ def __init__(self, codeenv_settings, version_settings): self.codeenv_settings = codeenv_settings self.settings = version_settings def get_raw(self): """ Get the raw code env version settings The structure depends on the type of code env. Notable fields are: * **versionId** : identifier of the code env version * **path** : (read-only) location of the version on disk * **desc** : definition of the code env, persisted on disk * **versionId** : type of code env. Possible values: DSS_INTERNAL, DESIGN_MANAGED, DESIGN_NON_MANAGED, PLUGIN_MANAGED, PLUGIN_NON_MANAGED, EXTERNAL_CONDA_NAMED * **conda** : if True, the code env is created using Conda. If False, using virtualenv (for Python) or by linking to the system R (for R) * **externalCondaEnvName** : for EXTERNAL_CONDA_NAMED code envs, the name of the associated conda env * **yarnPythonBin** or **yarnRBin** : path to Python (resp. R) on the cluster nodes, for use in Spark jobs running on Yarn * **pythonInterpreter** : type of Python used. Possible values: PYTHON27, PYTHON34, PYTHON35, PYTHON36, PYTHON37, PYTHON38, PYTHON39, PYTHON310, PYTHON311, CUSTOM * **customInterpreter** : if **pythonInterpreter** is "CUSTOM", the path to the Python binary * **corePackagesSet** : which set of core packages to instal in the code env. Possible values: LEGACY_PANDAS023, PANDAS10, PANDAS11, PANDAS12, PANDAS13 * **installJupyterSupport** : if True, the packages necessary for using the code env in notebooks are installed * **dockerImageResources** : behavior w.r.t. code env resources. Possible values: INIT (run initialization script), COPY (copy resources), NONE * **resourcesInitScript** : for Python code env, the contents resource script * **info** : (read-only) for Python code env, a dict with a **pythonVersion** field * **specPackageList** and **specCondaEnvironment** : list of packages requested by the user, as strings * **actualPackageList** and **actualCondaEnvironment** : (read-only) actual packages in the code env, as strings * **mandatoryPackageList** and **mandatoryCondaEnvironment** : (read-only) base packages added automatically by DSS on update, as strings :return: code env settings :rtype: dict """ return self.settings class DSSGlobalApiKey(object): """ A global API key on the DSS instance """ def __init__(self, client, key): self.client = client self.key = key ######################################################## # Key deletion ######################################################## def delete(self): """ Delete the api key .. note:: This call requires an API key with admin rights """ return self.client._perform_empty( "DELETE", "/admin/globalAPIKeys/%s" % self.key) ######################################################## # Key description ######################################################## def get_definition(self): """ Get the API key's definition .. note:: This call requires an API key with admin rights :return: the API key definition. Top-level fields are: * **id** : identifier of the key * **key** : value of the key * **label** : label of the key * **description** : longer description of the key * **createdOn** : timestamp of creation, in milliseconds * **createdBy** : login of user who created the key * **dssUserForImpersonation** : login of user that the key impersonates * **adminProperties** : dict of properties set by administrators * **userProperties** : dict of properties set by users with access to the key * **globalPermissions** : dict of instance-wide permissions (each field is a boolean for a permission) * **execSQLLike** : whether the key can run SQL queries * **projectFolders** : dict of project folder identifier to dict of permissions on that project folder * **projects** : dict of project key to dict of permissions on that project * **codeEnvs** : dict of code env name to dict of permissions on that code env * **clusters** : dict of cluster name to dict of permissions on that cluster * **codeStudioTemplates** : dict of code studio template identifier to dict of permissions on that code studio template * **plugins** : dict of plugin identifier to dict of permissions on that plugin * **pluginPresets** : dict of preset identifier to dict of permissions on that preset * **pluginParameterSets** : dict of parameter set name to dict of permissions on that parameter set * **unscopedDatasets** : list of dict giving permissions to specific datasets :rtype: dict """ return self.client._perform_json( "GET", "/admin/globalAPIKeys/%s" % (self.key)) def set_definition(self, definition): """ Set the API key's definition. .. note:: This call requires an API key with admin rights .. important:: You should only :meth:`set_definition` using an object that you obtained through :meth:`get_definition`, not create a new dict. Usage example .. code-block:: python # make an API key able to create projects key = client.get_global_api_key('my_api_key_secret') definition = key.get_definition() definition["globalPermissions"]["mayCreateProjects"] = True key.set_definition(definition) :param dict definition: the definition for the API key """ return self.client._perform_empty( "PUT", "/admin/globalAPIKeys/%s" % self.key, body = definition) class DSSGlobalApiKeyListItem(dict): """ An item in a list of personal API key. .. important:: Do not instantiate directly, use :meth:`dataikuapi.DSSClient.list_global_api_keys` instead. """ def __init__(self, client, data): super(DSSGlobalApiKeyListItem, self).__init__(data) self.client = client def to_global_api_key(self): """ Gets a handle corresponding to this item :rtype: :class:`DSSGlobalApiKey` """ return DSSGlobalApiKey(self.client, self["key"]) @property def id(self): """ Get the identifier of the API key :rtype: string """ return self["id"] @property def user_for_impersonation(self): """ Get the user associated to the API key :rtype: string """ return self.get("dssUserForImpersonation") @property def key(self): """ Get the API key :rtype: string """ return self["key"] @property def label(self): """ Get the label of the API key :rtype: string """ return self["label"] @property def description(self): """ Get the description of the API key :rtype: string """ return self.get("description") @property def created_on(self): """ Get the timestamp of when the API key was created :rtype: :class:`datetime.datetime` """ timestamp = self["createdOn"] return datetime.fromtimestamp(timestamp / 1000) if timestamp > 0 else None @property def created_by(self): """ Get the login of the user who created the API key :rtype: string """ return self["createdBy"] class DSSPersonalApiKey(object): """ A personal API key on the DSS instance. .. important:: Do not instantiate directly, use :meth:`dataikuapi.DSSClient.get_personal_api_key` instead. """ def __init__(self, client, id_): self.client = client self.id_ = id_ ######################################################## # Key description ######################################################## def get_definition(self): """ Get the API key's definition :return: the personal API key definition. Top level fields are: * **id** : identifier of the key * **key** : value of the key * **user** : login of the user that this key acts on behalf of * **label** : label of the key * **description** : longer description of the key * **createdOn** : timestamp of creation, in milliseconds * **createdBy** : login of the user who create the key :rtype: dict """ return self.client._perform_json( "GET", "/personal-api-keys/%s" % (self.id_)) ######################################################## # Key deletion ######################################################## def delete(self): """ Delete the API key """ return self.client._perform_empty( "DELETE", "/personal-api-keys/%s" % self.id_) class DSSPersonalApiKeyListItem(dict): """ An item in a list of personal API key. .. important:: Do not instantiate directly, use :meth:`dataikuapi.DSSClient.list_personal_api_keys` or :meth:`dataikuapi.DSSClient.list_all_personal_api_keys` instead. """ def __init__(self, client, data): super(DSSPersonalApiKeyListItem, self).__init__(data) self.client = client def to_personal_api_key(self): """ Gets a handle corresponding to this item :rtype: :class:`DSSPersonalApiKey` """ return DSSPersonalApiKey(self.client, self["id"]) @property def id(self): """ Get the identifier of the API key :rtype: string """ return self["id"] @property def user(self): """ Get the user associated to the API key :rtype: string """ return self["user"] @property def key(self): """ Get the API key :rtype: string """ return self["key"] @property def label(self): """ Get the label of the API key :rtype: string """ return self["label"] @property def description(self): """ Get the description of the API key :rtype: string """ return self["description"] @property def created_on(self): """ Get the timestamp of when the API key was created :rtype: :class:`datetime.datetime` """ timestamp = self["createdOn"] return datetime.fromtimestamp(timestamp / 1000) if timestamp > 0 else None @property def created_by(self): """ Get the login of the user who created the API key :rtype: string """ return self["createdBy"] class DSSCluster(object): """ A handle to interact with a cluster on the DSS instance. .. important:: Do not instantiate directly, use :meth:`dataikuapi.DSSClient.get_cluster` instead. """ def __init__(self, client, cluster_id): self.client = client self.cluster_id = cluster_id ######################################################## # Cluster deletion ######################################################## def delete(self): """ Deletes the cluster. .. important:: This does not previously stop it. """ self.client._perform_empty( "DELETE", "/admin/clusters/%s" % (self.cluster_id)) ######################################################## # Cluster description ######################################################## def get_settings(self): """ Get the cluster's settings. This includes opaque data for the cluster if this is a started managed cluster. The returned object can be used to save settings. :return: a :class:`DSSClusterSettings` object to interact with cluster settings :rtype: :class:`DSSClusterSettings` """ settings = self.client._perform_json( "GET", "/admin/clusters/%s" % (self.cluster_id)) return DSSClusterSettings(self.client, self.cluster_id, settings) def get_definition(self): """ Get the cluster's definition. This includes opaque data for the cluster if this is a started managed cluster. .. caution:: Deprecated, use :meth:`get_settings()` :return: the definition of the cluster. Fields are: * **id** : unique identifier of the cluster * **name** : name of the cluster, in the UI * **architecture** : kind of cluster (either HADOOP or KUBERNETES) * **origin** : agent who created the cluster (either MANUAL or SCENARIO) * **type** : type of cluster. Can be "manual" or a plugin cluster element type * **params** : for clusters from plugin components, the settings shown in the cluster's form. * **state** : (read-only) current state of the cluster. Possible values are NONE, STARTING, RUNNING, STOPPING * **data** : when in **state** "RUNNING", a dict of data for use by the cluster's plugin component. Contents depend on each cluster type. * **owner**, **usableByAll** and **permissions** : definition of permissions on cluster * **canUpdateCluster** : (read-only) whether the user can update the cluster's settings or state * **canManageUsersCluster** : (read-only) whether the user can manage the cluster's permissions * **XXXXSettings** : dict of settings (resp. override mask) for XXXX in Hadoop, Hive, Impala, Spark and Container. These settings apply on top of the corresponding settings in the instance's general settings :rtype: dict """ return self.client._perform_json( "GET", "/admin/clusters/%s" % (self.cluster_id)) def set_definition(self, cluster): """ Set the cluster's definition. The definition should come from a call to the get_definition() method. .. caution:: Deprecated, use :meth:`DSSClusterSettings.save()` .. important:: You should only :meth:`set_definition` using an object that you obtained through :meth:`get_definition`, not create a new dict. :param dict cluster: a cluster definition :return: the updated cluster definition :rtype: dict """ return self.client._perform_json( "PUT", "/admin/clusters/%s" % (self.cluster_id), body=cluster) def get_status(self): """ Get the cluster's status and usage :return: The cluster status, as a :class:`DSSClusterStatus` object :rtype: :class:`DSSClusterStatus` """ status = self.client._perform_json("GET", "/admin/clusters/%s/status" % (self.cluster_id)) return DSSClusterStatus(self.client, self.cluster_id, status) ######################################################## # Cluster actions ######################################################## def start(self): """ Starts or attaches the cluster .. caution:: This operation is only valid for a managed cluster. """ resp = self.client._perform_json( "POST", "/admin/clusters/%s/actions/start" % (self.cluster_id)) if resp is None: raise Exception('Cluster operation returned no data') if resp.get('messages', {}).get('error', False): raise Exception('Cluster operation failed : %s' % (json.dumps(resp.get('messages', {}).get('messages', {})))) return resp def stop(self, terminate=True, force_stop=False): """ Stops or detaches the cluster This operation is only valid for a managed cluster. :param boolean terminate: whether to delete the cluster after stopping it :param boolean force_stop: whether to try to force stop the cluster, useful if DSS expects the cluster to already be stopped """ resp = self.client._perform_json( "POST", "/admin/clusters/%s/actions/stop" % (self.cluster_id), params={'terminate': terminate, 'forceStop': force_stop}) if resp is None: raise Exception('Env update returned no data') if resp.get('messages', {}).get('error', False): raise Exception('Cluster operation failed : %s' % (json.dumps(resp.get('messages', {}).get('messages', {})))) return resp def run_kubectl(self, args): """ Runs an arbitrary kubectl command on the cluster. .. caution:: This operation is only valid for a Kubernetes cluster. .. note:: This call requires an API key with DSS instance admin rights :param string args: the arguments to pass to kubectl (without the "kubectl") :return: a dict containing the return value, standard output, and standard error of the command :rtype: dict """ return self.client._perform_json( "POST", "/admin/clusters/%s/k8s/actions/run-kubectl" % self.cluster_id, body={'args': args}) def delete_finished_jobs(self, delete_failed=False, namespace=None, label_filter=None, dry_run=False): """ Runs a kubectl command to delete finished jobs. .. caution:: This operation is only valid for a Kubernetes cluster. :param boolean delete_failed: if True, delete both completed and failed jobs, otherwise only delete completed jobs :param string namespace: the namespace in which to delete the jobs, if None, uses the namespace set in kubectl's current context :param string label_filter: delete only jobs matching a label filter :param boolean dry_run: if True, execute the command as a "dry run" :return: a dict containing whether the deletion succeeded, a list of deleted job names, and debug info for the underlying kubectl command :rtype: dict """ return self.client._perform_json( "POST", "/admin/clusters/%s/k8s/jobs/actions/delete-finished" % self.cluster_id, params={'deleteFailed': delete_failed, 'namespace': namespace, 'labelFilter': label_filter, 'dryRun': dry_run}) def delete_finished_pods(self, namespace=None, label_filter=None, dry_run=False): """ Runs a kubectl command to delete finished (succeeded and failed) pods. .. caution:: This operation is only valid for a Kubernetes cluster. :param string namespace: the namespace in which to delete the pods, if None, uses the namespace set in kubectl's current context :param string label_filter: delete only pods matching a label filter :param boolean dry_run: if True, execute the command as a "dry run" :return: a dict containing whether the deletion succeeded, a list of deleted pod names, and debug info for the underlying kubectl command :rtype: dict """ return self.client._perform_json( "POST", "/admin/clusters/%s/k8s/pods/actions/delete-finished" % self.cluster_id, params={'namespace': namespace, 'labelFilter': label_filter, 'dryRun': dry_run}) def delete_all_pods(self, namespace=None, label_filter=None, dry_run=False): """ Runs a kubectl command to delete all pods. .. caution:: This operation is only valid for a Kubernetes cluster. :param string namespace: the namespace in which to delete the pods, if None, uses the namespace set in kubectl's current context :param string label_filter: delete only pods matching a label filter :param boolean dry_run: if True, execute the command as a "dry run" :return: a dict containing whether the deletion succeeded, a list of deleted pod names, and debug info for the underlying kubectl command :rtype: dict """ return self.client._perform_json( "POST", "/admin/clusters/%s/k8s/pods/actions/delete-all" % self.cluster_id, params={'namespace': namespace, 'labelFilter': label_filter, 'dryRun': dry_run}) class DSSClusterSettings(object): """ The settings of a cluster. .. important:: Do not instantiate directly, use :meth:`DSSCluster.get_settings` instead. """ def __init__(self, client, cluster_id, settings): self.client = client self.cluster_id = cluster_id self.settings = settings def get_raw(self): """ Gets all settings as a raw dictionary. Changes made to the returned object will be reflected when saving. Fields that can be updated: * **permissions**, **usableByAll**, **owner** * **params** :return: reference to the raw settings, not a copy. See :meth:`DSSCluster.get_definition()` :rtype: dict """ return self.settings def get_plugin_data(self): """ Get the opaque data returned by the cluster's start. .. caution:: You should generally not modify this :return: the data stored by the plugin in the cluster, None if the cluster is not created by a plugin :rtype: dict """ return self.settings.get("data", None) def save(self): """ Saves back the settings to the cluster """ return self.client._perform_json( "PUT", "/admin/clusters/%s" % (self.cluster_id), body=self.settings) class DSSClusterStatus(object): """ The status of a cluster. .. important:: Do not instantiate directly, use :meth:`DSSCluster.get_status` instead. """ def __init__(self, client, cluster_id, status): self.client = client self.cluster_id = cluster_id self.status = status def get_raw(self): """ Gets the whole status as a raw dictionary. :return: status information, with fields: * **state** : current state of the cluster. Possible values are NONE, STARTING, RUNNING, STOPPING * **clusterType** : type of cluster. Can be "manual" or a plugin cluster element type * **usages** : list of usages of the cluster. Each usage is a dict with either **projectKey** (when cluster is set in the project's settings), or **scenarioId**, **scenarioProjectKey** and **scenarioRunId** (when the cluster is created and used by a scenario run) * **otherProjectUsagesCount** : number of projects that use the cluster but that the user cannot access (these projects are not in **usages**) * **otherScenarioUsagesCount** : number of scenarios that use the cluster but that the user cannot access (these scenarios are not in **usages**) * **error** : if the cluster start failed, a dict with error information :rtype: dict """ return self.status class DSSInstanceVariables(dict): """ Dict containing the instance variables. The variables can be modified directly in the dict and persisted using its :meth:`save` method. .. important:: Do not instantiate directly, use :meth:`dataikuapi.DSSClient.get_global_variables` instead. """ def __init__(self, client, variables): super(dict, self).__init__() self.update(variables) self.client = client def save(self): """ Save the changes made to the instance variables. .. note:: This call requires an API key with admin rights. """ return self.client._perform_empty("PUT", "/admin/variables/", body=self) class DSSGlobalUsageSummary(object): """ The summary of the usage of the DSS instance. .. important:: Do not instantiate directly, use :meth:`dataikuapi.DSSClient.get_global_usage_summary` instead. """ def __init__(self, data): self.data = data @property def raw(self): """ Get the usage summary structure The summary report has top-level fields per object type, like **projectSummaries** or **datasets**, each containing counts, usually a **all** global count, then several **XXXXByType** dict with counts by object sub-type (for example, for datasets the sub-type would be the type of the connection they're using) :rtype: dict """ return self.data @property def projects_count(self): """ Get the number of projects on the instance :rtype: int """ return self.data["projects"] @property def total_datasets_count(self): """ Get the number of datasets on the instance :rtype: int """ return self.data["datasets"]["all"] @property def total_recipes_count(self): """ Get the number of recipes on the instance :rtype: int """ return self.data["recipes"]["all"] @property def total_jupyter_notebooks_count(self): """ Get the number of code nobteooks on the instance :rtype: int """ return self.data["notebooks"]["nbJupyterNotebooks"] @property def total_sql_notebooks_count(self): """ Get the number of sql notebooks on the instance :rtype: int """ return self.data["notebooks"]["nbSqlNotebooks"] @property def total_scenarios_count(self): """ Get the number of scenarios on the instance :rtype: int """ return self.data["scenarios"]["all"] @property def total_active_with_trigger_scenarios_count(self): """ Get the number of active scenarios on the instance :rtype: int """ return self.data["scenarios"]["activeWithTriggers"] class DSSCodeStudioTemplateListItem(object): """ An item in a list of code studio templates. .. important:: Do not instantiate directly, use :meth:`dataikuapi.DSSClient.list_code_studio_templates` """ def __init__(self, client, data): self.client = client self._data = data def to_code_studio_template(self): """ Get the handle corresponding to this code studio template :rtype: :class:`DSSCodeStudioTemplate` """ return DSSCodeStudioTemplate(self.client, self._data["id"]) @property def label(self): """ Get the label of the template :rtype: string """ return self._data["label"] @property def id(self): """ Get the identifier of the template :rtype: string """ return self._data["id"] @property def build_for_configs(self): """ Get the list of container configurations this template is built for :return: a list of configuration name :rtype: list[string] """ return self._data.get("buildFor", []) @property def last_built(self): """ Get the timestamp of the last build of the template :return: a timestamp, or None if the template was never built :rtype: :class:`datetime.datetime` """ ts = self._data.get("lastBuilt", 0) if ts > 0: return datetime.fromtimestamp(ts / 1000) else: return None class DSSCodeStudioTemplate(object): """ A handle to interact with a code studio template on the DSS instance .. important:: Do not instantiate directly, use :meth:`dataikuapi.DSSClient.get_code_studio_template`. """ def __init__(self, client, template_id): self.client = client self.template_id = template_id ######################################################## # Template description ######################################################## def get_settings(self): """ Get the template's settings. :return: a :class:`DSSCodeStudioTemplateSettings` object to interact with code studio template settings :rtype: :class:`DSSCodeStudioTemplateSettings` """ settings = self.client._perform_json("GET", "/admin/code-studios/%s" % (self.template_id)) return DSSCodeStudioTemplateSettings(self.client, self.template_id, settings) ######################################################## # Building ######################################################## def build(self): """ Build or rebuild the template. .. note:: This call needs an API key which has an user to impersonate set, or a personal API key. :return: a handle to the task of building the image :rtype: :class:`~dataikuapi.dss.future.DSSFuture` """ future_response = self.client._perform_json("POST", "/admin/code-studios/%s/build" % (self.template_id)) return DSSFuture(self.client, future_response.get('jobId', None), future_response) class DSSCodeStudioTemplateSettings(object): """ The settings of a code studio template .. important:: Do not instantiate directly, use :meth:`DSSCodeStudioTemplate.get_settings` """ def __init__(self, client, template_id, settings): self.client = client self.template_id = template_id self.settings = settings def get_raw(self): """ Gets all settings as a raw dictionary. :return: a reference to the raw settings, not a copy. Keys are * **id** : unique identifier of the template * **type** : type of the template. Builtin values are 'manual' and 'block_based'; more types can be added via plugin components * **label** : label of the template in the UI * **icon** : icon to use for code studios on this template * **tags** : list of tags (strings) * **isEditor** : whether the template defines a code studio that can be used to edit objects in DSS * **owner**, **defaultPermission** and **permissions** : definition of the permissions on the template * **defaultContainerConf** : container config to use on code studios created on this template * **allowContainerConfOverride** : if True, the container config of code studios on this template can be overriden at the project level * **allContainerConfs** : if True, build the container images for all configs of the instance * **containerConfs** : if **allContainerConfs** is False, a list of container config names to build images for * **params** : definition of the contents of the template. Depends on the **type** :rtype: dict """ return self.settings def get_built_for_all_container_confs(self): """ Whether the template an image for each container config :rtype: boolean """ return self.settings.get("allContainerConfs", False) def get_built_container_confs(self): """ Get the list of container configs for which the template builds an image (if not all) :return: a list of container configuration names :rtype: list[string] """ return self.settings.get("containerConfs", []) def set_built_container_confs(self, *configs, **kwargs): """ Set the list of container configs for which the template builds an image :param boolean all: if True, an image is built for each config :param list[string] configs: list of configuration names to build images for """ all = kwargs.get("all", False) self.settings['allContainerConfs'] = all if not all: self.settings['containerConfs'] = configs def save(self): """ Saves the settings of the code studio template """ self.client._perform_empty("PUT", "/admin/code-studios/%s" % (self.template_id), body=self.settings) dataiku-api-client-python-11.4.0/dataikuapi/dss/analysis.py000066400000000000000000000406661440464736400237130ustar00rootroot00000000000000from ..utils import DataikuException from ..utils import DataikuUTF8CSVReader from ..utils import DataikuStreamedHttpUTF8CSVReader import json import time from .metrics import ComputedMetrics from .ml import DSSMLTask from .utils import DSSDatasetSelectionBuilder class DSSAnalysisStepBuilder(object): def __init__(self, step_type=None, step_name=None): self.step = {'metaType':'PROCESSOR', 'type':step_type, 'name':step_name, 'params':{}} def build(self): """Returns the built step dict""" return self.step def with_type(self, step_type): """Sets the step's type""" self.step["type"] = step_type return self def with_name(self, step_name): """Sets the step's name""" self.step["name"] = step_name return self class DSSAnalysisDefinition(): """ Object to manipulate the definition of a visual analysis """ def __init__(self, analysis, acp): self.analysis = analysis self.acp = acp def get_raw(self): """ Gets the raw dictionary of the visual analysis definition """ return self.acp def get_raw_script(self): """ Gets the raw dictionary of visual analysis' script settings (including steps, sampling, ...) """ acp = self.get_raw() if not 'script' in acp: acp['script'] = {'steps':[]} return acp['script'] def get_raw_script_steps(self): """ Gets the raw dictionary of visual analysis' script steps """ script = self.get_raw_script() if not 'steps' in script: script['steps'] = [] return script['steps'] def get_raw_script_sampling(self): """ Gets the raw dictionary of visual analysis' script sampling """ script = self.get_raw_script() if not 'explorationSampling' in script: script['explorationSampling'] = {} return script['explorationSampling'] def save(self): """ Shortcut to :meth:`DSSAnalysis.set_definition()` """ self.analysis.set_definition(self) def add_step(self, step): """ Add a step to the script :param object selection: A :class:`DSSAnalysisStepBuilder` to build the settings of the step. """ steps = self.get_raw_script_steps() if isinstance(step, DSSAnalysisStepBuilder): steps.append(step.build()) else: steps.append(step) def set_script_sampling_selection(self, selection): """ Sets the sampling for the script :param object selection: A :class:`DSSDatasetSelectionBuilder` to build the settings of the extract of the dataset. """ sampling = self.get_raw_script_sampling() if isinstance(selection, DSSDatasetSelectionBuilder): sampling['selection'] = selection.build() else: sampling['selection'] = selection class DSSAnalysis(object): """A handle to interact with a DSS visual analysis""" def __init__(self, client, project_key, analysis_id): self.client = client self.project_key = project_key self.analysis_id = analysis_id ######################################################## # Analysis deletion ######################################################## def delete(self, drop_data=False): """ Delete the dataset :param bool drop_data: Should the data of the dataset be dropped """ return self.client._perform_empty("DELETE", "/projects/%s/lab/%s/" % (self.project_key, self.analysis_id)) ######################################################## # Analysis definition ######################################################## def get_definition(self): """ Get the definition of the analysis :return: a DSSAnalysisDefinition object to interact with the settings :rtype: :class:`dataikuapi.dss.analysis.DSSAnalysisDefinition` """ acp = self.client._perform_json("GET", "/projects/%s/lab/%s/" % (self.project_key, self.analysis_id)) return DSSAnalysisDefinition(self, acp) def set_definition(self, definition): """ Set the definition of the analysis Args: definition: the definition, as a JSON object or a :class:`dataikuapi.dss.analysis.DSSAnalysisDefinition`. You should only set a definition object that has been retrieved using the get_definition call. """ if isinstance(definition, DSSAnalysisDefinition): acp = definition.get_raw() else: acp = definition return self.client._perform_json("PUT", "/projects/%s/lab/%s/" % (self.project_key, self.analysis_id), body=acp) ######################################################## # ML ######################################################## def create_prediction_ml_task(self, target_variable, ml_backend_type="PY_MEMORY", guess_policy="DEFAULT", prediction_type=None, wait_guess_complete=True): """Creates a new prediction task in this visual analysis lab for a dataset. :param string target_variable: Variable to predict :param string ml_backend_type: ML backend to use, one of PY_MEMORY, MLLIB or H2O :param string guess_policy: Policy to use for setting the default parameters. Valid values are: DEFAULT, SIMPLE_FORMULA, DECISION_TREE, EXPLANATORY and PERFORMANCE :param string prediction_type: The type of prediction problem this is. If not provided the prediction type will be guessed. Valid values are: BINARY_CLASSIFICATION, REGRESSION, MULTICLASS :param boolean wait_guess_complete: if False, the returned ML task will be in 'guessing' state, i.e. analyzing the input dataset to determine feature handling and algorithms. You should wait for the guessing to be completed by calling ``wait_guess_complete`` on the returned object before doing anything else (in particular calling ``train`` or ``get_settings``) :return :class dataiku.dss.ml.DSSMLTask """ obj = { "taskType" : "PREDICTION", "targetVariable" : target_variable, "backendType": ml_backend_type, "guessPolicy": guess_policy } if prediction_type is not None: obj["predictionType"] = prediction_type ref = self.client._perform_json("POST", "/projects/%s/lab/%s/models/" % (self.project_key, self.analysis_id), body=obj) mltask = DSSMLTask(self.client, self.project_key, self.analysis_id, ref["mlTaskId"]) if wait_guess_complete: mltask.wait_guess_complete() return mltask def create_clustering_ml_task(self, ml_backend_type="PY_MEMORY", guess_policy="KMEANS", wait_guess_complete=True): """Creates a new clustering task in this visual analysis lab for a dataset. The returned ML task will be in 'guessing' state, i.e. analyzing the input dataset to determine feature handling and algorithms. You should wait for the guessing to be completed by calling ``wait_guess_complete`` on the returned object before doing anything else (in particular calling ``train`` or ``get_settings``) :param string ml_backend_type: ML backend to use, one of PY_MEMORY, MLLIB or H2O :param string guess_policy: Policy to use for setting the default parameters. Valid values are: KMEANS and ANOMALY_DETECTION :param boolean wait_guess_complete: if False, the returned ML task will be in 'guessing' state, i.e. analyzing the input dataset to determine feature handling and algorithms. You should wait for the guessing to be completed by calling ``wait_guess_complete`` on the returned object before doing anything else (in particular calling ``train`` or ``get_settings``) """ obj = { "taskType" : "CLUSTERING", "backendType": ml_backend_type, "guessPolicy": guess_policy } ref = self.client._perform_json("POST", "/projects/%s/lab/%s/models/" % (self.project_key, self.analysis_id), body=obj) mltask = DSSMLTask(self.client, self.project_key, self.analysis_id, ref["mlTaskId"]) if wait_guess_complete: mltask.wait_guess_complete() return mltask def create_timeseries_forecasting_ml_task(self, target_variable, time_variable, timeseries_identifiers=None, guess_policy="TIMESERIES_DEFAULT", wait_guess_complete=True): """Creates a new time series forecasting task in this visual analysis lab for a dataset. :param string target_variable: The variable to forecast :param string time_variable: Column to be used as time variable. Should be a Date (parsed) column. :param list timeseries_identifiers: List of columns to be used as time series identifiers (when the dataset has multiple series) :param string guess_policy: Policy to use for setting the default parameters. Valid values are: TIMESERIES_DEFAULT, TIMESERIES_STATISTICAL, and TIMESERIES_DEEP_LEARNING :param boolean wait_guess_complete: If False, the returned ML task will be in 'guessing' state, i.e. analyzing the input dataset to determine feature handling and algorithms. You should wait for the guessing to be completed by calling ``wait_guess_complete`` on the returned object before doing anything else (in particular calling ``train`` or ``get_settings``) :return :class dataiku.dss.ml.DSSMLTask """ obj = { "taskType": "PREDICTION", "targetVariable": target_variable, "timeVariable": time_variable, "timeseriesIdentifiers": timeseries_identifiers, "backendType": "PY_MEMORY", "guessPolicy": guess_policy, "predictionType": "TIMESERIES_FORECAST" } ref = self.client._perform_json( "POST", "/projects/{project_key}/lab/{analysis_id}/models/".format(project_key=self.project_key, analysis_id=self.analysis_id), body=obj ) mltask = DSSMLTask(self.client, self.project_key, self.analysis_id, ref["mlTaskId"]) if wait_guess_complete: mltask.wait_guess_complete() return mltask def list_ml_tasks(self): """ List the ML tasks in this visual analysis Returns: the list of the ML tasks summaries, each one as a JSON object """ return self.client._perform_json("GET", "/projects/%s/lab/%s/models/" % (self.project_key, self.analysis_id)) def get_ml_task(self, mltask_id): """ Get a handle to interact with a specific ML task Args: mltask_id: the identifier of the desired ML task Returns: A :class:`dataikuapi.dss.ml.DSSMLTask` ML task handle """ return DSSMLTask(self.client, self.project_key, self.analysis_id, mltask_id) # some basic steps class DSSFormulaStepBuilder(DSSAnalysisStepBuilder): def __init__(self, step_name=None): super(DSSFormulaStepBuilder, self).__init__(step_type='CreateColumnWithGREL', step_name=step_name) def with_output_column(self, column_name): """Sets the step's output column's name""" self.step["params"]["column"] = column_name return self def with_error_column(self, column_name): """Sets the step's output column's name""" self.step["params"]["errorColumn"] = column_name return self def with_expression(self, expression): """Sets the step's expression""" self.step["params"]["expression"] = expression return self class AppliesToStepBuilder(DSSAnalysisStepBuilder): def __init__(self, step_type=None, step_name=None): super(AppliesToStepBuilder, self).__init__(step_type=step_type, step_name=step_name) self.step["params"]["appliesTo"] = 'SINGLE_COLUMN' def with_column_selection_mode(self, column_selection_mode): """Sets the step's column selection mode (SINGLE_COLUMN, COLUMNS, PATTERN, ALL)""" self.step["params"]["appliesTo"] = column_selection_mode return self def with_columns(self, *column_names): """Sets the step's selected columns""" self.step["params"]["columns"] = [c for c in column_names] return self def with_column_regex(self, regex): """Sets the step's column selection regular expression""" self.step["params"]["appliesToPattern"] = regex return self def with_single_column_selection(self, column_name): """Sets the step's as applying to a single column""" return self.with_column_selection_mode('SINGLE_COLUMN').with_columns(column_name) def with_multiple_column_selection(self, *column_names): """Sets the step's as applying to a single column""" return self.with_column_selection_mode('COLUMNS').with_columns(column_names) def with_regex_column_selection(self, regex): """Sets the step's as applying to a single column""" return self.with_column_selection_mode('PATTERN').with_column_regex(regex) def with_all_column_selection(self, column_name): """Sets the step's as applying to all columns""" return self.with_column_selection_mode('ALL') class FilterAndFlagStepBuilder(AppliesToStepBuilder): def __init__(self, step_type=None, step_name=None): super(FilterAndFlagStepBuilder, self).__init__(step_type=step_type, step_name=step_name) self.step["params"]["booleanMode"] = 'AND' self.step["params"]["action"] = 'REMOVE_ROW' def with_action(self, action): """Sets the step's action on match (KEEP_ROW, REMOVE_ROW, CLEAR_CELL, DONTCLEAR_CELL, FLAG)""" self.step["params"]["action"] = action return self def with_boolean_mode(self, boolean_mode): """Sets the step's mode for combining matches in different columns (AND, OR)""" self.step["params"]["booleanMode"] = boolean_mode return self def with_flag_column(self, column_name): """Sets the step's column for outputing the flag""" self.step["params"]["flagColumn"] = column_name return self class FilterOnValueStepBuilder(FilterAndFlagStepBuilder): def __init__(self, step_name=None): super(FilterOnValueStepBuilder, self).__init__(step_type='FlagOnValue', step_name=step_name) def with_values(self, *values): """Sets the step's flagged values""" self.step["params"]["values"] = [v for v in values] return self def with_matching_mode(self, matching_mode): """Sets the step's matching_mode (FULL_STRING, SUBSTRING, PATTERN)""" self.step["params"]["matchingMode"] = matching_mode return self def with_normalization_mode(self, normalization_mode): """Sets the step's normalization_mode (EXACT, LOWERCASE, NORMALIZED)""" self.step["params"]["normalizationMode"] = normalization_mode return self class FilterOnBadTypeStepBuilder(FilterAndFlagStepBuilder): def __init__(self, step_name=None): super(FilterOnBadTypeStepBuilder, self).__init__(step_type='FilterOnBadType', step_name=step_name) def with_meaning(self, meaning): """Sets the step's meaning to check""" self.step["params"]["type"] = meaning return self class RemoveRowsStepBuilder(AppliesToStepBuilder): def __init__(self, step_name=None): super(RemoveRowsStepBuilder, self).__init__(step_type='RemoveRowsOnEmpty', step_name=step_name) def with_meaning(self, keep): """Sets the step's behavior when an empty value is found : True=keep, False=drop (default)""" self.step["params"]["keep"] = keep return self dataiku-api-client-python-11.4.0/dataikuapi/dss/apideployer.py000066400000000000000000000561441440464736400244030ustar00rootroot00000000000000import json from .future import DSSFuture from ..utils import CallableStr class DSSAPIDeployer(object): """ Handle to interact with the API Deployer. Do not create this directly, use :meth:`dataikuapi.dss.DSSClient.get_apideployer` """ def __init__(self, client): self.client = client def list_deployments(self, as_objects=True): """ Lists deployments on the API Deployer :param boolean as_objects: if True, returns a list of :class:`DSSAPIDeployerDeployment`, else returns a list of dict. Each dict contains at least a field "id" indicating the identifier of this deployment :returns: a list - see as_objects for more information :rtype: list """ l = self.client._perform_json("GET", "/api-deployer/deployments") if as_objects: return [DSSAPIDeployerDeployment(self.client, x["deploymentBasicInfo"]["id"]) for x in l] else: return l def get_deployment(self, deployment_id): """ Returns a handle to interact with a single deployment, as a :class:`DSSAPIDeployerDeployment` :param str deployment_id: Identifier of the deployment to get :rtype: :class:`DSSAPIDeployerDeployment` """ return DSSAPIDeployerDeployment(self.client, deployment_id) def create_deployment(self, deployment_id, service_id, infra_id, version, ignore_warnings=False): """ Creates a deployment and returns the handle to interact with it. The returned deployment is not yet started and you need to call :meth:`~DSSAPIDeployerDeployment.start_update` :param str deployment_id: Identifier of the deployment to create :param str service_id: Identifier of the API Service to target :param str infra_id: Identifier of the deployment infrastructure to use :param str version_id: Identifier of the API Service version to deploy :param boolean ignore_warnings: ignore warnings concerning the governance status of the model version(s) to deploy :rtype: :class:`DSSAPIDeployerDeployment` """ settings = { "deploymentId" : deployment_id, "publishedServiceId" : service_id, "infraId" : infra_id, "version" : version } self.client._perform_json("POST", "/api-deployer/deployments", params={"ignoreWarnings": ignore_warnings}, body=settings) return self.get_deployment(deployment_id) def list_stages(self): """ Lists infrastructure stages of the API Deployer :rtype: list of dict. Each dict contains a field "id" for the stage identifier and "desc" for its description. :rtype: list """ return self.client._perform_json("GET", "/api-deployer/stages") def list_infras(self, as_objects=True): """ Lists deployment infrastructures on the API Deployer :param boolean as_objects: if True, returns a list of :class:`DSSAPIDeployerInfra`, else returns a list of dict. Each dict contains at least a field "id" indicating the identifier of this infra :returns: a list - see as_objects for more information :rtype: list """ l = self.client._perform_json("GET", "/api-deployer/infras") if as_objects: return [DSSAPIDeployerInfra(self.client, x["infraBasicInfo"]["id"]) for x in l] else: return l def create_infra(self, infra_id, stage, type, govern_check_policy="NO_CHECK"): """ Creates a new infrastructure on the API Deployer and returns the handle to interact with it. :param str infra_id: Unique Identifier of the infra to create :param str stage: Infrastructure stage. Stages are configurable on each API Deployer :param str type: STATIC or KUBERNETES :param str govern_check_policy: PREVENT, WARN, or NO_CHECK depending if the deployer will check wether the saved model versions deployed on this infrastructure has to be managed and approved in Dataiku Govern :rtype: :class:`DSSAPIDeployerInfra` """ settings = { "id": infra_id, "stage": stage, "type": type, "governCheckPolicy": govern_check_policy, } self.client._perform_json("POST", "/api-deployer/infras", body=settings) return self.get_infra(infra_id) def get_infra(self, infra_id): """ Returns a handle to interact with a single deployment infra, as a :class:`DSSAPIDeployerInfra` :param str infra_id: Identifier of the infra to get :rtype: :class:`DSSAPIDeployerInfra` """ return DSSAPIDeployerInfra(self.client, infra_id) def list_services(self, as_objects=True): """ Lists API services on the API Deployer :param boolean as_objects: if True, returns a list of :class:`DSSAPIDeployerService`, else returns a list of dict. Each dict contains at least a field "id" indicating the identifier of this Service :returns: a list - see as_objects for more information :rtype: list """ l = self.client._perform_json("GET", "/api-deployer/services") if as_objects: return [DSSAPIDeployerService(self.client, x["serviceBasicInfo"]["id"]) for x in l] else: return l def create_service(self, service_id): """ Creates a new API Service on the API Deployer and returns the handle to interact with it. :param str service_id: Identifier of the API Service to create :rtype: :class:`DSSAPIDeployerService` """ settings = { "publishedServiceId" : service_id } self.client._perform_json("POST", "/api-deployer/services", body=settings) return self.get_service(service_id) def get_service(self, service_id): """ Returns a handle to interact with a single service, as a :class:`DSSAPIDeployerService` :param str service_id: Identifier of the API service to get :rtype: :class:`DSSAPIDeployerService` """ return DSSAPIDeployerService(self.client, service_id) ############################################### # Infrastructures ############################################### class DSSAPIDeployerInfra(object): """ An API Deployer infrastructure. Do not create this directly, use :meth:`~dataikuapi.dss.apideployer.DSSAPIDeployer.get_infra` """ def __init__(self, client, infra_id): self.client = client self.infra_id = infra_id @property def id(self): return CallableStr(self.infra_id) def get_status(self): """ Returns status information about this infrastructure :rtype: :class:`dataikuapi.dss.apideployer.DSSAPIDeployerInfraStatus` """ light = self.client._perform_json("GET", "/api-deployer/infras/%s" % (self.infra_id)) return DSSAPIDeployerInfraStatus(self.client, self.infra_id, light) def get_settings(self): """ Gets the settings of this infra. If you want to modify the settings, you need to call :meth:`~dataikuapi.dss.apideployer.DSSAPIDeployerInfraSettings.save` on the returned object :returns: a :class:`dataikuapi.dss.apideployer.DSSAPIDeployerInfraSettings` """ settings = self.client._perform_json( "GET", "/api-deployer/infras/%s/settings" % (self.infra_id)) return DSSAPIDeployerInfraSettings(self.client, self.infra_id, settings) def delete(self): """ Deletes this infra You may only delete an infra if it has no deployments on it anymore. """ self.client._perform_empty( "DELETE", "/api-deployer/infras/%s" % (self.infra_id)) class DSSAPIDeployerInfraSettings(object): """ The settings of an API Deployer infrastructure Do not create this directly, use :meth:`~dataikuapi.dss.apideployer.DSSAPIDeployerInfra.get_settings` """ def __init__(self, client, infra_id, settings): self.client = client self.infra_id = infra_id self.settings = settings def get_type(self): """Gets the type of this infra, either STATIC or K8S""" return self.settings["type"] def add_apinode(self, url, api_key, graphite_prefix=None): """Adds an API node to the list of nodes of this infra. Only applicable to STATIC infrastructures""" new_node = { "url": url, "adminAPIKey" : api_key, "graphitePrefix" : graphite_prefix } self.settings["apiNodes"].append(new_node) def remove_apinode(self, node_url): """ Removes a node from the list of nodes of this infra. Only applicable to STATIC infrastructures :param str node_url: URL of the node to remove """ api_nodes = list(self.settings["apiNodes"]) for node in api_nodes: if node.get("url") == node_url: self.settings["apiNodes"].remove(node) def get_raw(self): """ Gets the raw settings of this infra. This returns a reference to the raw settings, not a copy, so changes made to the returned object will be reflected when saving. :rtype: dict """ return self.settings def save(self): """Saves back these settings to the infra""" self.client._perform_empty( "PUT", "/api-deployer/infras/%s/settings" % (self.infra_id), body = self.settings) class DSSAPIDeployerInfraStatus(object): """ The status of an API Deployer infrastructure. Do not create this directly, use :meth:`~dataikuapi.dss.apideployer.DSSAPIDeployerInfra.get_status` """ def __init__(self, client, infra_id, light_status): self.client = client self.infra_id = infra_id self.light_status = light_status def get_deployments(self): """ Returns the deployments that are deployed on this infrastructure :returns: a list of deployments :rtype: list of :class:`dataikuapi.dss.apideployer.DSSAPIDeployerDeployment` """ return [DSSAPIDeployerDeployment(self.client, deployment["id"]) for deployment in self.light_status["deployments"]] def get_raw(self): """ Gets the raw status information. This returns a dictionary with various information about the infrastructure :rtype: dict """ return self.light_status ############################################### # Deployments ############################################### class DSSAPIDeployerDeployment(object): """ A Deployment on the API Deployer Do not create this directly, use :meth:`~dataikuapi.dss.apideployer.DSSAPIDeployer.get_deployment` """ def __init__(self, client, deployment_id): self.client = client self.deployment_id = deployment_id @property def id(self): return CallableStr(self.deployment_id) def get_status(self): """ Returns status information about this deployment :rtype: dataikuapi.dss.apideployer.DSSAPIDeployerDeploymentStatus """ light = self.client._perform_json("GET", "/api-deployer/deployments/%s" % (self.deployment_id)) heavy = self.client._perform_json("GET", "/api-deployer/deployments/%s/status" % (self.deployment_id)) return DSSAPIDeployerDeploymentStatus(self.client, self.deployment_id, light, heavy) def get_governance_status(self, version=""): """ Returns the governance status about this deployment if applicable It covers all the embedded model versions :param str version: (Optional) The specific package version of the published service to get status from. If empty, consider all the versions used in the deployment generation mapping. :rtype: dict InforMessages containing the governance status """ return self.client._perform_json("GET", "/api-deployer/deployments/%s/governance-status" % (self.deployment_id), params={ "version": version }) def get_settings(self): """ Gets the settings of this deployment. If you want to modify the settings, you need to call :meth:`~dataikuapi.dss.apideployer.DSSAPIDeployerDeploymentSettings.save` on the returned object :returns: a :class:`dataikuapi.dss.apideployer.DSSAPIDeployerDeploymentSettings` """ settings = self.client._perform_json( "GET", "/api-deployer/deployments/%s/settings" % (self.deployment_id)) return DSSAPIDeployerDeploymentSettings(self.client, self.deployment_id, settings) def start_update(self): """ Starts an asynchronous update of this deployment to try to match the actual state to the current settings :returns: a :class:`dataikuapi.dss.future.DSSFuture` tracking the progress of the update. Call :meth:`~dataikuapi.dss.future.DSSFuture.wait_for_result` on the returned object to wait for completion (or failure) """ future_response = self.client._perform_json( "POST", "/api-deployer/deployments/%s/actions/update" % (self.deployment_id)) return DSSFuture(self.client, future_response.get('jobId', None), future_response) def delete(self, disable_first=False): """ Deletes this deployment. The disable_first flag automatically disables the deployment before its deletion. :param boolean disable_first: If True, automatically disables this deployment before deleting it. If False, will raise an Exception if this deployment is enabled. """ # Check if the deployment is disabled is_enabled = self.get_status().light_status["deploymentBasicInfo"].get("enabled") if is_enabled and not disable_first: raise Exception("Deployment {} deletion failed: deployment must be disabled first.".format(self.deployment_id)) if is_enabled: settings = self.get_settings() settings.set_enabled(enabled=False) settings.save() self.client._perform_empty( "DELETE", "/api-deployer/deployments/%s" % (self.deployment_id)) class DSSAPIDeployerDeploymentSettings(object): """ The settings of an API Deployer deployment. Do not create this directly, use :meth:`~dataikuapi.dss.apideployer.DSSAPIDeployerDeployment.get_settings` """ def __init__(self, client, deployment_id, settings): self.client = client self.deployment_id = deployment_id self.settings = settings def get_raw(self): """ Gets the raw settings of this deployment. This returns a reference to the raw settings, not a copy, so changes made to the returned object will be reflected when saving. :rtype: dict """ return self.settings def set_enabled(self, enabled): """ Enables or disables this deployment """ self.settings["enabled"] = enabled def set_single_version(self, version): """ Sets this deployment to target a single version of the API service :param str version: Identifier of the version to set """ self.settings["generationsMapping"] = { "mode": "SINGLE_GENERATION", "generation": version } def save(self, ignore_warnings=False): """ Saves back these settings to the deployment :param boolean ignore_warnings: ignore warnings concerning the governance status of the model version(s) to deploy """ self.client._perform_empty( "PUT", "/api-deployer/deployments/%s/settings" % (self.deployment_id), params = { "ignoreWarnings" : ignore_warnings }, body = self.settings) class DSSAPIDeployerDeploymentStatus(object): """ The status of an API Deployer deployment. Do not create this directly, use :meth:`~dataikuapi.dss.apideployer.DSSAPIDeployerDeployment.get_status` """ def __init__(self, client, deployment_id, light_status, heavy_status): self.client = client self.deployment_id = deployment_id self.light_status = light_status self.heavy_status = heavy_status def get_light(self): """ Gets the 'light' (summary) status. This returns a dictionary with various information about the deployment, but not the actual health of the deployment :rtype: dict """ return self.light_status def get_heavy(self): """ Gets the 'heavy' (full) status. This returns a dictionary with various information about the deployment :rtype: dict """ return self.heavy_status def get_service_urls(self): """ Returns service-level URLs for this deployment (ie without the enpdoint-specific suffix) """ if "deployedServiceId" in self.light_status["deploymentBasicInfo"]: service_id = self.light_status["deploymentBasicInfo"]["deployedServiceId"] else: service_id = self.light_status["deploymentBasicInfo"]["publishedServiceId"] if "apiNodes" in self.heavy_status: return ["%s/public/api/v1/%s" % (x["url"], service_id) for x in self.heavy_status["apiNodes"]] elif "publicURL" in self.heavy_status: return ["%s/public/api/v1/%s" % (self.heavy_status["publicURL"], service_id)] else: raise ValueError("PublicURL not available for this deployment. It might still be initializing") def get_health(self): """ Returns the health of this deployment as a string :returns: HEALTHY if the deployment is working properly, various other status otherwise :rtype: string """ return self.heavy_status["health"] def get_health_messages(self): """Returns messages about the health of this deployment""" return self.heavy_status["healthMessages"] ############################################### # Published Service ############################################### class DSSAPIDeployerService(object): """ An API service on the API Deployer Do not create this directly, use :meth:`~dataikuapi.dss.apideployer.DSSAPIDeployer.get_service` """ def __init__(self, client, service_id): self.client = client self.service_id = service_id @property def id(self): return CallableStr(self.service_id) def get_status(self): """ Returns status information about this service. This is used mostly to get information about which versions are available and which deployments are exposing this service :rtype: dataikuapi.dss.apideployer.DSSAPIDeployerServiceStatus """ light = self.client._perform_json("GET", "/api-deployer/services/%s" % (self.service_id)) return DSSAPIDeployerServiceStatus(self.client, self.service_id, light) def import_version(self, fp): """ Imports a new version for an API service from a file-like object pointing to a version package Zip file :param string fp: A file-like object pointing to a version package Zip file """ return self.client._perform_empty("POST", "/api-deployer/services/%s/versions" % (self.service_id), files={"file":fp}) def get_settings(self): """ Gets the settings of this service. If you want to modify the settings, you need to call :meth:`~dataikuapi.dss.apideployer.DSSAPIDeployerServiceSettings.save` on the returned object. The main things that can be modified in a service settings are permissions :returns: a :class:`dataikuapi.dss.apideployer.DSSAPIDeployerServiceSettings` """ settings = self.client._perform_json( "GET", "/api-deployer/services/%s/settings" % (self.service_id)) return DSSAPIDeployerServiceSettings(self.client, self.service_id, settings) def delete_version(self, version): """ Deletes a version from this service :param string version: The version to delete """ self.client._perform_empty( "DELETE", "/api-deployer/services/%s/versions/%s" % (self.service_id, version)) def delete(self): """ Deletes this service You may only delete a service if it has no deployments on it anymore. """ self.client._perform_empty( "DELETE", "/api-deployer/services/%s" % (self.service_id)) class DSSAPIDeployerServiceSettings(object): """ The settings of an API Deployer Service. Do not create this directly, use :meth:`~dataikuapi.dss.apideployer.DSSAPIDeployerService.get_settings` """ def __init__(self, client, service_id, settings): self.client = client self.service_id = service_id self.settings = settings def get_raw(self): """ Gets the raw settings of this deployment. This returns a reference to the raw settings, not a copy, so changes made to the returned object will be reflected when saving. :rtype: dict """ return self.settings def save(self): """ Saves back these settings to the API service """ self.client._perform_empty( "PUT", "/api-deployer/services/%s/settings" % (self.service_id), body = self.settings) class DSSAPIDeployerServiceStatus(object): """ The status of an API Deployer Service. Do not create this directly, use :meth:`~dataikuapi.dss.apideployer.DSSAPIDeployerService.get_status` """ def __init__(self, client, service_id, light_status): self.client = client self.service_id = service_id self.light_status = light_status def get_deployments(self, infra_id=None): """ Returns the deployments that have been created from this published project :param str infra_id: Identifier of an infra, allows to only keep in the returned list the deployments on this infra. If not set, the list contains all the deployments using this published project, across every infra of the Project Deployer. :returns: a list of deployments :rtype: list of :class:`dataikuapi.dss.apideployer.DSSAPIDeployerDeployment` """ if infra_id is None: return [DSSAPIDeployerDeployment(self.client, deployment["id"]) for deployment in self.light_status["deployments"]] return [DSSAPIDeployerDeployment(self.client, deployment["id"]) for deployment in self.light_status["deployments"] if infra_id == deployment["infraId"]] def get_versions(self): """ Returns the versions of this service that have been published on the API Service Each version is a dict that contains at least a "id" field, which is the version identifier :returns: a list of versions, each as a dict containing a "id" field :rtype: list of dicts """ return self.light_status["packages"] def get_raw(self): """ Gets the raw status information. This returns a dictionary with various information about the service, :rtype: dict """ return self.light_status dataiku-api-client-python-11.4.0/dataikuapi/dss/apiservice.py000066400000000000000000000270211440464736400242100ustar00rootroot00000000000000import json class DSSAPIServiceListItem(dict): """ An item in a list of API services. .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.list_api_services()` """ def __init__(self, client, data): super(DSSAPIServiceListItem, self).__init__(data) self.client = client def to_api_service(self): """ Get a handle corresponding to this API service. :rtype: :class:`DSSAPIService` """ return DSSAPIService(self.client, self["projectKey"], self["id"]) @property def name(self): """ Get the name of the API service. :rtype: string """ return self["name"] @property def id(self): """ Get the identifier of the API service. :rtype: string """ return self["id"] @property def auth_method(self): """ Get the method used to authenticate on the API service. Usage example: .. code-block:: python # list all public API services for service in project.list_api_services(as_type="list_item"): if service.auth_method == 'PUBLIC': print("Service {} isn't authenticating requests".format(service.id)) :return: an authentication method. Possible values: PUBLIC (no authentication), API_KEYS, OAUTH2 :rtype: string """ return self["authMethod"] @property def endpoints(self): """ Get the endpoints in this API service. :return: a list of endpoints, each one a dict with fields: * **id** : identifier of the endpoint * **type** : type of endpoint. Possible values: STD_PREDICTION, STD_CLUSTERING, STD_FORECAST, STD_CAUSAL_PREDICTION, CUSTOM_PREDICTION, CUSTOM_R_PREDICTION, R_FUNCTION, PY_FUNCTION, DATASETS_LOOKUP, SQL_QUERY :rtype: list[dict] """ return self["endpoints"] class DSSAPIServiceSettings(object): """ The settings of an API Service in the API Designer. .. important:: Do not instantiate directly, use :meth:`DSSAPIService.get_settings`. """ def __init__(self, client, project_key, service_id, settings): self.client = client self.project_key = project_key self.service_id = service_id self.settings = settings def get_raw(self): """ Get the raw settings of this API Service. This returns a reference to the raw settings, not a copy, so changes made to the returned object will be reflected when saving. :return: the settings of the API service, as a dict. Notable fields are: * **projectKey** and **id** : identifier of the API service * **name** : name of API service in UI * **tags** : list of tags (each a string) * **authMethod** : method used to authenticate calls on the service. Possible values: PUBLIC, API_KEYS, OAUTH2 * **authRealm** : dict of API keys settings, used when **authMethod** is API_KEYS. This dict has a sub-field: * **queryKeys** : list of API keys allowed on queries, each key being a string * **oauth2Config** : dict of OAuth2 settings, used when **authMethod** is OAUTH2 * **endpoints** : list of endpoints, each one a dict. Endpoint have different fields depending on their type, but always have at least: * **id** : identifier of the endpoint * **type** : type of endpoint. Possible values: STD_PREDICTION, STD_CLUSTERING, STD_FORECAST, STD_CAUSAL_PREDICTION, CUSTOM_PREDICTION, CUSTOM_R_PREDICTION, R_FUNCTION, PY_FUNCTION, DATASETS_LOOKUP, SQL_QUERY :rtype: dict """ return self.settings @property def auth_method(self): """ Get the method used to authenticate on the API service :return: an authentication method. Possible values: PUBLIC (no authentication), API_KEYS, OAUTH2 :rtype: string """ return self.settings["authMethod"] @property def endpoints(self): """ Get the list of endpoints of this API service :return: ist of endpoints, each one a dict. Endpoint have different fields depending on their type, but always have at least: * **id** : identifier of the endpoint * **type** : type of endpoint. Possible values: STD_PREDICTION, STD_CLUSTERING, STD_FORECAST, STD_CAUSAL_PREDICTION, CUSTOM_PREDICTION, CUSTOM_R_PREDICTION, R_FUNCTION, PY_FUNCTION, DATASETS_LOOKUP, SQL_QUERY :rtype: list[dict] """ return self.settings["endpoints"] def add_prediction_endpoint(self, endpoint_id, saved_model_id): """ Add a new "visual prediction" endpoint to this API service. :param string endpoint_id: identifier of the new endpoint to create :param string saved_model_id: identifier of the saved model (that is currently deployed to the Flow) to use """ self.settings["endpoints"].append({ "id" : endpoint_id, "type" : "STD_PREDICTION", "modelRef": saved_model_id }) def add_clustering_endpoint(self, endpoint_id, saved_model_id): """ Add a new "visual clustering" endpoint to this API service. :param string endpoint_id: identifier of the new endpoint to create :param string saved_model_id: identifier of the saved model (that is currently deployed to the Flow) to use """ self.settings["endpoints"].append({ "id" : endpoint_id, "type" : "STD_CLUSTERING", "modelRef": saved_model_id }) def add_forecasting_endpoint(self, endpoint_id, saved_model_id): """ Add a new "visual time series forecasting" endpoint to this API service. :param string endpoint_id: identifier of the new endpoint to create :param string saved_model_id: identifier of the saved model (that is currently deployed to the Flow) to use """ self.settings["endpoints"].append({ "id" : endpoint_id, "type" : "STD_FORECAST", "modelRef": saved_model_id }) def save(self): """ Save back these settings to the API Service. """ self.client._perform_empty( "PUT", "/projects/%s/apiservices/%s/settings" % (self.project_key, self.service_id), body = self.settings) class DSSAPIService(object): """ An API Service from the API Designer on the DSS instance. .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.get_api_service` """ def __init__(self, client, project_key, service_id): self.client = client self.project_key = project_key self.service_id = service_id @property def id(self): """ Get the API service's identifier :rtype: string """ return self.service_id def get_settings(self): """ Get the settings of this API Service. Usage example: .. code-block:: python # list all API services using a given model model_lookup = "my_saved_model_id" model = project.get_saved_model(model_lookup) model_name = model.get_settings().get_raw()["name"] for service in project.list_api_services(as_type='object'): settings = service.get_settings() endpoints_on_model = [e for e in settings.endpoints if e.get("modelRef", '') == model_lookup] if len(endpoints_on_model) > 0: print("Service {} uses model {}".format(service.id, model_name)) :return: a handle on the settings :rtype: :class:`DSSAPIServiceSettings` """ settings = self.client._perform_json( "GET", "/projects/%s/apiservices/%s/settings" % (self.project_key, self.service_id)) return DSSAPIServiceSettings(self.client, self.project_key, self.service_id, settings) def list_packages(self): """ List the versions of this API service. :return: a list of pacakges, each one as a dict. Each dict has fields: * **id** : version (identifier) of the package * **createdOn** : timestamp in milliseconds of when the package was created :rtype: list[dict] """ return self.client._perform_json( "GET", "/projects/%s/apiservices/%s/packages" % (self.project_key, self.service_id)) def create_package(self, package_id): """ Create a new version of this API service. :param string package_id: version (identifier) of the package to create """ self.client._perform_empty( "POST", "/projects/%s/apiservices/%s/packages/%s" % (self.project_key, self.service_id, package_id)) def delete_package(self, package_id): """ Delete a version of this API service. :param string package_id: version (identifier) of the package to delete """ self.client._perform_empty( "DELETE", "/projects/%s/apiservices/%s/packages/%s" % (self.project_key, self.service_id, package_id)) def download_package_stream(self, package_id): """ Download an archive of a package as a stream. The archive can then be deployed in a DSS API Node. .. warning:: This call will monopolize the DSSClient until the stream it returns is closed. :param string package_id: version (identifier) of the package to download :return: the package archive, as a HTTP stream :rtype: file-like """ return self.client._perform_raw( "GET", "/projects/%s/apiservices/%s/packages/%s/archive" % (self.project_key, self.service_id, package_id)).raw def download_package_to_file(self, package_id, path): """ Download an archive of a package to a local file. The archive can then be deployed in a DSS API Node. :param string package_id: version (identifier) of the package to download :param string path: absolute or relative path to a file in which the package is downloaded """ package_stream = self.client._perform_raw( "GET", "/projects/%s/apiservices/%s/packages/%s/archive" % (self.project_key, self.service_id, package_id)) with open(path, 'wb') as f: for chunk in package_stream.iter_content(chunk_size=10000): if chunk: f.write(chunk) f.flush() def publish_package(self, package_id, published_service_id=None): """ Publish a package on the API Deployer. :param string package_id: version (identifier) of the package to publish :param string published_service_id: identifier of the API service on the API Deployer in which the package will be published. A new published API service will be created if none matches the identifier. If the parameter is not set, the identifier from the current :class:`DSSAPIService` is used. """ params = None if published_service_id is not None: params = {"publishedServiceId": published_service_id} self.client._perform_empty("POST", "/projects/%s/apiservices/%s/packages/%s/publish" % (self.project_key, self.service_id, package_id), params=params) dataiku-api-client-python-11.4.0/dataikuapi/dss/app.py000066400000000000000000000247161440464736400226460ustar00rootroot00000000000000import random import re import string from .future import DSSFuture from .utils import DSSTaggableObjectListItem def random_string(length): return ''.join(random.choice(string.ascii_letters) for _ in range(length)) class DSSAppListItem(DSSTaggableObjectListItem): """ An item in a list of apps. .. important:: Do not instantiate this class, use :meth:`dataikuapi.DSSClient.list_apps` instead """ def __init__(self, client, data): super(DSSAppListItem, self).__init__(data) self.client = client def to_app(self): """ Get a handle corresponding to this app. :return: a handle to interact with the app :rtype: :class:`dataikuapi.dss.app.DSSApp` """ return DSSApp(self.client, self._data["appId"]) class DSSApp(object): """ A handle to interact with an application on the DSS instance. .. important:: Do not instantiate this class directly, instead use :meth:`dataikuapi.DSSClient.get_app` """ def __init__(self, client, app_id): self.client = client self.app_id = app_id ######################################################## # Instances ######################################################## def create_instance(self, instance_key, instance_name, wait=True): """ Create a new instance of this application. Each instance must have a globally unique instance key, separate from any other project key across the whole DSS instance. :param string instance_key: project key for the new created app instance :param string instance_name: name for the new created app instance :param boolean wait: if False, the method returns immediately with a :class:`dataikuapi.dss.future.DSSFuture` on which to wait for the app instance to be created :return: a handle to interact with the app instance :rtype: :class:`~DSSAppInstance` """ future_resp = self.client._perform_json( "POST", "/apps/%s/instances" % self.app_id, body={ "targetProjectKey": instance_key, "targetProjectName": instance_name }) future = DSSFuture(self.client, future_resp.get("jobId", None), future_resp) if wait: future.wait_for_result() return DSSAppInstance(self.client, instance_key) else: return future def make_random_project_key(self): """ Create a new project key based on this app name. This method suffixes the app's name with a random string to generate a unique app instance key. :return: a project key :rtype: string """ slugified_app_id = re.sub(r'[^A-Za-z_0-9]+', '_', self.app_id) return "%s_tmp_%s" % (slugified_app_id, random_string(10)) def create_temporary_instance(self): """ Create a new temporary instance of this application. The return value should be used as a Python context manager. Upon exit, the temporary app instance is deleted. :return: an app instance :rtype: :class:`TemporaryDSSAppInstance` """ key = self.make_random_project_key() self.create_instance(key, key, True) return TemporaryDSSAppInstance(self.client, key) def list_instance_keys(self): """ List the keys of the existing instances of this app. :return: a list of instance keys :rtype: list[string] """ return [x["projectKey"] for x in self.list_instances()] def list_instances(self): """ List the existing instances of this app. :return a list of instances, each as a dict containing at least a "projectKey" field :rtype: list """ return self.client._perform_json( "GET", "/apps/%s/instances/" % self.app_id) def get_instance(self, instance_key): """ Get a particular instance of this app. :return: an app instance :rtype: :class:`~DSSAppInstance` """ return DSSAppInstance(self.client, instance_key) def get_manifest(self): """ Get the manifest of this app. :return: an app manifest :rtype: :class:`~DSSAppManifest` """ raw_data = self.client._perform_json("GET", "/apps/%s/" % self.app_id) project_key = self.app_id[8:] if self.app_id.startswith('PROJECT_') else None return DSSAppManifest(self.client, raw_data, project_key) class DSSAppManifest(object): """ Handle on the manifest of an app or an app instance. .. important:: Do not instantiate this class directly, use :meth:`dataikuapi.dss.app.DSSApp.get_manifest()` or :meth:`dataikuapi.dss.app.DSSAppInstance.get_manifest()` """ def __init__(self, client, raw_data, project_key=None): self.client = client self.raw_data = raw_data self.project_key = project_key def get_raw(self): """ Get the raw definition of the manifest. Usage example: .. code-block:: python # list all app templates that anybody can instantiate for app in client.list_apps(as_type="objects"): manifest = app.get_manifest() if manifest.get_raw()["instantiationPermission"] == 'EVERYBODY': print(app.app_id) :return: the definition of the manifest, as a dict. Top-level fields are: * **label**, **shortDesc** and **tags** : metadata of the app * **imgColo** and **imgPattern** : definition of the app's vignette (if not a manually added image) * **instantiationPermission** : define which users can use the app: one of 'EVERYBODY' or 'USE_APP_MASTER_PERMISSIONS' (ie limit to bearers of the "execute app" permission on the template project) * **limitedVisibilityEnabled** and **accessRequestsEnabled** : access settings * **projectExportManifest** : settings for which data the of app template project are copied to the app instances * **instanceFeatures** : show/hide part of the DSS UI in the app instances * **homepageSections** : definition of the actions of the app instances. This is a list of sections, each as a dict of: * **sectionTitle** and **sectionText** : metadata for the section * **tiles** : list of actions in the section, as a list of dict: * **type** : the type of the action * **prompt** : label of the action in the form * **help** and **helpTitle** : metadata for showing a help button on the action * ... and additional fields depending on the type, to hold the action's setup * **useAsRecipeSettings** : if defined, the app is actually an app-as-recipe. This field holds the settings of the app-as-recipe, as a dict of: * **icon** and **category** : metadata for display in the recipe menu * **inputRoles** and **outputRoles** : definitions of the inputs and outputs of the recipe * **variablesEditionTile** : definition of the form of the app-as-recipe that's shown in recipe instances * **runScenarioTile** : definition of the scenario to run in the app instance, that does the actual recipe computations :rtype: dict """ return self.raw_data def get_all_actions(self): """ Get the flat list of all actions. :return: a list of action defintions, each one a dict. Each action has fields * **type** : the type of the action * **prompt** : label of the action in the form * **help** and **helpTitle** : metadata for showing a help button on the action * ... and additional fields depending on the type, to hold the action's setup :rtype: list """ return [x for section in self.raw_data["homepageSections"] for x in section["tiles"]] def get_runnable_scenarios(self): """ Get the scenario identifiers that are declared as actions for this app. :return: a list of scenario identifiers :rtype: list[string] """ return [x["scenarioId"] for x in self.get_all_actions() if x["type"] == "SCENARIO_RUN"] def save(self): """ Save the changes to this manifest object back to the template project. """ if self.project_key is None: raise Exception("This manifest object wasn't created from a project, cannot be saved back") self.client._perform_empty("PUT", "/projects/%s/app-manifest" % self.project_key, body=self.raw_data) class DSSAppInstance(object): """ Handle on an instance of an app. .. important:: Do not instantiate this class directly, use :meth:`dataikuapi.dss.app.DSSApp.get_instance()` """ def __init__(self, client, project_key): self.client = client self.project_key = project_key def get_as_project(self): """ Get a handle on the project corresponding to this application instance. :return: a handle on a DSS prohect :rtype: :class:`dataikuapi.dss.project.DSSProject` """ return self.client.get_project(self.project_key) def get_manifest(self): """ Get the manifest of this app instance. :return: an app manifest :rtype: :class:`~DSSAppManifest` """ raw_data = self.client._perform_json("GET", "/projects/%s/app-manifest" % self.project_key) return DSSAppManifest(self.client, raw_data) class TemporaryDSSAppInstance(DSSAppInstance): """ Variant of :class:`~DSSAppInstance` that can be used as a Python context. .. important:: Do not instantiate this class directly, use :meth:`dataikuapi.dss.app.DSSApp.create_temporary_instance()` """ def __init__(self, client, project_key): DSSAppInstance.__init__(self, client,project_key) def close(self): """ Delete the app instance. """ self.get_as_project().delete(clear_managed_datasets=True) def __enter__(self,): return self def __exit__(self, type, value, traceback): self.close() dataiku-api-client-python-11.4.0/dataikuapi/dss/codestudio.py000066400000000000000000000421411440464736400242200ustar00rootroot00000000000000from ..utils import DataikuException import json from datetime import datetime from .future import DSSFuture class DSSCodeStudioObjectListItem(object): """ An item in a list of code studios. .. important:: Do not instantiate this class, use :meth:`dataikuapi.dss.project.DSSProject.list_code_studios` """ def __init__(self, client, project_key, data): self.client = client self.project_key = project_key self._data = data def to_code_studio(self): """ Get a handle to interact with this code studio :return: a handle on the code studio :rtype: :class:`dataikuapi.dss.codestudio.DSSCodeStudioObject` """ return DSSCodeStudioObject(self.client, self.project_key, self._data["id"]) @property def name(self): """ Get the name of the code studio .. note:: The name is user-provided and not necessarily unique. :rtype: string """ return self._data["name"] @property def id(self): """ Get the identifier of the code studio .. note:: The id is generated by DSS upon creation and random. :rtype: string """ return self._data["id"] @property def owner(self): """ Get the login of the owner of the code studio :rtype: string """ return self._data["owner"] @property def template_id(self): """ Get the identifier of the template that this code studio was created from :rtype: string """ return self._data["templateId"] @property def template_label(self): """ Get the label of the template that this code studio was created from :rtype: string """ return self._data.get('desc', {}).get('label', self._data['id']) @property def template_description(self): """ Get the description of the template that this code studio was created from :rtype: string """ return self._data.get('desc', {}).get('shortDesc', '') class DSSCodeStudioObject(object): """ A handle to manage a code studio in a project .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.get_code_studio` or :meth:`dataikuapi.dss.project.DSSProject.create_code_studio` instead """ def __init__(self, client, project_key, code_studio_id): self.client = client self.project_key = project_key self.code_studio_id = code_studio_id def delete(self): """ Delete the code studio """ self.client._perform_empty("DELETE", "/projects/%s/code-studios/%s" % (self.project_key, self.code_studio_id)) def get_settings(self): """ Get the code studio's definition Usage example .. code-block:: python # list code studios of some user for code_studio in project.list_code_studios(as_type="objects"): settings = code_studio.get_settings() if settings.owner == 'the_user_login': print("User owns code studio %s from template %s" % (settings.name, settings.template_id)) :return: a handle to inspect the code studio definition :rtype: :class:`dataikuapi.dss.codestudio.DSSCodeStudioObjectSettings` """ settings = self.client._perform_json("GET", "/projects/%s/code-studios/%s" % (self.project_key, self.code_studio_id)) return DSSCodeStudioObjectSettings(self.client, self.project_key, self.code_studio_id, settings) def get_status(self): """ Get the code studio's state Usage example .. code-block:: python # print list of currently running code studios for code_studio in project.list_code_studios(as_type="objects"): status = code_studio.get_status() if status.state == 'RUNNING': settings = code_studio.get_settings() print("Code studio %s from template %s is running" % (settings.name, settings.template_id)) :return: a handle to inspect the code studio state :rtype: :class:`dataikuapi.dss.codestudio.DSSCodeStudioObjectStatus` """ status = self.client._perform_json("GET", "/projects/%s/code-studios/%s/status" % (self.project_key, self.code_studio_id)) return DSSCodeStudioObjectStatus(self.client, self.project_key, self.code_studio_id, status) def stop(self): """ Stop a running code studio :return: a future to wait on the stop, or None if already stopped :rtype: :class:`dataikuapi.dss.future.DSSFuture` """ ret = self.client._perform_json("POST", "/projects/%s/code-studios/%s/stop" % (self.project_key, self.code_studio_id)) return DSSFuture.from_resp(self.client, ret) def restart(self): """ Start or restart a code studio :return: a future to wait on the start :rtype: :class:`dataikuapi.dss.future.DSSFuture` """ ret = self.client._perform_json("POST", "/projects/%s/code-studios/%s/restart" % (self.project_key, self.code_studio_id)) return DSSFuture.from_resp(self.client, ret) def check_conflicts(self, zone): """ Checks whether the files in a zone of the code studio have conflicting changes with what the DSS instance has. Usage example .. code-block:: python # stop a code studio if there's no conflict in any zone status = code_studio.get_status() conflict_count = 0 for zone in status.get_zones(as_type="names"): conflicts = code_studio.check_conflicts(zone) conflict_count += len(conflicts.added) conflict_count += len(conflicts.modified) conflict_count += len(conflicts.deleted) if conflict_count == 0: code_studio.stop().wait_for_result() :param string zone: name of the zone to check (see :meth:`dataikuapi.dss.codestudio.DSSCodeStudioObjectStatus.get_zones`) :return: a summary of the conflicts that were found. :rtype: :class:`DSSCodeStudioObjectConflicts` """ conflicts = self.client._perform_json("GET", "/projects/%s/code-studios/%s/conflicts/%s" % (self.project_key, self.code_studio_id, zone)) return DSSCodeStudioObjectConflicts(zone, conflicts) def pull_from_code_studio(self, zone): """ Copies the files from a zone of the code studio to the DSS instance :param str zone: name of the zone to pull (see :meth:`dataikuapi.dss.codestudio.DSSCodeStudioObjectStatus.get_zones`) """ return self.client._perform_json("GET", "/projects/%s/code-studios/%s/pull/%s" % (self.project_key, self.code_studio_id, zone)) def push_to_code_studio(self, zone): """ Copies the files from the DSS instance to a zone of the code studio :param str zone: name of the zone to push (see :meth:`dataikuapi.dss.codestudio.DSSCodeStudioObjectStatus.get_zones`) :return: a dictionary of {count: , size: } :rtype: dict """ return self.client._perform_json("GET", "/projects/%s/code-studios/%s/push/%s" % (self.project_key, self.code_studio_id, zone)) class DSSCodeStudioObjectConflicts(dict): """ Summary of the conflicts on zones of a code studio. .. note:: Only conflicting files are listed, that is, files added, modified or deleted in the code studio for which the corresponding file in the DSS instance has been also added, modified or deleted. """ def __init__(self, zone, conflicts): super(DSSCodeStudioObjectConflicts, self).__init__(conflicts) self._zone = zone self._conflicts = conflicts def get_raw(self): """ Get the raw conflicts summary. :return: a summary of the conflicts that were found, as a dict. The top-level field is the zone checked, or 'error' if it wasn't found. The value of the field is a dict of changes: * **commitFrom** : hashes of the commit of the files on the DSS instance at the time of the last sync to the code studio pod * **commitTo** : hashes of the commit of the files on the DSS instance now * **authors** : if **commitTo** is different from **commitFrom**, list of names of the uses who made changes on the DSS instance files * **added** : list of relative paths of files added in the code studio that conflict with changes on the DSS instances * **modified** : list of relative paths of files modified in the code studio that conflict with changes on the DSS instances * **deleted** : list of relative paths of files deleted in the code studio that conflict with changes on the DSS instances :rtype: dict """ return self._conflicts @property def is_error(self): """ Whether fetching the conflicts failed. Typically this can happen if the name of the zone(s) for which the conflicts where requested is invalid. :rtype: boolean """ return "error" in self._conflicts @property def authors(self): """ Get the authors of changes to the files. :return: a list of logins of users who modified the conflicting files in the DSS instance. :rtype: list[string] """ return self._conflicts.get(self._zone, {}).get("authors", []) @property def added(self): """ Get the list of files added in the code studio. :return: a list of paths to files that were added in the code studio and conflict with changes on the DSS instance. :rtype: list[string] """ return self._conflicts.get(self._zone, {}).get("added", []) @property def modified(self): """ Get the list of files modified in the code studio. :return: a list of paths to files that were modified in the code studio and conflict with changes on the DSS instance. :rtype: list[string] """ return self._conflicts.get(self._zone, {}).get("modified", []) @property def deleted(self): """ Get the list of files deleted in the code studio. :return: a list of paths to files that were deleted in the code studio and conflict with changes on the DSS instance. :rtype: list[string] """ return self._conflicts.get(self._zone, {}).get("deleted", []) class DSSCodeStudioObjectSettings(object): """ Settings for a code studio .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.codestudio.DSSCodeStudioObject.get_settings` instead """ def __init__(self, client, project_key, code_studio_id, settings): self.client = client self.project_key = project_key self.code_studio_id = code_studio_id self.settings = settings def get_raw(self): """ Gets all settings as a raw dictionary. :return: the settings, as a dict. Fields are: * **projectKey** and **id** : identify the code studio * **name** : name (label) of the code studio * **templateId** : identifier of the template of the code studio * **libName** : name of the folder for the code studio resources * **owner** : login of the owner of the code studio * **versionTag**, **creationTag**, **checklists**, **tags**, **customFields** : common fields on DSS objects :rtype: dict """ return self.settings @property def id(self): """ Get the identifier of the code studio. .. note:: The id is generated by DSS upon creation and random. :rtype: string """ return self.settings["id"] @property def name(self): """ Get the name of the code studio. .. note:: The name is user-provided and not necessarily unique. :rtype: string """ return self.settings["name"] @property def template_id(self): """ Get the identifier of the template that the code studio was create from. :rtype: string """ return self.settings["templateId"] @property def lib_name(self): """ Get the name of the folder resource files of the code studio are stored in. The path to the resources files is then "/lib/code_studio///" :rtype: string """ return self.settings["libName"] @property def owner(self): """ Get the login of the owner of the code studio. Only the owner of a code studio can use it. Administrators of the project can merely stop/start code studios, not use them. :rtype: string """ return self.settings["owner"] class DSSCodeStudioObjectStatus(object): """ Handle to inspect the status of a code studio .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.codestudio.DSSCodeStudioObject.get_status()` instead """ def __init__(self, client, project_key, code_studio_id, status): self.client = client self.project_key = project_key self.code_studio_id = code_studio_id self.status = status def get_raw(self): """ Gets the status as a raw dict. .. note:: Some fields are only defined when the code studio is running. For instance, **exposed** and **syncedZones** are empty when the code studio is not running. :return: top-level fields are * **state** : the current state of the code studio (possible values: STOPPED, STARTING, RUNNING, STOPPING) * **lastStateChange** : timestamp in milliseconds of the last change to **state** * **lastTemplateBuilt** : timestamp in milliseconds of the last build of the template of this code studio * **jobId** : identifier of the future that runs the code studio * **runAsUser** : login of the DSS user that this code studio currently uses * **exposableCount** : number of ports exposed in the pod of this code studio * **exposed** : list of ports exposed by this code studio. Each tab is a dict of * **label** : label of the tab * **exposedHtml** : if True the tab is shown in the UI * **exposedPort** : port exposed in the pod * **url** : path (relative to DSS base href) to the HTTP server that the pod exposes * **syncedZones** : list of files zones in the pod that are synchronized with the DSS instance. Each zone is a dict of * **id** : identifier of the zone in this code studio (note: changes at each restart of the code studio) * **zone** : type of the zone * **pathInContainer** : location of the zone inside the pod * **oneWay** : if True, the zone is read-only w.r.t. the DSS instance :rtype: dict """ return self.status @property def state(self): """ Get the current state of the code studio. Possible values are STOPPED, STARTING, RUNNING, STOPPING :rtype: string """ return self.status["state"] @property def last_state_change(self): """ Get the timestamp of the last change of the state. :return: a datetime, or None if the code studio was never started :rtype: `datetime.datetime` """ ts = self.status.get("lastStateChange", 0) if ts > 0: return datetime.fromtimestamp(ts / 1000) else: return None def get_zones(self, as_type="names"): """ Get the list of the zones synchronized inside the code studio :param string as_type: if set to "names", then return a list of zone identifiers; if set to "objects", then return a list of zone definitions :return: the list of zones, each one either a string (if `as_type` is "names), or a dict of: * **id** : identifier of the zone in this code studio (note: changes at each restart of the code studio) * **zone** : type of the zone * **pathInContainer** : location of the zone inside the pod * **oneWay** : if True, the zone is read-only w.r.t. the DSS instance :rtype: list """ zones = self.status.get("syncedZones", []) if as_type == 'name' or as_type == 'names': return [z.get('id') for z in zones] if as_type == 'object' or as_type == 'objects': return zones else: raise Exception("Only 'names' or 'objects' is supported for as_type") dataiku-api-client-python-11.4.0/dataikuapi/dss/continuousactivity.py000066400000000000000000000103341440464736400260400ustar00rootroot00000000000000import time import sys from ..utils import DataikuException class DSSContinuousActivity(object): """ A handle to interact with the execution of a continuous recipe on the DSS instance. .. important:: Do not create this class directly, instead use :meth:`dataikuapi.dss.project.DSSProject.get_continuous_activity` """ def __init__(self, client, project_key, recipe_id): self.client = client self.recipe_id = recipe_id self.project_key = project_key def start(self, loop_params={}): """ Start the continuous activity :param dict loop_params: controls how the recipe is restarted after a failure, and the delay before the restarting. Default is to restart indefinitely without delay. Fields are: * **abortAfterCrashes** : when reaching this number of failures, the recipe isn't restarted anymore. Use -1 as 'no limit on number of failures' * **initialRestartDelayMS** : initial delay to wait before restarting after a failure * **restartDelayIncMS** : increase to the delay before restarting upon subsequent failures * **maxRestartDelayMS** : max delay before restarting after failure """ return self.client._perform_json( "POST", "/projects/%s/continuous-activities/%s/start" % (self.project_key, self.recipe_id), body=loop_params) def stop(self): """ Stop the continuous activity. """ self.client._perform_empty( "POST", "/projects/%s/continuous-activities/%s/stop" % (self.project_key, self.recipe_id)) def get_status(self): """ Get the current status of the continuous activity. Usage example: .. code-block:: python # stop a continuous activity via its future from dataikuapi.dss.future import DSSFuture activity = project.get_continuous_activity("my_continuous_recipe") status = activity.get_status() future = DSSFuture(a.client, status["mainLoopState"]['futureId'], status["mainLoopState"]['futureInfo']) future.abort() # this is equivalent to simply stop() activity.stop() :returns: the state of the continuous activity. Top-level fields are: * **projectKey** and **recipeId** : identify the continuous recipe * **recipeType** : type of continuous recipe ('csync', cpython', 'ksql' or 'streaming_spark_scala') * **desiredState** : whether the recipe should be running or not (values: 'STARTED' or 'STOPPED') * **loopParams** : parameters for the restarting of the recipe upon failure. Sub-fields are: * **abortAfterCrashes** : when reaching this number of failures, the recipe isn't restarted anymore. Use -1 as 'no limit on number of failures' * **initialRestartDelayMS** : initial delay to wait before restarting after a failure * **restartDelayIncMS** : increase to the delay before restarting upon subsequent failures * **maxRestartDelayMS** : max delay before restarting after failure * **mainLoopState** : state of the current run of the recipe. Sub-fields are: * **runId** : identifier of the run of the recipe (equivalent to a call or action to start) * **attemptId** : identifier of the (re)start of the recipe within the run * **lastCrashLogTail** and **lastError** : information about the last failure * **futureInfo** and **futureId** : information about the current run :rtype: dict """ return self.client._perform_json( "GET", "/projects/%s/continuous-activities/%s/" % (self.project_key, self.recipe_id)) def get_recipe(self): """ Get a handle on the associated recipe. :rtype: :class:`dataikuapi.dss.recipe.DSSRecipe` """ from .recipe import DSSRecipe return DSSRecipe(self.client, self.project_key, self.recipe_id) dataiku-api-client-python-11.4.0/dataikuapi/dss/dataset.py000066400000000000000000001237241440464736400235120ustar00rootroot00000000000000import datetime from ..utils import DataikuException from ..utils import DataikuUTF8CSVReader from ..utils import DataikuStreamedHttpUTF8CSVReader from .future import DSSFuture import json, warnings from .utils import DSSTaggableObjectListItem, DSSTaggableObjectSettings from .future import DSSFuture from .metrics import ComputedMetrics from .discussion import DSSObjectDiscussions from .statistics import DSSStatisticsWorksheet from . import recipe try: basestring except NameError: basestring = str class DSSDatasetListItem(DSSTaggableObjectListItem): """ An item in a list of datasets. .. caution:: Do not instantiate this class, use :meth:`dataikuapi.dss.project.DSSProject.list_datasets` """ def __init__(self, client, data): super(DSSDatasetListItem, self).__init__(data) self.client = client def to_dataset(self): """ Gets a handle on the corresponding dataset. :returns: a handle on a dataset :rtype: :class:`DSSDataset` """ return DSSDataset(self.client, self._data["projectKey"], self._data["name"]) @property def name(self): """ Get the name of the dataset. :rtype: string """ return self._data["name"] @property def id(self): """ Get the identifier of the dataset. :rtype: string """ return self._data["name"] @property def type(self): """ Get the type of the dataset. :rtype: string """ return self._data["type"] @property def schema(self): """ Get the schema of the dataset. :returns: a list of column definitions. See :meth:`DSSDataset.get_schema()` :rtype: list[dict] """ return self._data["schema"] @property def connection(self): """ Get the name of the connection on which this dataset is attached, or None if there is no connection for this dataset. :rtype: string """ if not "params" in self._data: return None return self._data["params"].get("connection", None) def get_column(self, column): """ Get the a given column in the schema of the dataset, by its name. :param str column: name of the column to find :returns: the column settings or None if column does not exist :rtype: dict """ matched = [col for col in self.schema["columns"] if col["name"] == column] return None if len(matched) == 0 else matched[0] class DSSDataset(object): """ A dataset on the DSS instance. Do not instantiate this class, use :meth:`dataikuapi.dss.project.DSSProject.get_dataset` """ def __init__(self, client, project_key, dataset_name): self.client = client self.project = client.get_project(project_key) self.project_key = project_key self.dataset_name = dataset_name @property def id(self): return self.dataset_name @property def name(self): return self.dataset_name ######################################################## # Dataset deletion ######################################################## def delete(self, drop_data=False): """ Delete the dataset :param bool drop_data: Should the data of the dataset be dropped, defaults to False """ return self.client._perform_empty( "DELETE", "/projects/%s/datasets/%s" % (self.project_key, self.dataset_name), params = { "dropData" : drop_data }) ######################################################## # Dataset definition ######################################################## def get_settings(self): """ Get the settings of this dataset as a :class:`DSSDatasetSettings`, or one of its subclasses. Know subclasses of :class:`DSSDatasetSettings` include :class:`FSLikeDatasetSettings` and :class:`SQLDatasetSettings` You must use :meth:`~DSSDatasetSettings.save()` on the returned object to make your changes effective on the dataset. .. code-block:: python # Example: activating discrete partitioning on a SQL dataset dataset = project.get_dataset("my_database_table") settings = dataset.get_settings() settings.add_discrete_partitioning_dimension("country") settings.save() :rtype: :class:`DSSDatasetSettings` """ data = self.client._perform_json("GET", "/projects/%s/datasets/%s" % (self.project_key, self.dataset_name)) if data["type"] in self.__class__._FS_TYPES: return FSLikeDatasetSettings(self, data) elif data["type"] in self.__class__._SQL_TYPES: return SQLDatasetSettings(self, data) else: return DSSDatasetSettings(self, data) def get_definition(self): """ Get the raw settings of the dataset as a dict .. caution:: Deprecated. Use :meth:`get_settings` :rtype: dict """ warnings.warn("Dataset.get_definition is deprecated, please use get_settings", DeprecationWarning) return self.client._perform_json( "GET", "/projects/%s/datasets/%s" % (self.project_key, self.dataset_name)) def set_definition(self, definition): """ Set the definition of the dataset .. caution:: Deprecated. Use :meth:`get_settings` and :meth:`DSSDatasetSettings.save` :param definition: the definition, as a dict. You should only set a definition object that has been retrieved using the get_definition call. """ warnings.warn("Dataset.set_definition is deprecated, please use get_settings", DeprecationWarning) return self.client._perform_json( "PUT", "/projects/%s/datasets/%s" % (self.project_key, self.dataset_name), body=definition) def exists(self): """ Test if the dataset exists. :returns: whether this dataset exists :rtype: bool """ try: self.get_metadata() return True except Exception as e: return False ######################################################## # Dataset metadata ######################################################## def get_schema(self): """ Get the schema of the dataset :returns: a JSON object of the schema, with the list of columns """ return self.client._perform_json( "GET", "/projects/%s/datasets/%s/schema" % (self.project_key, self.dataset_name)) def set_schema(self, schema): """ Set the schema of the dataset :param schema: the desired schema for the dataset, as a JSON object. All columns have to provide their name and type """ return self.client._perform_json( "PUT", "/projects/%s/datasets/%s/schema" % (self.project_key, self.dataset_name), body=schema) def get_metadata(self): """ Get the metadata attached to this dataset. The metadata contains label, description checklists, tags and custom metadata of the dataset :returns: a dict object. For more information on available metadata, please see https://doc.dataiku.com/dss/api/11.0/rest/ """ return self.client._perform_json( "GET", "/projects/%s/datasets/%s/metadata" % (self.project_key, self.dataset_name)) def set_metadata(self, metadata): """ Set the metadata on this dataset. :param metadata: the new state of the metadata for the dataset. You should only set a metadata object that has been retrieved using the get_metadata call. """ return self.client._perform_json( "PUT", "/projects/%s/datasets/%s/metadata" % (self.project_key, self.dataset_name), body=metadata) ######################################################## # Dataset data ######################################################## def iter_rows(self, partitions=None): """ Get the dataset's data :returns: an iterator over the rows, each row being a tuple of values. The order of values in the tuples is the same as the order of columns in the schema returned by get_schema """ csv_stream = self.client._perform_raw( "GET" , "/projects/%s/datasets/%s/data/" %(self.project_key, self.dataset_name), params = { "format" : "tsv-excel-noheader", "partitions" : partitions }) return DataikuStreamedHttpUTF8CSVReader(self.get_schema()["columns"], csv_stream).iter_rows() def list_partitions(self): """ Get the list of all partitions of this dataset :returns: the list of partitions, as a list of strings """ return self.client._perform_json( "GET", "/projects/%s/datasets/%s/partitions" % (self.project_key, self.dataset_name)) def clear(self, partitions=None): """ Clear all data in this dataset :param partitions: (optional) a list of partitions to clear. When not provided, the entire dataset is cleared """ return self.client._perform_json( "DELETE", "/projects/%s/datasets/%s/data" % (self.project_key, self.dataset_name), params={"partitions" : partitions}) def copy_to(self, target, sync_schema=True, write_mode="OVERWRITE"): """ Copy the data of this dataset to another dataset :param target Dataset: a :class:`dataikuapi.dss.dataset.DSSDataset` representing the target of this copy :returns: a DSSFuture representing the operation """ dqr = { "targetProjectKey" : target.project_key, "targetDatasetName": target.dataset_name, "syncSchema": sync_schema, "writeMode" : write_mode } future_resp = self.client._perform_json("POST", "/projects/%s/datasets/%s/actions/copyTo" % (self.project_key, self.dataset_name), body=dqr) return DSSFuture(self.client, future_resp.get("jobId", None), future_resp) ######################################################## # Dataset actions ######################################################## def build(self, job_type="NON_RECURSIVE_FORCED_BUILD", partitions=None, wait=True, no_fail=False): """ Start a new job to build this dataset and wait for it to complete. Raises if the job failed. .. code-block:: python job = dataset.build() print("Job %s done" % job.id) :param job_type: The job type. One of RECURSIVE_BUILD, NON_RECURSIVE_FORCED_BUILD or RECURSIVE_FORCED_BUILD :param partitions: If the dataset is partitioned, a list of partition ids to build :param no_fail: if True, does not raise if the job failed. :returns: the :class:`dataikuapi.dss.job.DSSJob` job handle corresponding to the built job :rtype: :class:`dataikuapi.dss.job.DSSJob` """ jd = self.project.new_job(job_type) jd.with_output(self.dataset_name, partition=partitions) if wait: return jd.start_and_wait() else: return jd.start() def synchronize_hive_metastore(self): """ Synchronize this dataset with the Hive metastore """ self.client._perform_empty( "POST" , "/projects/%s/datasets/%s/actions/synchronizeHiveMetastore" %(self.project_key, self.dataset_name)) def update_from_hive(self): """ Resynchronize this dataset from its Hive definition """ self.client._perform_empty( "POST", "/projects/%s/datasets/%s/actions/updateFromHive" %(self.project_key, self.dataset_name)) def compute_metrics(self, partition='', metric_ids=None, probes=None): """ Compute metrics on a partition of this dataset. If neither metric ids nor custom probes set are specified, the metrics setup on the dataset are used. """ url = "/projects/%s/datasets/%s/actions" % (self.project_key, self.dataset_name) if metric_ids is not None: return self.client._perform_json( "POST" , "%s/computeMetricsFromIds" % url, params={'partition':partition}, body={"metricIds" : metric_ids}) elif probes is not None: return self.client._perform_json( "POST" , "%s/computeMetrics" % url, params={'partition':partition}, body=probes) else: return self.client._perform_json( "POST" , "%s/computeMetrics" % url, params={'partition':partition}) def run_checks(self, partition='', checks=None): """ Run checks on a partition of this dataset. If the checks are not specified, the checks setup on the dataset are used. """ if checks is None: return self.client._perform_json( "POST" , "/projects/%s/datasets/%s/actions/runChecks" %(self.project_key, self.dataset_name), params={'partition':partition}) else: return self.client._perform_json( "POST" , "/projects/%s/datasets/%s/actions/runChecks" %(self.project_key, self.dataset_name), params={'partition':partition}, body=checks) def uploaded_add_file(self, fp, filename): """ Add a file to an "uploaded files" dataset :param file fp: A file-like object that represents the file to upload :param str filename: The filename for the file to upload """ self.client._perform_empty("POST", "/projects/%s/datasets/%s/uploaded/files" % (self.project_key, self.dataset_name), files={"file":(filename, fp)}) def uploaded_list_files(self): """ List the files in an "uploaded files" dataset """ return self.client._perform_json("GET", "/projects/%s/datasets/%s/uploaded/files" % (self.project_key, self.dataset_name)) ######################################################## # Lab and ML # Don't forget to synchronize with DSSProject.* ######################################################## def create_prediction_ml_task(self, target_variable, ml_backend_type="PY_MEMORY", guess_policy="DEFAULT", prediction_type=None, wait_guess_complete=True): """ Create a new prediction task in a new visual analysis lab for a dataset. :param string input_dataset: the dataset to use for training/testing the model :param string target_variable: the variable to predict :param string ml_backend_type: ML backend to use, one of PY_MEMORY, MLLIB or H2O :param string guess_policy: Policy to use for setting the default parameters. Valid values are: DEFAULT, SIMPLE_FORMULA, DECISION_TREE, EXPLANATORY and PERFORMANCE :param string prediction_type: The type of prediction problem this is. If not provided the prediction type will be guessed. Valid values are: BINARY_CLASSIFICATION, REGRESSION, MULTICLASS :param boolean wait_guess_complete: if False, the returned ML task will be in 'guessing' state, i.e. analyzing the input dataset to determine feature handling and algorithms. You should wait for the guessing to be completed by calling ``wait_guess_complete`` on the returned object before doing anything else (in particular calling ``train`` or ``get_settings``) """ return self.project.create_prediction_ml_task(self.dataset_name, target_variable = target_variable, ml_backend_type = ml_backend_type, guess_policy = guess_policy, prediction_type = prediction_type, wait_guess_complete = wait_guess_complete) def create_clustering_ml_task(self, input_dataset, ml_backend_type="PY_MEMORY", guess_policy="KMEANS", wait_guess_complete=True): """ Create a new clustering task in a new visual analysis lab for a dataset. The returned ML task will be in 'guessing' state, i.e. analyzing the input dataset to determine feature handling and algorithms. You should wait for the guessing to be completed by calling ``wait_guess_complete`` on the returned object before doing anything else (in particular calling ``train`` or ``get_settings``) :param string ml_backend_type: ML backend to use, one of PY_MEMORY, MLLIB or H2O :param string guess_policy: Policy to use for setting the default parameters. Valid values are: KMEANS and ANOMALY_DETECTION :param boolean wait_guess_complete: if False, the returned ML task will be in 'guessing' state, i.e. analyzing the input dataset to determine feature handling and algorithms. You should wait for the guessing to be completed by calling ``wait_guess_complete`` on the returned object before doing anything else (in particular calling ``train`` or ``get_settings``) """ return self.project.create_clustering_ml_task(self.dataset_name, ml_backend_type=ml_backend_type, guess_policy=guess_policy, wait_guess_complete=wait_guess_complete) def create_timeseries_forecasting_ml_task(self, target_variable, time_variable, timeseries_identifiers=None, guess_policy="TIMESERIES_DEFAULT", wait_guess_complete=True): """ Create a new time series forecasting task in a new visual analysis lab for a dataset. :param string target_variable: The variable to forecast :param string time_variable: Column to be used as time variable. Should be a Date (parsed) column. :param list timeseries_identifiers: List of columns to be used as time series identifiers (when the dataset has multiple series) :param string guess_policy: Policy to use for setting the default parameters. Valid values are: TIMESERIES_DEFAULT, TIMESERIES_STATISTICAL, and TIMESERIES_DEEP_LEARNING :param boolean wait_guess_complete: If False, the returned ML task will be in 'guessing' state, i.e. analyzing the input dataset to determine feature handling and algorithms. You should wait for the guessing to be completed by calling ``wait_guess_complete`` on the returned object before doing anything else (in particular calling ``train`` or ``get_settings``) """ return self.project.create_timeseries_forecasting_ml_task(self.dataset_name, target_variable=target_variable, time_variable=time_variable, timeseries_identifiers=timeseries_identifiers, guess_policy=guess_policy, wait_guess_complete=wait_guess_complete) def create_analysis(self): """ Create a new visual analysis lab """ return self.project_create_analysis(self.dataset_name) def list_analyses(self, as_type="listitems"): """ List the visual analyses on this dataset :param str as_type: How to return the list. Supported values are "listitems" and "objects", defaults to "listitems" :returns: The list of the analyses. If "as_type" is "listitems", each one as a dict, If "as_type" is "objects", each one as a :class:`dataikuapi.dss.analysis.DSSAnalysis` :rtype: list """ analysis_list = [al for al in self.project.list_analyses() if self.dataset_name == al.get('inputDataset')] if as_type == "listitems" or as_type == "listitem": return analysis_list elif as_type == "objects" or as_type == "object": return [self.project.get_analysis(item["analysisId"])for item in analysis_list] else: raise ValueError("Unknown as_type") def delete_analyses(self, drop_data=False): """ Delete all analyses that have this dataset as input dataset. Also deletes ML tasks that are part of the analysis :param bool drop_data: whether to drop data for all ML tasks in the analysis, defaults to False """ [analysis.delete(drop_data=drop_data) for analysis in self.list_analyses(as_type="objects")] ######################################################## # Statistics worksheets ######################################################## def list_statistics_worksheets(self, as_objects=True): """ List the statistics worksheets associated to this dataset. :rtype: list of :class:`dataikuapi.dss.statistics.DSSStatisticsWorksheet` """ worksheets = self.client._perform_json( "GET", "/projects/%s/datasets/%s/statistics/worksheets/" % (self.project_key, self.dataset_name)) if as_objects: return [self.get_statistics_worksheet(worksheet['id']) for worksheet in worksheets] else: return worksheets def create_statistics_worksheet(self, name="My worksheet"): """ Create a new worksheet in the dataset, and return a handle to interact with it. :param string input_dataset: input dataset of the worksheet :param string worksheet_name: name of the worksheet :returns: a statistic worksheet handle :rtype: :class:`dataikuapi.dss.statistics.DSSStatisticsWorksheet` """ worksheet_definition = { "projectKey": self.project_key, "name": name, "dataSpec": { "inputDatasetSmartName": self.dataset_name, "datasetSelection": { "partitionSelectionMethod": "ALL", "maxRecords": 30000, "samplingMethod": "FULL" } } } created_worksheet = self.client._perform_json( "POST", "/projects/%s/datasets/%s/statistics/worksheets/" % (self.project_key, self.dataset_name), body=worksheet_definition ) return self.get_statistics_worksheet(created_worksheet['id']) def get_statistics_worksheet(self, worksheet_id): """ Get a handle to interact with a statistics worksheet :param string worksheet_id: the ID of the desired worksheet :returns: a statistic worksheet handle :rtype: :class:`dataikuapi.dss.statistics.DSSStatisticsWorksheet` """ return DSSStatisticsWorksheet(self.client, self.project_key, self.dataset_name, worksheet_id) ######################################################## # Metrics ######################################################## def get_last_metric_values(self, partition=''): """ Get the last values of the metrics on this dataset :returns: a list of metric objects and their value """ return ComputedMetrics(self.client._perform_json( "GET", "/projects/%s/datasets/%s/metrics/last/%s" % (self.project_key, self.dataset_name, 'NP' if len(partition) == 0 else partition))) def get_metric_history(self, metric, partition=''): """ Get the history of the values of the metric on this dataset :returns: an object containing the values of the metric, cast to the appropriate type (double, boolean,...) """ return self.client._perform_json( "GET", "/projects/%s/datasets/%s/metrics/history/%s" % (self.project_key, self.dataset_name, 'NP' if len(partition) == 0 else partition), params={'metricLookup' : metric if isinstance(metric, str) or isinstance(metric, unicode) else json.dumps(metric)}) def get_info(self): """ Retrieve all the information about a dataset :returns: a :class:`DSSDatasetInfo` containing all the information about a dataset. :rtype: :class:`DSSDatasetInfo` """ data = self.client._perform_json("GET", "/projects/%s/datasets/%s/info" % (self.project_key, self.dataset_name)) return DSSDatasetInfo(self, data) ######################################################## # Misc ######################################################## def get_zone(self): """ Get the flow zone of this dataset :rtype: :class:`dataikuapi.dss.flow.DSSFlowZone` """ return self.project.get_flow().get_zone_of_object(self) def move_to_zone(self, zone): """ Move this object to a flow zone :param object zone: a :class:`dataikuapi.dss.flow.DSSFlowZone` where to move the object """ if isinstance(zone, basestring): zone = self.project.get_flow().get_zone(zone) zone.add_item(self) def share_to_zone(self, zone): """ Share this object to a flow zone :param object zone: a :class:`dataikuapi.dss.flow.DSSFlowZone` where to share the object """ if isinstance(zone, basestring): zone = self.project.get_flow().get_zone(zone) zone.add_shared(self) def unshare_from_zone(self, zone): """ Unshare this object from a flow zone :param object zone: a :class:`dataikuapi.dss.flow.DSSFlowZone` from where to unshare the object """ if isinstance(zone, basestring): zone = self.project.get_flow().get_zone(zone) zone.remove_shared(self) def get_usages(self): """ Get the recipes or analyses referencing this dataset :returns: a list of usages """ return self.client._perform_json("GET", "/projects/%s/datasets/%s/usages" % (self.project_key, self.dataset_name)) def get_object_discussions(self): """ Get a handle to manage discussions on the dataset :returns: the handle to manage discussions :rtype: :class:`dataikuapi.discussion.DSSObjectDiscussions` """ return DSSObjectDiscussions(self.client, self.project_key, "DATASET", self.dataset_name) ######################################################## # Test / Autofill ######################################################## _FS_TYPES = ["Filesystem", "UploadedFiles", "FilesInFolder", "HDFS", "S3", "Azure", "GCS", "FTP", "SCP", "SFTP"] # HTTP is FSLike but not FS _SQL_TYPES = ["JDBC", "PostgreSQL", "MySQL", "Vertica", "Snowflake", "Redshift", "Greenplum", "Teradata", "Oracle", "SQLServer", "SAPHANA", "Netezza", "BigQuery", "Athena", "hiveserver2", "Synapse", "Databricks"] def test_and_detect(self, infer_storage_types=False): """Used internally by autodetect_settings. It is not usually required to call this method""" settings = self.get_settings() if settings.type in self.__class__._FS_TYPES: future_resp = self.client._perform_json("POST", "/projects/%s/datasets/%s/actions/testAndDetectSettings/fsLike"% (self.project_key, self.dataset_name), body = {"detectPossibleFormats" : True, "inferStorageTypes" : infer_storage_types }) return DSSFuture(self.client, future_resp.get('jobId', None), future_resp) elif settings.type in self.__class__._SQL_TYPES: return self.client._perform_json("POST", "/projects/%s/datasets/%s/actions/testAndDetectSettings/externalSQL"% (self.project_key, self.dataset_name)) elif settings.type == "ElasticSearch": return self.client._perform_json("POST", "/projects/%s/datasets/%s/actions/testAndDetectSettings/elasticsearch"% (self.project_key, self.dataset_name)) else: raise ValueError("don't know how to test/detect on dataset type:%s" % settings.type) def autodetect_settings(self, infer_storage_types=False): """ Detect appropriate settings for this dataset using Dataiku detection engine :returns: new suggested settings that you can :meth:`DSSDatasetSettings.save` :rtype: :class:`DSSDatasetSettings` or a subclass """ settings = self.get_settings() if settings.type in self.__class__._FS_TYPES: future = self.test_and_detect(infer_storage_types) result = future.wait_for_result() if not "format" in result or not result["format"]["ok"]: raise DataikuException("Format detection failed, complete response is " + json.dumps(result)) settings.get_raw()["formatType"] = result["format"]["type"] settings.get_raw()["formatParams"] = result["format"]["params"] settings.get_raw()["schema"] = result["format"]["schemaDetection"]["newSchema"] return settings elif settings.type in self.__class__._SQL_TYPES: result = self.test_and_detect() if not "schemaDetection" in result: raise DataikuException("Format detection failed, complete response is " + json.dumps(result)) settings.get_raw()["schema"] = result["schemaDetection"]["newSchema"] return settings elif settings.type == "ElasticSearch": result = self.test_and_detect() if not "schemaDetection" in result: raise DataikuException("Format detection failed, complete response is " + json.dumps(result)) settings.get_raw()["schema"] = result["schemaDetection"]["newSchema"] return settings else: raise ValueError("don't know how to test/detect on dataset type:%s" % settings.type) def get_as_core_dataset(self): """ Get the :class:`dataiku.Dataset` object corresponding to this dataset """ import dataiku return dataiku.Dataset("%s.%s" % (self.project_key, self.dataset_name)) ######################################################## # Creation of recipes ######################################################## def new_code_recipe(self, type, code=None, recipe_name=None): """ Start the creation of a new code recipe taking this dataset as input :param str type: Type of the recipe ('python', 'r', 'pyspark', 'sparkr', 'sql', 'sparksql', 'hive', ...) :param str code: The code of the recipe """ if type == "python": builder = recipe.PythonRecipeCreator(recipe_name, self.project) else: builder = recipe.CodeRecipeCreator(recipe_name, type, self.project) builder.with_input(self.dataset_name) if code is not None: builder.with_script(code) return builder def new_recipe(self, type, recipe_name=None): """ Start the creation of a new recipe taking this dataset as input. For more details, please see :meth:`dataikuapi.dss.project.DSSProject.new_recipe` :param str type: Type of the recipe """ builder = self.project.new_recipe(type=type, name=recipe_name) builder.with_input(self.dataset_name) return builder class DSSDatasetSettings(DSSTaggableObjectSettings): """ Base settings class for a DSS dataset. .. caution:: Do not instantiate this class directly, use :meth:`DSSDataset.get_settings` Use :meth:`save` to save your changes """ def __init__(self, dataset, settings): super(DSSDatasetSettings, self).__init__(settings) self.dataset = dataset self.settings = settings def get_raw(self): """Get the raw dataset settings as a dict""" return self.settings def get_raw_params(self): """Get the type-specific params, as a raw dict""" return self.settings["params"] @property def type(self): return self.settings["type"] @property def schema_columns(self): return self.settings["schema"]["columns"] def remove_partitioning(self): self.settings["partitioning"] = {"dimensions" : []} def add_discrete_partitioning_dimension(self, dim_name): self.settings["partitioning"]["dimensions"].append({"name": dim_name, "type": "value"}) def add_time_partitioning_dimension(self, dim_name, period="DAY"): self.settings["partitioning"]["dimensions"].append({"name": dim_name, "type": "time", "params":{"period": period}}) def add_raw_schema_column(self, column): self.settings["schema"]["columns"].append(column) @property def is_feature_group(self): """ Indicates whether the Dataset is defined as a Feature Group, available in the Feature Store. :rtype: bool """ return self.settings["featureGroup"] def set_feature_group(self, status): """ (Un)sets the dataset as a Feature Group, available in the Feature Store. Changes of this property will be applied when calling :meth:`save` and require the "Manage Feature Store" permission. :param status: whether the dataset should be defined as a feature group :type status: bool """ self.settings["featureGroup"] = status def save(self): self.dataset.client._perform_empty( "PUT", "/projects/%s/datasets/%s" % (self.dataset.project_key, self.dataset.dataset_name), body=self.settings) class FSLikeDatasetSettings(DSSDatasetSettings): """ Settings for a files-based dataset. This class inherits from :class:`DSSDatasetSettings`. .. caution:: Do not instantiate this class directly, use :meth:`DSSDataset.get_settings` Use :meth:`save` to save your changes """ def __init__(self, dataset, settings): super(FSLikeDatasetSettings, self).__init__(dataset, settings) def set_connection_and_path(self, connection, path): self.settings["params"]["connection"] = connection self.settings["params"]["path"] = path def get_raw_format_params(self): """Get the raw format parameters as a dict""" return self.settings["formatParams"] def set_format(self, format_type, format_params = None): if format_params is None: format_params = {} self.settings["formatType"] = format_type self.settings["formatParams"] = format_params def set_csv_format(self, separator=",", style="excel", skip_rows_before=0, header_row=True, skip_rows_after=0): format_params = { "style" : style, "separator": separator, "skipRowsBeforeHeader": skip_rows_before, "parseHeaderRow": header_row, "skipRowsAfterHeader": skip_rows_after } self.set_format("csv", format_params) def set_partitioning_file_pattern(self, pattern): self.settings["partitioning"]["filePathPattern"] = pattern class SQLDatasetSettings(DSSDatasetSettings): """ Settings for a SQL dataset. This class inherits from :class:`DSSDatasetSettings`. .. caution:: Do not instantiate this class directly, use :meth:`DSSDataset.get_settings` Use :meth:`save` to save your changes """ def __init__(self, dataset, settings): super(SQLDatasetSettings, self).__init__(dataset, settings) def set_table(self, connection, schema, table, catalog=None): """ Sets this SQL dataset in 'table' mode, targeting a particular table of a connection Leave catalog to None to target the default database associated with the connection """ self.settings["params"].update({ "connection": connection, "mode": "table", "schema": schema, "table": table, "catalog": catalog }) class DSSManagedDatasetCreationHelper(object): """Provide an helper to create partitioned dataset .. code-block:: python import dataiku client = dataiku.api_client() project_key = dataiku.default_project_key() project = client.get_project(project_key) #create the dataset builder = project.new_managed_dataset("py_generated") builder.with_store_into("filesystem_folders") dataset = builder.create(overwrite=True) #setup format & schema settings ds_settings = ds.get_settings() ds_settings.set_csv_format() ds_settings.add_raw_schema_column({'name':'id', 'type':'int'}) ds_settings.add_raw_schema_column({'name':'name', 'type':'string'}) ds_settings.save() #put some data data = ["foo", "bar"] with ds.get_as_core_dataset().get_writer() as writer: for idx, val in enumerate(data): writer.write_row_array((idx, val)) .. caution:: do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.new_managed_dataset` """ def __init__(self, project, dataset_name): self.project = project self.dataset_name = dataset_name self.creation_settings = { "specificSettings" : {} } def get_creation_settings(self): return self.creation_settings def with_store_into(self, connection, type_option_id = None, format_option_id = None): """ Sets the connection into which to store the new managed dataset :param str connection: Name of the connection to store into :param str type_option_id: If the connection accepts several types of datasets, the type :param str format_option_id: Optional identifier of a file format option :returns: self """ self.creation_settings["connectionId"] = connection if type_option_id is not None: self.creation_settings["typeOptionId"] = type_option_id if format_option_id is not None: self.creation_settings["specificSettings"]["formatOptionId"] = format_option_id return self def with_copy_partitioning_from(self, dataset_ref, object_type='DATASET'): """ Sets the new managed dataset to use the same partitioning as an existing dataset_name :param str dataset_ref: Name of the dataset to copy partitioning from :returns: self """ code = 'dataset' if object_type == 'DATASET' else 'folder' self.creation_settings["partitioningOptionId"] = "copy:%s:%s" % (code, dataset_ref) return self def create(self, overwrite=False): """ Executes the creation of the managed dataset according to the selected options :param overwrite: If the dataset being created already exists, delete it first (removing data), defaults to False :type overwrite: bool, optional :returns: the newly created dataset :rtype: :class:`DSSDataset` """ if overwrite and self.already_exists(): self.project.get_dataset(self.dataset_name).delete(drop_data = True) self.project.client._perform_json("POST", "/projects/%s/datasets/managed" % self.project.project_key, body = { "name": self.dataset_name, "creationSettings": self.creation_settings }) return DSSDataset(self.project.client, self.project.project_key, self.dataset_name) def already_exists(self): """ :returns: whether this managed dataset already exists :rtype: bool """ dataset = self.project.get_dataset(self.dataset_name) try: dataset.get_metadata() return True except Exception as e: return False class DSSDatasetInfo(object): """ Info class for a DSS dataset (Read-Only). .. caution:: Do not instantiate this class directly, use :meth:`DSSDataset.get_info` """ def __init__(self, dataset, info): self.dataset = dataset self.info = info def get_raw(self): """ Get the raw dataset full information as a dict :returns: the raw dataset full information :rtype: dict """ return self.info @property def last_build_start_time(self): """ The last build start time of the dataset as a :class:`datetime.datetime` or None if there is no last build information. :returns: the last build start time :rtype: :class:`datetime.datetime` or None """ last_build_info = self.info.get("lastBuild", dict()) timestamp = last_build_info.get("buildStartTime", None) return datetime.datetime.fromtimestamp(timestamp / 1000) if timestamp is not None else None @property def last_build_end_time(self): """ The last build end time of the dataset as a :class:`datetime.datetime` or None if there is no last build information. :returns: the last build end time :rtype: :class:`datetime.datetime` or None """ last_build_info = self.info.get("lastBuild", dict()) timestamp = last_build_info.get("buildEndTime", None) return datetime.datetime.fromtimestamp(timestamp / 1000) if timestamp is not None else None @property def is_last_build_successful(self): """ Get whether the last build of the dataset is successful. :returns: True if the last build is successful :rtype: bool """ last_build_info = self.info.get("lastBuild", dict()) success = last_build_info.get("buildSuccess", False) return success dataiku-api-client-python-11.4.0/dataikuapi/dss/discussion.py000066400000000000000000000162711440464736400242460ustar00rootroot00000000000000import json import sys if sys.version_info >= (3,0): import urllib.parse dku_quote_fn = urllib.parse.quote else: import urllib dku_quote_fn = urllib.quote class DSSObjectDiscussions(object): """ A handle to manage discussions on a DSS object. .. important:: Do not create this class directly, instead use :meth:`dataikuapi.DSSClient.get_object_discussions` on any commentable DSS object. """ def __init__(self, client, project_key, object_type, object_id): self.client = client self.project_key = project_key self.object_type = object_type self.object_id = object_id # encode in UTF-8 if its python2 and unicode if sys.version_info < (3,0) and isinstance(self.object_id, unicode): self.object_id = self.object_id.encode('utf-8') def list_discussions(self): """ Gets the list of discussions on the object. :returns: list of discussions on the object :rtype: list of :class:`dataikuapi.dss.discussion.DSSDiscussion` """ data = self.client._perform_json("GET", "/projects/%s/discussions/%s/%s/" % (self.project_key, self.object_type, self.object_id)) return [DSSDiscussion(self.client, self.project_key, self.object_type, self.object_id, discu_data['id'], discu_data, False) for discu_data in data] def create_discussion(self, topic, message): """ Creates a new discussion with one message. :param str topic: the discussion topic :param str message: the markdown formatted first message :returns: the newly created discussion :rtype: :class:`dataikuapi.dss.discussion.DSSDiscussion` """ creation_data = { "topic" : topic, "reply" : message } discu_data = self.client._perform_json("POST", "/projects/%s/discussions/%s/%s/" % (self.project_key, self.object_type, self.object_id), body=creation_data) return DSSDiscussion(self.client, self.project_key, self.object_type, self.object_id, discu_data['id'], discu_data, True) def get_discussion(self, discussion_id): """ Gets a specific discussion. :param str discussion_id: the discussion ID :returns: the discussion :rtype: :class:`dataikuapi.dss.discussion.DSSDiscussion` """ discu_data = self.client._perform_json("GET", "/projects/%s/discussions/%s/%s/%s" % (self.project_key, self.object_type, self.object_id, discussion_id)) return DSSDiscussion(self.client, self.project_key, self.object_type, self.object_id, discussion_id, discu_data, True) class DSSDiscussion(object): """ A handle to interact with a discussion. .. important:: Do not call directly, use :meth:`dataikuapi.dss.discussion.DSSObjectDiscussions.get_discussion`, :meth:`dataikuapi.dss.discussion.DSSObjectDiscussions.create_discussion` or :meth:`dataikuapi.dss.discussion.DSSObjectDiscussions.list_discussions`. """ def __init__(self, client, project_key, object_type, object_id, discussion_id, discussion_data, discussion_data_has_replies): """ :param :class:`dataikuapi.dssclient.DSSClient` client: an api client to connect to the DSS backend :param str project_key: identifier of the project to access :param str object_type: DSS object type :param str object_id: DSS object ID :param str discussion_id: identifier of the discussion :param dict discussion_data: the discussion data :param bool discussion_data_has_replies: a flag that indicates if the replies has been loaded """ self.client = client self.project_key = project_key self.object_type = object_type self.object_id = object_id self.discussion_id = discussion_id self.discussion_data = discussion_data self.discussion_data_has_replies = discussion_data_has_replies def _get_with_replies(self): """ Reloads the discussion data from the backend including the replies. """ self.discussion_data = self.client._perform_json("GET", "/projects/%s/discussions/%s/%s/%s" % (self.project_key, self.object_type, self.object_id, self.discussion_id)) self.discussion_data_has_replies = True def get_metadata(self): """ Gets the discussion metadata. :returns: the discussion metadata :rtype: dict """ metadata = dict(self.discussion_data) if "replies" in metadata: del metadata["replies"] return metadata def set_metadata(self, discussion_metadata): """ Updates the discussion metadata. :param dict discussion_metadata: the discussion metadata """ if not self.discussion_data_has_replies: self._get_with_replies() edited_metadata = dict(discussion_metadata) edited_metadata["replies"] = self.discussion_data["replies"] self.discussion_data = self.client._perform_json("PUT", "/projects/%s/discussions/%s/%s/%s" % (self.project_key, self.object_type, self.object_id, self.discussion_id), body=edited_metadata) self.discussion_data_has_replies = True def get_replies(self): """ Gets the list of replies in this discussion. :returns: a list of replies :rtype: list of :class:`dataikuapi.dss.discussion.DSSDiscussionReply` """ if not self.discussion_data_has_replies: self._get_with_replies() return [DSSDiscussionReply(reply_data) for reply_data in self.discussion_data["replies"]] def add_reply(self, text): """ Adds a reply to a discussion. :param str text: the markdown formatted text to reply """ reply_data = { "reply": text } self.discussion_data = self.client._perform_json("POST", "/projects/%s/discussions/%s/%s/%s/replies/" % (self.project_key, self.object_type, self.object_id, self.discussion_id), body=reply_data) self.discussion_data_has_replies = True class DSSDiscussionReply(object): """ A read-only handle to access a discussion reply. .. important:: Do not create this class directly, use :meth:`dataikuapi.dss.discussion.DSSDiscussion.get_replies` """ def __init__(self, reply_data): self.reply_data = reply_data def get_raw_data(self): """ Gets the reply raw data. :returns: the reply data :rtype: dict """ return self.reply_data def get_text(self): """ Gets the reply text. :returns: the reply text :rtype: str """ return self.reply_data["text"] def get_author(self): """ Gets the reply author. :returns: the author ID :rtype: str """ return self.reply_data["author"] def get_timestamp(self): """ Gets the reply timestamp. :returns: the reply timestamp :rtype: long """ return self.reply_data["time"] def get_edited_timestamp(self): """ Gets the last edition timestamp. :returns: the last edition timestamp :rtype: long """ return self.reply_data["editedOn"] dataiku-api-client-python-11.4.0/dataikuapi/dss/feature_store.py000066400000000000000000000023761440464736400247330ustar00rootroot00000000000000from dataikuapi.dss.dataset import DSSDataset class DSSFeatureGroupListItem(object): def __init__(self, client, project_key, name): self.client = client self.project_key = project_key self.name = name @property def id(self): return self.project_key + "." + self.name def get_as_dataset(self): """ Gets the feature group as a dataset :return: a handle on the dataset :rtype: :class:`dataikuapi.dss.dataset.DSSDataset` """ return DSSDataset(self.client, self.project_key, self.name) class DSSFeatureStore(object): def __init__(self, client): """ A handle on the Feature Store. Do not create this class directly, use :meth:`DSSClient.get_feature_store` """ self.client = client def list_feature_groups(self): """ Get a list of feature groups on which the user has at least read permissions :return: list of feature groups :rtype: list of :class:`dataikuapi.feature_store.DSSFeatureGroupListItem` """ items = self.client._perform_json("GET", "/feature-store/feature-groups") return [DSSFeatureGroupListItem(self.client, item["projectKey"], item["name"]) for item in items] dataiku-api-client-python-11.4.0/dataikuapi/dss/flow.py000066400000000000000000001026541440464736400230330ustar00rootroot00000000000000from ..utils import _write_response_content_to_file from .utils import AnyLoc from .dataset import DSSDataset from .managedfolder import DSSManagedFolder from .savedmodel import DSSSavedModel from .modelevaluationstore import DSSModelEvaluationStore from .recipe import DSSRecipe, DSSRecipeDefinitionAndPayload from .future import DSSFuture from .streaming_endpoint import DSSStreamingEndpoint import logging, json class DSSProjectFlow(object): def __init__(self, client, project): self.client = client self.project = project def get_graph(self): """ Get the flow graph. :return: A handle to use the flow graph :rtype: :class:`.DSSProjectFlowGraph` """ data = self.client._perform_json("GET", "/projects/%s/flow/graph/" % (self.project.project_key)) return DSSProjectFlowGraph(self, data) def create_zone(self, name, color="#2ab1ac"): """ Creates a new flow zone. :param str name: new flow zone name :param str color: new flow zone color, in hexadecimal format #RRGGBB (defaults to **#2ab1ac**) :returns: the newly created zone :rtype: :class:`DSSFlowZone` """ data = self.client._perform_json("POST", "/projects/%s/flow/zones" % (self.project.project_key), body={ "name": name, "color": color }) return DSSFlowZone(self, data) def get_zone(self, id): """ Gets a single Flow zone by id. :param str id: flow zone id :returns: A flow zone :rtype: :class:`DSSFlowZone` """ data = self.client._perform_json("GET", "/projects/%s/flow/zones/%s" % (self.project.project_key, id)) return DSSFlowZone(self, data) def get_default_zone(self): """ Returns the default zone of the Flow. :returns: A flow zone :rtype: :class:`DSSFlowZone` """ return self.get_zone("default") def list_zones(self): """ Lists all zones in the Flow. :returns: A list of flow zones :rtype: list of :class:`DSSFlowZone` """ data = self.client._perform_json("GET", "/projects/%s/flow/zones" % (self.project.project_key)) return [DSSFlowZone(self, z) for z in data] def get_zone_of_object(self, obj): """ Finds the zone to which this object belongs. If the object is not found in any specific zone, it belongs to the default zone, and the default zone is returned. :param obj: An object to search :type obj: :class:`.DSSDataset`, :class:`.DSSManagedFolder`, :class:`.DSSSavedModel`, :class:`.DSSRecipe` :class:`.DSSModelEvaluationStore` or :class:`.DSSStreamingEndpoint` :returns: The flow zone containing the object :rtype: :class:`DSSFlowZone` """ sr = self._to_smart_ref(obj) for zone in self.list_zones(): for item in zone._raw["items"]: if json.dumps(sr) == json.dumps(item): return zone return self.get_default_zone() def replace_input_computable(self, current_ref, new_ref, type="DATASET"): """ This method replaces all references to a "computable" (Dataset, Managed Folder or Saved Model) as input of recipes in the whole Flow by a reference to another computable. No specific checks are performed. It is your responsibility to ensure that the schema of the new dataset will be compatible with the previous one (in the case of datasets). If `new_ref` references an object in a foreign project, this method will automatically ensure that `new_ref` is exposed to the current project. :param str current_ref: Either a "simple" object id (dataset name or model/folder/model evaluation store/streaming endpoint id) or a foreign object reference in the form "FOREIGN_PROJECT_KEY.local_id" :param str new_ref: Either a "simple" object id (dataset name or model/folder/model evaluation store/streaming endpoint id) or a foreign object reference in the form "FOREIGN_PROJECT_KEY.local_id" :param str type: The type of object being replaced (DATASET, SAVED_MODEL, MANAGED_FOLDER, MODEL_EVALUATION_STORE, STREAMING_ENDPOINT) (defaults to **DATASET**) """ new_loc = AnyLoc.from_ref(self.project.project_key, new_ref) if new_loc.project_key != self.project.project_key: logging.info("New ref is in project %s, exposing it to project %s" % (new_loc.project_key, self.project.project_key)) new_ref_src_project = self.client.get_project(new_loc.project_key) settings = new_ref_src_project.get_settings() settings.add_exposed_object(type, new_loc.object_id, self.project.project_key) settings.save() for recipe in self.project.list_recipes(): recipe_handle = self.project.get_recipe(recipe["name"]) fake_rap = DSSRecipeDefinitionAndPayload(recipe_handle, {"recipe": recipe}) if fake_rap.has_input(current_ref): logging.info("Recipe %s has %s as input, performing the replacement by %s" % \ (recipe["name"], current_ref, new_ref)) recipe_obj = self.project.get_recipe(recipe["name"]) dap = recipe_obj.get_definition_and_payload() dap.replace_input(current_ref, new_ref) recipe_obj.set_definition_and_payload(dap) def generate_documentation(self, folder_id=None, path=None): """ Start the flow document generation from a template docx file in a managed folder, or from the default template if no folder id and path are specified. :param str folder_id: the id of the managed folder (defaults to **None**) :param str path: the path to the file from the root of the folder (defaults to **None**) :return: The flow document generation future :rtype: :class:`.DSSFuture` """ if bool(folder_id) != bool(path): raise ValueError("Both folder id and path arguments are required to use a template from folder. " + "Use without argument to generate the flow documentation using the default template") template_mode_url = "" if folder_id is None and path is None else "-with-template-in-folder" f = self.client._perform_json("POST", "/projects/%s/flow/documentation/generate%s" % (self.project.project_key, template_mode_url), params={"folderId": folder_id, "path": path}) return DSSFuture(self.client, f["jobId"]) def generate_documentation_from_custom_template(self, fp): """ Start the flow document generation from a docx template (as a file object). :param object fp: A file-like object pointing to a template docx file :return: The flow document generation future :rtype: :class:`.DSSFuture` """ files = {'file': fp} f = self.client._perform_json("POST", "/projects/%s/flow/documentation/generate-with-template" % self.project.project_key, files=files) return DSSFuture(self.client, f["jobId"]) def download_documentation_stream(self, export_id): """ Download a flow documentation, as a binary stream. .. warning :: You need to close the stream after download. Failure to do so will result in the DSSClient becoming unusable. :param str export_id: the id of the generated flow documentation returned as the result of the future :returns: the generated flow documentation file as a stream :rtype: :class:`requests.Response` """ return self.client._perform_raw("GET", "/projects/%s/flow/documentation/generated/%s" % (self.project.project_key, export_id)) def download_documentation_to_file(self, export_id, path): """ Download a flow documentation into the given output file. :param str export_id: the id of the generated flow documentation returned as the result of the future :param str path: the path where to download the flow documentation """ stream = self.download_documentation_stream(export_id) _write_response_content_to_file(stream, path) ######################################################## # Flow tools ######################################################## def start_tool(self, type, data={}): """ Start a tool or open a view in the flow. :param str type: one of {COPY, CHECK_CONSISTENCY, PROPAGATE_SCHEMA} (tools) or {TAGS, CUSTOM_FIELDS, CONNECTIONS, COUNT_OF_RECORDS, FILESIZE, FILEFORMATS, RECIPES_ENGINES, RECIPES_CODE_ENVS, IMPALA_WRITE_MODE, HIVE_MODE, SPARK_ENGINE, SPARK_CONFIG, SPARK_PIPELINES, SQL_PIPELINES, PARTITIONING, PARTITIONS, SCENARIOS, CREATION, LAST_MODIFICATION, LAST_BUILD, RECENT_ACTIVITY, WATCH} (views) :param dict data: initial data for the tool (defaults to **{}**) :returns: A handle to interact with the newly-created tool or view :rtype: :class:`.DSSFlowTool` """ tool_id = self.client._perform_text("POST", "/projects/%s/flow/tools/" % self.project.project_key, params={'type': type}, body=data) return DSSFlowTool(self.client, self.project.project_key, tool_id) def new_schema_propagation(self, dataset_name): """ Start an automatic schema propagation from a dataset. :param str dataset_name: name of a dataset to start propagating from :returns: A handle to set options and start the propagation :rtype: :class:`.DSSSchemaPropagationRunBuilder` """ return DSSSchemaPropagationRunBuilder(self.project, self.client, dataset_name) def _to_smart_ref(self, obj): if isinstance(obj, DSSDataset): ot = "DATASET" elif isinstance(obj, DSSManagedFolder): ot = "MANAGED_FOLDER" elif isinstance(obj, DSSSavedModel): ot = "SAVED_MODEL" elif isinstance(obj, DSSModelEvaluationStore): ot = "MODEL_EVALUATION_STORE" elif isinstance(obj, DSSRecipe): ot = "RECIPE" elif isinstance(obj, DSSStreamingEndpoint): ot = "STREAMING_ENDPOINT" else: raise ValueError("Cannot transform to DSS object ref: %s" % obj) if obj.project_key == self.project.project_key: return { "objectId": obj.id, "objectType": ot } else: return { "projectKey": obj.project_key, "objectId": obj.id, "objectType": ot } class DSSSchemaPropagationRunBuilder(object): """ .. important :: Do not create this directly, use :meth:`DSSProjectFlow.new_schema_propagation`. """ def __init__(self, project, client, dataset_name): self.project = project self.client = client self.dataset_name = dataset_name self.settings = { 'recipeUpdateOptions': { "byType": {}, "byName": {} }, 'defaultPartitionValuesByDimension': {}, 'partitionsByComputable': {}, 'excludedRecipes': [], 'markAsOkRecipes': [], 'autoRebuild': True } def set_auto_rebuild(self, auto_rebuild): """ Sets whether to automatically rebuild datasets if needed while propagating. :param bool auto_rebuild: whether to automatically rebuild datasets if needed (defaults to **True**) """ self.settings["autoRebuild"] = auto_rebuild def set_default_partitioning_value(self, dimension, value): """ In the case of partitioned flows, sets the default partition value to use when rebuilding, for a specific dimension name. :param str dimension: a partitioning dimension name :param str value: a partitioning dimension value """ self.settings["defaultPartitionValuesByDimension"][dimension] = value def set_partition_for_computable(self, full_id, partition): """ In the case of partitioned flows, sets the partition id to use when building a particular computable. Overrides the default partitioning value per dimension. :param str full_id: Full name of the computable, in the form PROJECTKEY.id :param str partition: a full partition id (all dimensions) """ self.settings["partitionsByComputable"][full_id] = partition def stop_at(self, recipe_name): """ Sets the given recipe as a schema propagation stop mark. :param str recipe_name: the name of the recipe """ self.settings["excludedRecipes"].append(recipe_name) def mark_recipe_as_ok(self, name): """ Marks a recipe as always considered as OK during propagation. :param str name: recipe to mark as ok """ self.settings["markAsOkRecipes"].append(name) def set_grouping_update_options(self, recipe=None, remove_missing_aggregates=True, remove_missing_keys=True, new_aggregates={}): """ Sets update options for grouping recipes. :param str recipe: if None, applies to all grouping recipes. Else, applies only to this name (defaults to **None**) :param bool remove_missing_aggregates: whether to remove missing aggregates (defaults to **True**) :param bool remove_missing_keys: whether to remove missing keys (defaults to **True**) :param dict new_aggregates: new aggregates (defaults to **{}**) """ data = { "removeMissingAggregates": remove_missing_aggregates, "removeMissingKeys": remove_missing_keys, "newAggregates": new_aggregates } if recipe is None: self.settings["recipeUpdateOptions"]["byType"]["grouping"] = data else: self.settings["recipeUpdateOptions"]["byName"][recipe] = data def set_window_update_options(self, recipe=None, remove_missing_aggregates=True, remove_missing_in_window=True, new_aggregates={}): """ Sets update options for window recipes. :param str recipe: if None, applies to all window recipes. Else, applies only to this name (defaults to **None**) :param bool remove_missing_aggregates: whether to remove missing aggregates (defaults to **True**) :param bool remove_missing_in_window: whether to remove missing keys in windows (defaults to **True**) :param dict new_aggregates: new aggregates (defaults to **{}**) """ data = { "removeMissingAggregates": remove_missing_aggregates, "removeMissingInWindow": remove_missing_in_window, "newAggregates": new_aggregates } if recipe is None: self.settings["recipeUpdateOptions"]["byType"]["window"] = data else: self.settings["recipeUpdateOptions"]["byName"][recipe] = data def set_join_update_options(self, recipe=None, remove_missing_join_conditions=True, remove_missing_join_values=True, new_selected_columns={}): """ Sets update options for join recipes. :param str recipe: if None, applies to all join recipes. Else, applies only to this name (defaults to **None**) :param bool remove_missing_join_conditions: whether to remove missing join conditions (defaults to **True**) :param bool remove_missing_join_values: whether to remove missing join values (defaults to **True**) :param dict new_selected_columns: new selected columns (defaults to **{}**) """ data = { "removeMissingJoinConditions": remove_missing_join_conditions, "removeMissingJoinValues": remove_missing_join_values, "newSelectedColumns": new_selected_columns } if recipe is None: self.settings["recipeUpdateOptions"]["byType"]["join"] = data else: self.settings["recipeUpdateOptions"]["byName"][recipe] = data def start(self): """ Starts the actual propagation. Returns a future to wait for completion. :returns: A future representing the schema propagation job :rtype: :class:`.DSSFuture` """ future_resp = self.client._perform_json("POST", "/projects/%s/flow/tools/propagate-schema/%s/" % (self.project.project_key, self.dataset_name), body=self.settings) return DSSFuture.from_resp(self.client, future_resp) class DSSFlowZone(object): """ A zone in the Flow. .. important :: Do not create this object manually, use :meth:`DSSProjectFlow.get_zone` or :meth:`DSSProjectFlow.list_zones` """ def __init__(self, flow, data): self.flow = flow self.client = flow.client self._raw = data @property def id(self): return self._raw["id"] @property def name(self): return self._raw["name"] @property def color(self): return self._raw["color"] def __repr__(self): return "" % (self.id, self.name) def get_settings(self): """Gets the settings of this zone in order to modify them. :returns: The settings of the flow zone :rtype: :class:`DSSFlowZoneSettings` """ return DSSFlowZoneSettings(self) def _to_native_obj(self, zone_item): if not "projectKey" in zone_item or zone_item["projectKey"] == self.flow.project.project_key: p = self.flow.project else: p = self.client.get_project(zone_item["projectKey"]) if zone_item["objectType"] == "DATASET": return p.get_dataset(zone_item["objectId"]) elif zone_item["objectType"] == "MANAGED_FOLDER": return p.get_managed_folder(zone_item["objectId"]) elif zone_item["objectType"] == "SAVED_MODEL": return p.get_saved_model(zone_item["objectId"]) elif zone_item["objectType"] == "RECIPE": return p.get_recipe(zone_item["objectId"]) elif zone_item["objectType"] == "STREAMING_ENDPOINT": return p.get_streaming_endpoint(zone_item["objectId"]) else: raise ValueError("Cannot transform to DSS object: %s" % zone_item) def add_item(self, obj): """ Adds an item to this zone. The item will automatically be moved from its existing zone. Additional items may be moved to this zone as a result of the operation (notably the recipe generating `obj`). :param obj: object to add to the zone :type obj: :class:`.DSSDataset`, :class:`.DSSManagedFolder`, :class:`.DSSSavedModel`, :class:`.DSSRecipe`, :class:`.DSSModelEvaluationStore` or :class:`.DSSStreamingEndpoint` """ self._raw = self.client._perform_json("POST", "/projects/%s/flow/zones/%s/items" % (self.flow.project.project_key, self.id), body=self.flow._to_smart_ref(obj)) def add_items(self, items): """ Adds items to this zone. The items will automatically be moved from their existing zones. Additional items may be moved to this zone as a result of the operations (notably the recipe generating the `items`). :param items: A list of objects to add to the zone :type items: list of :class:`.DSSDataset`, :class:`.DSSManagedFolder`, :class:`.DSSSavedModel`, :class:`.DSSRecipe`, :class:`.DSSModelEvaluationStore` or :class:`.DSSStreamingEndpoint` """ smart_refs = [] for item in items: smart_refs.append(self.flow._to_smart_ref(item)) self._raw = self.client._perform_json("POST", "/projects/%s/flow/zones/%s/add-items" % (self.flow.project.project_key, self.id), body=smart_refs) @property def items(self): """ The list of items explicitly belonging to this zone. This list is read-only, to modify it, use :meth:`add_item` and :meth:`remove_item`. Note that the "default" zone never has any items, as it contains all items that are not explicitly in a zone. To get the full list of items in a zone, including in the "default" zone, use the :meth:`get_graph` method. :returns: the items in the zone :rtype: list of :class:`.DSSDataset`, :class:`.DSSManagedFolder`, :class:`.DSSSavedModel`, :class:`.DSSRecipe`, :class:`.DSSModelEvaluationStore` or :class:`.DSSStreamingEndpoint` """ return [self._to_native_obj(i) for i in self._raw["items"]] def add_shared(self, obj): """ Share an item to this zone. The item will not be automatically unshared from its existing zone. :param obj: object to share to the zone :type obj: :class:`.DSSDataset`, :class:`.DSSManagedFolder`, :class:`.DSSSavedModel`, :class:`.DSSRecipe`, :class:`.DSSModelEvaluationStore` or :class:`.DSSStreamingEndpoint` """ self._raw = self.client._perform_json("POST", "/projects/%s/flow/zones/%s/shared" % (self.flow.project.project_key, self.id), body=self.flow._to_smart_ref(obj)) def remove_shared(self, obj): """ Remove a shared item from this zone. :param obj: object to remove from the zone :type obj: :class:`.DSSDataset`, :class:`.DSSManagedFolder`, :class:`.DSSSavedModel`, :class:`.DSSModelEvaluationStore` or :class:`.DSSStreamingEndpoint` """ smartRef = self.flow._to_smart_ref(obj) self._raw = self.client._perform_json("DELETE", "/projects/%s/flow/zones/%s/shared/%s/%s" % ( self.flow.project.project_key, self.id, smartRef['objectType'], smartRef['objectId'])) @property def shared(self): """ The list of items that have been explicitly pre-shared to this zone. This list is read-only, to modify it, use :meth:`add_shared` and :meth:`remove_shared` :returns: the items shared to this zone :rtype: list of :class:`.DSSDataset`, :class:`.DSSManagedFolder`, :class:`.DSSSavedModel`, :class:`.DSSModelEvaluationStore` or :class:`.DSSStreamingEndpoint` """ return [self._to_native_obj(i) for i in self._raw["shared"]] def get_graph(self): """ Get the flow graph. :return: A handle to use the flow graph :rtype: :class:`DSSProjectFlowGraph` """ data = self.client._perform_json("GET", "/projects/%s/flow/zones/%s/graph" % (self.flow.project.project_key, self.id)) return DSSProjectFlowGraph(self.flow, data) def delete(self): """ Delete the zone, all items will be moved to the default zone. """ return self.client._perform_empty("DELETE", "/projects/%s/flow/zones/%s" % (self.flow.project.project_key, self.id)) class DSSFlowZoneSettings(object): """ The settings of a flow zone. .. important :: Do not create this directly, use :meth:`DSSFlowZone.get_settings`. """ def __init__(self, zone): self._zone = zone self._raw = zone._raw def get_raw(self): """ Gets the raw settings of the zone. .. note :: You cannot modify the `items` and `shared` elements through this class. Instead, use :meth:`DSSFlowZone.add_item` and others """ return self._raw @property def name(self): return self._raw["name"] @name.setter def name(self, new_name): self._raw["name"] = new_name @property def color(self): return self._raw["color"] @color.setter def color(self, new_color): self._raw["color"] = new_color def save(self): """Saves the settings of the zone""" self._zone.client._perform_empty("PUT", "/projects/%s/flow/zones/%s" % (self._zone.flow.project.project_key, self._zone.id), body=self._raw) class DSSProjectFlowGraph(object): def __init__(self, flow, data): self.flow = flow self.data = data self.nodes = data["nodes"] def get_source_computables(self, as_type="dict"): """ :param str as_type: How to return the source computables. Possible values are "dict" and "object" (defaults to **dict**) :returns: The list of source computables :rtype: If as_type=dict, each computable is returned as a dict containing at least "ref" and "type". If as_type=object, each computable is returned as a :class:`.DSSDataset`, :class:`.DSSManagedFolder`, :class:`.DSSSavedModel`, :class:`.DSSModelEvaluationStore` or :class:`.DSSStreamingEndpoint` """ ret = [] for node in self.nodes.values(): if len(node["predecessors"]) == 0 and node["type"].startswith("COMPUTABLE"): ret.append(node) return self._convert_nodes_list(ret, as_type) def get_source_recipes(self, as_type="dict"): """ :param str as_type: How to return the source recipes. Possible values are "dict" and "object" (defaults to **dict**) :returns: The list of source recipes :rtype: If as_type=dict, each recipe is returned as a dict containing at least "ref" and "type". If as_type=object, each recipe is returned as a :class:`.DSSRecipe`. """ ret = [] for node in self.nodes.values(): if len(node["predecessors"]) == 0 and node["type"] == "RUNNABLE_RECIPE": ret.append(node) return self._convert_nodes_list(ret, as_type) def get_source_datasets(self): """ :returns: The list of source datasets for this project :rtype: List of :class:`.DSSDataset` """ return [self._get_object_from_graph_node(x) for x in self.get_source_computables() if x["type"] == "COMPUTABLE_DATASET"] def get_successor_recipes(self, node, as_type="dict"): """ :param node: Either a name or a dataset object :type node: str or :class:`.DSSDataset` :param str as_type: How to return the successor recipes. Possible values are "dict" and "object" (defaults to **dict**) :returns: A list of recipes that are a successor of the given graph node :rtype: If as_type=dict, each recipe is returned as a dict containing at least "ref" and "type". If as_type=object, each recipe is returned as a :class:`.DSSRecipe`. """ if isinstance(node, DSSDataset): node = node.dataset_name computable = self.nodes.get(node, None) if computable is None: raise ValueError("Computable %s not found in Flow graph" % node) runnables = [self.nodes[x] for x in computable["successors"]] return self._convert_nodes_list(runnables, as_type) def get_successor_computables(self, node, as_type="dict"): """ :param node: Either a name or a recipe object :type node: str or :class:`.DSSRecipe` :param str as_type: How to return the successor computables. Possible values are "dict" and "object" (defaults to **dict**). :returns: A list of computables that are a successor of a given graph node :rtype: If as_type=dict, each computable is returned as a dict containing at least "ref" and "type". If as_type=object, each computable is returned as a :class:`.DSSDataset`. :class:`.DSSManagedFolder`, :class:`.DSSSavedModel`, :class:`.DSSModelEvaluationStore` or :class:`.DSSStreamingEndpoint` """ if isinstance(node, DSSRecipe): node = node.recipe_name runnable = self.nodes.get(node, None) if runnable is None: raise ValueError("Runnable %s not found in Flow graph" % node) computables = [self.nodes[x] for x in runnable["successors"]] return self._convert_nodes_list(computables, as_type) def _convert_nodes_list(self, nodes, as_type): actual_nodes = [node for node in nodes if node['type'] != 'RUNNABLE_IMPLICIT_RECIPE'] if as_type == "object" or as_type == "objects": return [self._get_object_from_graph_node(node) for node in actual_nodes] else: return nodes def _get_object_from_graph_node(self, node): if node["type"] == "COMPUTABLE_DATASET": return DSSDataset(self.flow.client, self.flow.project.project_key, node["ref"]) elif node["type"] == "RUNNABLE_RECIPE": return DSSRecipe(self.flow.client, self.flow.project.project_key, node["ref"]) elif node["type"] == "COMPUTABLE_FOLDER": return DSSManagedFolder(self.flow.client, self.flow.project.project_key, node["ref"]) elif node["type"] == "COMPUTABLE_SAVED_MODEL": return DSSSavedModel(self.flow.client, self.flow.project.project_key, node["ref"]) elif node["type"] == "COMPUTABLE_STREAMING_ENDPOINT": return DSSStreamingEndpoint(self.flow.client, self.flow.project.project_key, node["ref"]) else: # TODO add streaming elements raise Exception("unsupported node type: %s" % node["type"]) def get_items_in_traversal_order(self, as_type="dict"): """ Get the list of nodes in left to right order. :param str as_type: How to return the nodes. Possible values are "dict" and "object" (defaults to **dict**). :returns: A list of nodes :rtype: If as_type=dict, each item is returned as a dict containing at least "ref" and "type". If as_type=object, each item is returned as a :class:`.DSSDataset`. :class:`.DSSManagedFolder`, :class:`.DSSSavedModel`, :class:`.DSSModelEvaluationStore`, :class:`.DSSStreamingEndpoint` or :class:`.DSSRecipe` """ ret = [] def add_to_set(node): ret.append(node) def in_set(obj): for candidate in ret: if candidate["type"] == obj["type"] and candidate["ref"] == obj["ref"]: return True return False def add_from(graph_node): # To keep traversal order, we recurse to predecessors first for predecessor_ref in graph_node["predecessors"]: predecessor_node = self.nodes[predecessor_ref] if not in_set(predecessor_node): add_from(predecessor_node) # Then add ourselves if not in_set(graph_node): add_to_set(graph_node) # Then recurse to successors for successor_ref in graph_node["successors"]: successor_node = self.nodes[successor_ref] if not in_set(successor_node): add_from(successor_node) for source_computable in self.get_source_computables(): add_from(source_computable) for source_recipe in self.get_source_recipes(): add_from(source_recipe) return self._convert_nodes_list(ret, as_type) class DSSFlowTool(object): """ Handle to interact with a flow tool. """ def __init__(self, client, project_key, tool_id): self.client = client self.project_key = project_key self.tool_id = tool_id def stop(self): """ Stops the tool and releases the resources held by it. """ return self.client._perform_json("POST", "/projects/%s/flow/tools/%s/stop" % (self.project_key, self.tool_id)) def get_state(self, options={}): """ Get the current state of the tool or view. :param dict options: options (defaults to **{}**) :returns: the state :rtype: dict """ return self.client._perform_json("GET", "/projects/%s/flow/tools/%s/state" % (self.project_key, self.tool_id), body=options) def do(self, action): """ Perform a manual user action on the tool. :param dict action: the action to do. It must contain: * a string `name` (the object id) * a string `type` (`"MARK_DATASET_AS_REBUILT"` or `"MARK_RECIPE_AS_OK"`) If `type` is `"MARK_RECIPE_AS_OK"`, a boolean `updated` can be added to the dict, if `True`, marks the recipe as OK after the update. :returns: the current state :rtype: dict """ return self.client._perform_json("PUT", "/projects/%s/flow/tools/%s/action" % (self.project_key, self.tool_id), body=action) def update(self, options={}): """ (for tools only) Start updating the tool state. :params options dict: options for the update (defaults to **{}**) :returns: A handle to interact with task of performing the update :rtype: :class:`.DSSFuture` """ update_future = self.client._perform_json("POST", "/projects/%s/flow/tools/%s/update" % (self.project_key, self.tool_id), body=options) return DSSFuture(self.client, update_future.get('jobId', None), update_future) dataiku-api-client-python-11.4.0/dataikuapi/dss/future.py000066400000000000000000000117161440464736400233740ustar00rootroot00000000000000import time class DSSFuture(object): """ A future represents a long-running task on a DSS instance. It allows you to track the state of the task, retrieve its result when it is ready or abort it. Usage example: .. code-block:: python # In this example, 'folder' is a DSSManagedFolder future = folder.compute_metrics() # Waits until the metrics are computed metrics = future.wait_for_result() .. note:: This class does not need to be instantiated directly. A :class:`dataikuapi.dss.future.DSSFuture` is usually returned by the API calls that are initiating long running-tasks. """ def __init__(self, client, job_id, state=None, result_wrapper=lambda result: result): self.client = client self.job_id = job_id self.state = state self.state_is_peek = True self.result_wrapper = result_wrapper @staticmethod def from_resp(client, resp, result_wrapper=lambda result: result): """ Creates a :class:`dataikuapi.dss.future.DSSFuture` from the response of an endpoint that initiated a long-running task. :param client: An api client to connect to the DSS backend :type client: :class:`dataikuapi.dssclient.DSSClient` :param resp: The response of the API call that initiated a long-running task. :type resp: dict :param result_wrapper: (optional) a function to apply to the result of the future, before it is returned by `get_result` or `wait_for_result` methods. :type result_wrapper: callable :return: :class:`dataikuapi.dss.future.DSSFuture` """ return DSSFuture(client, resp.get('jobId', None), state=resp, result_wrapper=result_wrapper) @classmethod def get_result_wait_if_needed(cls, client, ret): """ :meta private: """ if 'jobId' in ret: future = DSSFuture(client, ret["jobId"], ret) future.wait_for_result() return future.get_result() else: return ret['result'] def abort(self): """ Aborts the long-running task. """ self.client._perform_empty("DELETE", "/futures/%s" % self.job_id) def get_state(self): """ Queries the state of the future, and fetches the result if it's ready. :return: the state of the future, including the result if it is ready. :rtype: dict """ self.state = self.client._perform_json( "GET", "/futures/%s" % self.job_id, params={'peek': False}) self.state_is_peek = False return self.state def peek_state(self): """ Queries the state of the future, without fetching the result. :return: the state of the future :rtype: dict """ self.state = self.client._perform_json( "GET", "/futures/%s" % self.job_id, params={'peek': True}) self.state_is_peek = True return self.state def get_result(self): """ Returns the future's result if it's ready. .. note:: if a custom result_wrapper was provided, it will be applied on the result that will be returned. :return: the result of the future :rtype: object :raises Exception: if the result is not ready (i.e. the task is still running, or it has failed) """ if self.state is None or not self.state.get('hasResult', False) or self.state_is_peek: self.get_state() if self.state.get('hasResult', False): return self.result_wrapper(self.state.get('result', None)) else: raise Exception("Result not ready") def has_result(self): """ Checks whether the task is completed successfully and has a result. :return: True if the task has successfully returned a result, False otherwise :rtype: bool """ if self.state is None or not self.state.get('hasResult', False): self.get_state() return self.state.get('hasResult', False) def wait_for_result(self): """ Waits for the completion of the long-running task, and returns its result. .. note:: if a custom result_wrapper was provided, it will be applied on the result that will be returned. :return: the result of the future :rtype: object :raises Exception: if the task failed """ if self.state is not None and self.state.get('hasResult', False): # no future created in backend, result already in the state return self.result_wrapper(self.state.get('result', None)) if self.state is None or not self.state.get('hasResult', False) or self.state_is_peek: self.get_state() while not self.state.get('hasResult', False): time.sleep(5) self.get_state() if self.state.get('hasResult', False): return self.result_wrapper(self.state.get('result', None)) else: raise Exception("No result") dataiku-api-client-python-11.4.0/dataikuapi/dss/job.py000066400000000000000000000057671440464736400226450ustar00rootroot00000000000000import time import sys from ..utils import DataikuException class DSSJob(object): """ A job on the DSS instance. .. important:: Do not instantiate this class directly, instead use :meth:`dataikuapi.dss.project.DSSProject.get_job` or :meth:`dataikuapi.dss.project.DSSProject.start_job`. """ def __init__(self, client, project_key, id): self.client = client self.id = id self.project_key = project_key def abort(self): """ Aborts the job. :returns: a confirmation message for the request :rtype: dict """ return self.client._perform_json( "POST", "/projects/%s/jobs/%s/abort" % (self.project_key, self.id)) def get_status(self): """ Gets the current status of the job. :returns: the state of the job :rtype: dict """ return self.client._perform_json( "GET", "/projects/%s/jobs/%s/" % (self.project_key, self.id)) def get_log(self, activity=None): """ Gets the logs of the job. If an activity is passed in the parameters the logs will be scoped to that activity. :param string activity: (optional) the name of the activity in the job whose log is requested (defaults to **None**) :returns: the job logs :rtype: string """ return self.client._perform_text( "GET", "/projects/%s/jobs/%s/log" % (self.project_key, self.id), params={ "activity" : activity }) class DSSJobWaiter(object): """ Creates a helper to wait for the completion of a job. :param job: The job to wait for :type job: :class:`dataikuapi.dss.job.DSSJob` """ def __init__(self, job): self.job = job def wait(self, no_fail=False): """ Waits for the job. While waiting, if the job was aborted or if it failed an exception is raised depending on the `no_fail` parameter. If the job already stopped before entering this function its state will be returned regardless and no error will be raised. :param boolean no_fail: (optional) should an error be raised if the job finished with another status than `DONE` (defaults to **False**) :raises DataikuException: when the job does not complete successfully :returns: the job state :rtype: dict """ job_state = self.job.get_status().get("baseStatus", {}).get("state", "") sleep_time = 2 while job_state not in ["DONE", "ABORTED", "FAILED"]: sleep_time = 60 if sleep_time >= 60 else sleep_time * 1.2 time.sleep(int(sleep_time)) job_state = self.job.get_status().get("baseStatus", {}).get("state", "") if job_state in ["ABORTED", "FAILED"]: if no_fail: break else: raise DataikuException("Job run did not finish. Status: %s" % (job_state)) return job_state dataiku-api-client-python-11.4.0/dataikuapi/dss/jupyternotebook.py000066400000000000000000000256701440464736400253310ustar00rootroot00000000000000from .discussion import DSSObjectDiscussions from .utils import DSSTaggableObjectListItem class DSSJupyterNotebookListItem(DSSTaggableObjectListItem): """ An item in a list of Jupyter notebooks. .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.list_jupyter_notebooks` """ def __init__(self, client, data): super(DSSJupyterNotebookListItem, self).__init__(data) self.client = client def to_notebook(self): """ Get a handle on the corresponding notebook :rtype: :class:`DSSJupyterNotebook` """ return DSSJupyterNotebook(self.client, self._data["projectKey"], self._data["name"]) @property def name(self): """ Get the name of the notebook. Used as identifier. """ return self._data["name"] @property def language(self): """ Get the language of the notebook. Possible values are 'python', 'R', 'toree' (scala) :rtype: string """ return self._data["language"] @property def kernel_spec(self): """ Get the raw kernel spec of the notebook. The kernel spec is internal data for Jupyter, that identifies the kernel. :return: the spec of a Jupyter kernel. Top-level fields are: * **name** : identifier of the kernel in Jupyter, for example 'python2' or 'python3' * **display_name** : name of the kernel as shown in the Jupyter UI * **language** : language of the notebook (informative field) :rtype: dict """ return self._data["kernelSpec"] class DSSJupyterNotebook(object): """ A handle on a Python/R/scala notebook. .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.get_jupyter_notebook()` or :meth:`dataikuapi.dss.project.DSSProject.create_jupyter_notebook()`. """ def __init__(self, client, project_key, notebook_name): self.client = client self.project_key = project_key self.notebook_name = notebook_name def unload(self, session_id=None): """ Stop this Jupyter notebook and release its resources. """ sessions = self.get_sessions() if sessions is None: raise Exception("Notebook isn't running") if len(sessions) == 0: raise Exception("Notebook isn't running") if session_id is None: if len(sessions) > 1: raise Exception("Several sessions of the notebook are running, choose one") else: session_id = sessions[0].get('sessionId', None) return self.client._perform_json("DELETE", "/projects/%s/jupyter-notebooks/%s/sessions/%s" % (self.project_key, self.notebook_name, session_id)) def get_sessions(self, as_objects=False): """ Get the list of running sessions of this Jupyter notebook. Usage example: .. code-block:: python # list the running notebooks in a project for notebook in project.list_jupyter_notebooks(as_type="object"): if len(notebook.get_sessions()) > 0: print("Notebook %s is running" % notebook.notebook_name) :param boolean as_objects: if True, each returned item will be a :class:`dataikuapi.dss.jupyternotebook.DSSNotebookSession` :return: a list of :class:`dataikuapi.dss.jupyternotebook.DSSNotebookSession` if **as_objects** is True, a list of dict otherwise. The dict has fields: * **sessionId** : identifier of the session in the Jupyter server * **notebookName** : name of the notebook * **projectKey** : key of the DSS project containing the notebookd * **kernelId** : identifier of the running kernel in the Jupyter server (a session can change the kernel, for example if you restart it) * **kernelConnections** : how many kernels are connected to this session * **kernelExecutionState** : state of the notebook in Jupyter. Common states are 'idle' and 'busy' * **kernelId** : identifier of the running kernel in the Jupyter server * **kernelLastActivityTime** : timestamp in milliseconds of the last run of a cell in the notebook * **kernelPid** : pid of the process running the kernel * **sessionCreator** : name of user that started the session * **sessionCreatorDisplayName** : display name of user that started the session * **sessionStartTime** : timestamp in milliseconds of the session creation :rtype: list """ sessions = self.client._perform_json("GET", "/projects/%s/jupyter-notebooks/%s/sessions" % (self.project_key, self.notebook_name)) if as_objects: return [DSSNotebookSession(self.client, session) for session in sessions] else: return sessions def get_content(self): """ Get the full contents of this Jupyter notebook. The content comprises metadata, cells, notebook format info. Usage example: .. code-block:: python # collect all the source code of a notebook lines = [] for cell in notebook.get_content().get_cells(): if cell["cell_type"] != 'code': continue lines = lines + cell["source"] print('\\n'.join(lines)) :return: a list of :class:`dataikuapi.dss.jupyternotebook.DSSNotebookContent` :rtype: list """ raw_content = self.client._perform_json("GET", "/projects/%s/jupyter-notebooks/%s" % (self.project_key, self.notebook_name)) return DSSNotebookContent(self.client, self.project_key, self.notebook_name, raw_content) def delete(self): """ Delete this Jupyter notebook and stop all of its active sessions. """ return self.client._perform_json("DELETE", "/projects/%s/jupyter-notebooks/%s" % (self.project_key, self.notebook_name)) ######################################################## # Discussions ######################################################## def get_object_discussions(self): """ Get a handle to manage discussions on the notebook. :return: the handle to manage discussions :rtype: :class:`dataikuapi.dss.discussion.DSSObjectDiscussions` """ return DSSObjectDiscussions(self.client, self.project_key, "JUPYTER_NOTEBOOK", self.notebook_name) class DSSNotebookContent(object): """ The content of a Jupyter Notebook. This is the actual notebook data, see the `nbformat doc `_ . .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.jupyternotebook.DSSJupyterNotebook.get_content` """ def __init__(self, client, project_key, notebook_name, content): self.client = client self.project_key = project_key self.notebook_name = notebook_name self.content = content def get_raw(self): """ Get the raw content of this Jupyter notebook. The content comprises metadata, cells, notebook format info. :return: a dict containing the full content of a notebook. Notable fields are: * **metadata** : the metadata of the notebook, as a dict (see :meth:`get_metadata()`) * **nbformat** and **nbformat_minor** : the version of the notebook format * **cells** : list of cells, each one a dict (see :meth:`get_cells()`) :rtype: dict """ return self.content def get_metadata(self): """ Get the metadata associated to this Jupyter notebook. :return: the Jupyter metadata of the notebook, with fields: * **kernelspec** : identification of the kernel used to run the notebook * **creator** : name of the user that created the notebook * **createdOn** : timestamp of creation, in milliseconds * **modifiedBy** : name of last modifier * **language_info** : information on the language of the notebook :rtype: dict """ return self.content["metadata"] def get_cells(self): """ Get the cells of this Jupyter notebook. :return: a list of cells, as defined by Jupyter. Each cell is a dict, with notable fields: * **cell_type** : type of cell, for example 'code' * **metadata** : notebook metadata in the cell * **executionCount** : index of the last run of this cell * **source** : content of the cell, as a list of string * **output** : list of outputs of the last run of the cell, as a list of dict with fields **output_type**, **name** and **text**. Exact contents and meaning depend on the cell type. :rtype: list[dict] """ return self.content["cells"] def save(self): """ Save the content of this Jupyter notebook. """ return self.client._perform_json("PUT", "/projects/%s/jupyter-notebooks/%s" % (self.project_key, self.notebook_name), body=self.content) class DSSNotebookSession(object): """ Metadata associated to the session of a Jupyter Notebook. .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.jupyternotebook.DSSJupyterNotebook.get_sessions()` """ def __init__(self, client, session): self.client = client self.project_key = session.get("projectKey") self.notebook_name = session.get("notebookName") self.session_creator = session.get("sessionCreator") self.session_creator_display_name = session.get("sessionCreatorDisplayName") self.session_unix_owner = session.get("sessionUnixOwner") self.session_id = session.get("sessionId") self.kernel_id = session.get("kernelId") self.kernel_pid = session.get("kernelPid") self.kernel_connections = session.get("kernelConnections") self.kernel_last_activity_time = session.get("kernelLastActivityTime") self.kernel_execution_state = session.get("kernelExecutionState") self.session_start_time = session.get("sessionStartTime") def unload(self): """ Stop this Jupyter notebook and release its resources. """ return self.client._perform_json("DELETE", "/projects/%s/jupyter-notebooks/%s/sessions/%s" % (self.project_key, self.notebook_name, self.session_id)) dataiku-api-client-python-11.4.0/dataikuapi/dss/macro.py000066400000000000000000000151431440464736400231610ustar00rootroot00000000000000import time import sys, json from ..utils import DataikuException class DSSMacro(object): """ A macro on the DSS instance. .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.get_macro()` """ def __init__(self, client, project_key, runnable_type, definition=None): self.client = client self.runnable_type = runnable_type self.project_key = project_key self.definition = definition def get_definition(self): """ Get the macro definition. .. note:: The **adminParams** field is empty unless the authentication of the API client covers admin rights. :return: the definition (read-only), as a dict with fields: * **runnableType** : identifier of the macro * **ownerPluginId** : identifier of the plugin where the macro is defined * **meta** : display information about the macro, as a dict of: * **category** : (optional) name of a category, used to group macros in the list * **label** : display name * **description** : short description of the macro * **icon** : icon used for the macro in the UI * **longDescription** : (optional) longer description. Can use Markdown * **resultType** : type of result produced by the macro. Possible values: NONE, RESULT_TABLE, FILE, FOLDER_FILE, HTML, JSON_OBJECT, URL * **resultLabel** : name for the result * **mimeType** : (optional) MIME type of the result * **extension** : (optional) extension of the result, used when **resultType** is FILE or FOLDER_FILE * **params** : the list of parameters to pass in order to start a run. Each parameter is a dict with at least **name** and **type** fields. See `the doc `_ for available parameter definitions. * **adminParams** : parameters only visible (and usable) by admins :rtype: dict """ if self.definition is None: self.definition = self.client._perform_json( "GET", "/projects/%s/runnables/%s" % (self.project_key, self.runnable_type)) return self.definition def run(self, params=None, admin_params=None, wait=True): """ Run the macro from the project .. note:: If the authentication of the api client does not have admin rights, admin params are ignored. Usage example: .. code-block:: python # list all datasets on a connection. connection_name = 'filesystem_managed' macro = project.get_macro('pyrunnable_builtin-macros_list-datasets-using-connection') run_id = macro.run(params={'connection': connection_name}, wait=True) # the result of this builtin macro is of type RESULT_TABLE result = macro.get_result(run_id, as_type="json") for record in result["records"]: print("Used by %s" % record[0]) :param dict params: parameters to the macro run (defaults to `{}`) :param dict admin_params: admin parameters to the macro run (defaults to `{}`) :param boolean wait: if True, the call blocks until the run is finished :return: a run identifier to use with :meth:`abort()`, :meth:`get_status()` and :meth:`get_result()` :rtype: string """ if params is None: params = {} if admin_params is None: admin_params = {} return self.client._perform_json( "POST", "/projects/%s/runnables/%s" % (self.project_key, self.runnable_type), params={'wait':wait}, body={'params':params, 'adminParams':admin_params})['runId'] def abort(self, run_id): """ Abort a run of the macro. :param string run_id: a run identifier, as returned by :meth:`run()` """ self.client._perform_empty( "POST", "/projects/%s/runnables/%s/abort/%s" % (self.project_key, self.runnable_type, run_id)) def get_status(self, run_id): """ Poll the status of a run of the macro. .. note:: Once the run is done, when :meth:`get_result()` is called, the run ceases to exist. Afterwards :meth:`get_status()` will answer that the run doesn't exist. :param string run_id: a run identifier, as returned by :meth:`run()` :return: the status, as a dict of: * **exists** : whether there is a run with this identifier * **running** : whether the run is still ongoing * **progress** : when **running** is True, a dict of the progress of the run * **empty** : for runs that don't run anymore, whether there is no result * **type** : type of result, when there's one. Possible values: NONE, RESULT_TABLE, FILE, FOLDER_FILE, HTML, JSON_OBJECT, URL * **storedError** : dict of the error, when some error failed the run :rtype: dict """ return self.client._perform_json( "GET", "/projects/%s/runnables/%s/state/%s" % (self.project_key, self.runnable_type, run_id)) def get_result(self, run_id, as_type=None): """ Retrieve the result of a run of the macro. .. note:: If the macro is still running, an Exception is raised. The type of the contents of the result to expect can be checked using :meth:`get_definition()`, in particular the "resultType" field. :param string run_id: a run identifier, as returned by :meth:`run()` method :param string as_type: if not None, one of 'string' or 'json'. Use 'json' when the type of result advertised by the macro is RESULT_TABLE or JSON_OBJECT. :return: the contents of the result of the macro run, as a file-like is **as_type** is None; as a str if **as_type** is "string"; as an object if **as_type** is "json". :rtype: file-like, or string """ resp = self.client._perform_raw( "GET", "/projects/%s/runnables/%s/result/%s" % (self.project_key, self.runnable_type, run_id)) if as_type == 'string': with resp.raw as s: return s.read() elif as_type == 'json': with resp.raw as s: return json.load(s) else: return resp.raw dataiku-api-client-python-11.4.0/dataikuapi/dss/managedfolder.py000066400000000000000000000551311440464736400246510ustar00rootroot00000000000000from ..utils import DataikuException from ..utils import DataikuUTF8CSVReader from ..utils import DataikuStreamedHttpUTF8CSVReader from .utils import DSSTaggableObjectListItem, DSSTaggableObjectSettings import json import os from requests import utils from .metrics import ComputedMetrics from .future import DSSFuture from .discussion import DSSObjectDiscussions try: basestring except NameError: basestring = str class DSSManagedFolder(object): """ A handle to interact with a managed folder on the DSS instance. .. important:: Do not create this class directly, instead use :meth:`dataikuapi.dss.project.get_managed_folder` """ def __init__(self, client, project_key, odb_id): self.client = client self.project = client.get_project(project_key) self.project_key = project_key self.odb_id = odb_id @property def id(self): """ Returns the internal identifier of the managed folder, which is a 8-character random string, not to be confused with the managed folder's name. :rtype: string """ return self.odb_id ######################################################## # Managed folder deletion ######################################################## def delete(self): """ Delete the managed folder from the flow, and objects using it (recipes or labeling tasks) .. attention:: This call doesn't delete the managed folder's contents """ return self.client._perform_empty( "DELETE", "/projects/%s/managedfolders/%s" % (self.project_key, self.odb_id)) ######################################################## # Managed folder definition ######################################################## def get_definition(self): """ Get the definition of this managed folder. The definition contains name, description checklists, tags, connection and path parameters, metrics and checks setup. .. caution:: Deprecated. Please use :meth:`get_settings` :returns: the managed folder definition. :rtype: dict """ return self.client._perform_json( "GET", "/projects/%s/managedfolders/%s" % (self.project_key, self.odb_id)) def set_definition(self, definition): """ Set the definition of this managed folder. .. caution:: Deprecated. Please use :meth:`get_settings` then :meth:`~DSSManagedFolderSettings.save()` .. note:: the fields `id` and `projectKey` can't be modified Usage example: .. code-block:: python folder_definition = folder.get_definition() folder_definition['tags'] = ['tag1','tag2'] folder.set_definition(folder_definition) :param dict definition: the new state of the definition for the folder. You should only set a definition object that has been retrieved using the :meth:`get_definition` call :returns: a message upon successful completion of the definition update. Only contains one `msg` field :rtype: dict """ return self.client._perform_json( "PUT", "/projects/%s/managedfolders/%s" % (self.project_key, self.odb_id), body=definition) def get_settings(self): """ Returns the settings of this managed folder as a :class:`DSSManagedFolderSettings`. You must use :meth:`~DSSManagedFolderSettings.save()` on the returned object to make your changes effective on the managed folder. .. code-block:: python # Example: activating discrete partitioning folder = project.get_managed_folder("my_folder_id") settings = folder.get_settings() settings.add_discrete_partitioning_dimension("country") settings.save() :returns: the settings of the managed folder :rtype: :class:`DSSManagedFolderSettings` """ data = self.client._perform_json("GET", "/projects/%s/managedfolders/%s" % (self.project_key, self.odb_id)) return DSSManagedFolderSettings(self, data) ######################################################## # Managed folder contents ######################################################## def list_contents(self): """ Get the list of files in the managed folder Usage example: .. code-block:: python for content in folder.list_contents()['items']: last_modified_seconds = content["lastModified"] / 1000 last_modified_str = datetime.fromtimestamp(last_modified_seconds).strftime("%Y-%m-%d %H:%m:%S") print("size=%s mtime=%s %s" % (content["size"], last_modified_str, content["path"])) :returns: the list of files, in the `items` field. Each item has fields: * **path** : path of the file inside the folder * **size** : size of the file in bytes * **lastModified** : last modification time, in milliseconds since epoch :rtype: dict """ return self.client._perform_json( "GET", "/projects/%s/managedfolders/%s/contents" % (self.project_key, self.odb_id)) def get_file(self, path): """ Get a file from the managed folder Usage example: .. code-block:: python with folder.get_file("/kaggle_titanic_train.csv") as fd: df = pandas.read_csv(fd.raw) :param string path: the path of the file to read within the folder :returns: the HTTP request to stream the data from :rtype: :class:`requests.models.Response` """ return self.client._perform_raw( "GET", "/projects/%s/managedfolders/%s/contents/%s" % (self.project_key, self.odb_id, utils.quote(path))) def delete_file(self, path): """ Delete a file from the managed folder :param string path: the path of the file to read within the folder .. note:: No error is raised if the file doesn't exist """ return self.client._perform_empty( "DELETE", "/projects/%s/managedfolders/%s/contents/%s" % (self.project_key, self.odb_id, utils.quote(path))) def put_file(self, path, f): """ Upload the file to the managed folder. If the file already exists in the folder, it is overwritten. Usage example: .. code-block:: python with open("./some_local.csv") as fd: uploaded = folder.put_file("/target.csv", fd).json() print("Uploaded %s bytes" % uploaded["size"]) :param string path: the path of the file to write within the folder :param file f: a file-like .. note:: if using a string for the `f` parameter, the string itself is taken as the file content to upload :returns: information on the file uploaded to the folder, as a dict of: * **path** : path of the file inside the folder * **size** : size of the file in bytes * **lastModified** : last modification time, in milliseconds since epoch :rtype: dict """ return self.client._perform_json_upload( "POST", "/projects/%s/managedfolders/%s/contents/%s" % (self.project_key, self.odb_id, utils.quote(path)), "", f).json() def upload_folder(self, path, folder): """ Upload the content of a folder to a managed folder. .. note:: `upload_folder("/some/target", "./a/source/")` will result in "target" containing the contents of "source", but not the "source" folder being a child of "target" :param str path: the destination path of the folder in the managed folder :param str folder: local path (absolute or relative) of the source folder to upload """ for root, _, files in os.walk(folder): for file in files: filename = os.path.join(root, file) with open(filename, "rb") as f: rel_posix_path = "/".join(os.path.relpath(filename, folder).split(os.sep)) self.put_file("{}/{}".format(path, rel_posix_path), f) ######################################################## # Managed folder actions ######################################################## def compute_metrics(self, metric_ids=None, probes=None): """ Compute metrics on this managed folder. Usage example: .. code-block:: python future = folder.compute_metrics() metrics = future.wait_for_result() print("Computed in %s ms" % (metrics["endTime"] - metrics["startTime"])) for computed in metrics["computed"]: print("Metric %s = %s" % (computed["metricId"], computed["value"])) :param metric_ids: (optional) identifiers of metrics to compute, among the metrics defined on the folder :type metric_ids: list[string] :param probes: (optional) definition of metrics probes to use, in place of the ones defined on the folder. The current set of probes on the folder is the `probes` field in the dict returned by :meth:`get_definition` :type probes: dict :returns: a DSSFuture representing the task of computing the probes :rtype: :class:`dataikuapi.dss.future.DSSFuture` """ url = "/projects/%s/managedfolders/%s/actions" % (self.project_key, self.odb_id) if metric_ids is not None: future_resp = self.client._perform_json( "POST" , "%s/computeMetricsFromIds" % url, body={"metricIds" : metric_ids}) elif probes is not None: future_resp = self.client._perform_json( "POST" , "%s/computeMetrics" % url, body=probes) else: future_resp = self.client._perform_json( "POST" , "%s/computeMetrics" % url) return DSSFuture(self.client, future_resp.get("jobId", None), future_resp) ######################################################## # Metrics ######################################################## def get_last_metric_values(self): """ Get the last values of the metrics on this managed folder. :returns: a handle on the values of the metrics :rtype: :class:`dataikuapi.dss.metrics.ComputedMetrics` """ return ComputedMetrics(self.client._perform_json( "GET", "/projects/%s/managedfolders/%s/metrics/last" % (self.project_key, self.odb_id))) def get_metric_history(self, metric): """ Get the history of the values of a metric on this managed folder. Usage example: .. code-block:: python history = folder.get_metric_history("basic:COUNT_FILES") for value in history["values"]: time_str = datetime.fromtimestamp(value["time"] / 1000).strftime("%Y-%m-%d %H:%m:%S") print("%s : %s" % (time_str, value["value"])) :param string metric: identifier of the metric to get values of :returns: an object containing the values of the metric, cast to the appropriate type (double, boolean,...). Top-level fields are: * **metricId** : identifier of the metric * **metric** : dict of the metric's definition * **valueType** : type of the metric values in the **values** array * **lastValue** : most recent value, as a dict of - **time** : timestamp of the value computation - **value** : value of the metric at **time** * **values** : list of values, each one a dict of the same structure as **lastValue** :rtype: dict """ return self.client._perform_json( "GET", "/projects/%s/managedfolders/%s/metrics/history" % (self.project_key, self.odb_id), params={'metricLookup' : metric if isinstance(metric, str) or isinstance(metric, unicode) else json.dumps(metric)}) ######################################################## # Misc ######################################################## def get_zone(self): """ Get the flow zone of this managed folder. :returns: a flow zone :rtype: :class:`dataikuapi.dss.flow.DSSFlowZone` """ return self.project.get_flow().get_zone_of_object(self) def move_to_zone(self, zone): """ Move this object to a flow zone. :param object zone: a :class:`dataikuapi.dss.flow.DSSFlowZone` where to move the object, or its identifier """ if isinstance(zone, basestring): zone = self.project.get_flow().get_zone(zone) zone.add_item(self) def share_to_zone(self, zone): """ Share this object to a flow zone. :param object zone: a :class:`dataikuapi.dss.flow.DSSFlowZone` where to share the object, or its identifier """ if isinstance(zone, basestring): zone = self.project.get_flow().get_zone(zone) zone.add_shared(self) def unshare_from_zone(self, zone): """ Unshare this object from a flow zone. :param object zone: a :class:`dataikuapi.dss.flow.DSSFlowZone` from where to unshare the object, or its identifier """ if isinstance(zone, basestring): zone = self.project.get_flow().get_zone(zone) zone.remove_shared(self) def get_usages(self): """ Get the recipes referencing this folder. Usage example: .. code-block:: python for usage in folder.get_usages(): if usage["type"] == 'RECIPE_INPUT': print("Used as input of %s" % usage["objectId"]) :returns: a list of usages, each one a dict of: * **type** : the type of usage, either "RECIPE_INPUT" or "RECIPE_OUTPUT" * **objectId** : name of the recipe * **objectProjectKey** : project of the recipe :rtype: list[dict] """ return self.client._perform_json("GET", "/projects/%s/managedfolders/%s/usages" % (self.project_key, self.odb_id)) def get_object_discussions(self): """ Get a handle to manage discussions on the managed folder. :returns: the handle to manage discussions :rtype: :class:`dataikuapi.dss.discussion.DSSObjectDiscussions` """ return DSSObjectDiscussions(self.client, self.project_key, "MANAGED_FOLDER", self.odb_id) ######################################################## # utilities ######################################################## def copy_to(self, target, write_mode="OVERWRITE"): """ Copy the data of this folder to another folder. :param object target: a :class:`dataikuapi.dss.managedfolder.DSSManagedFolder` representing the target location of this copy :returns: a DSSFuture representing the operation :rtype: :class:`dataikuapi.dss.future.DSSFuture` """ dqr = { "targetProjectKey" : target.project_key, "targetFolderId": target.odb_id, "writeMode" : write_mode } future_resp = self.client._perform_json("POST", "/projects/%s/managedfolders/%s/actions/copyTo" % (self.project_key, self.odb_id), body=dqr) return DSSFuture(self.client, future_resp.get("jobId", None), future_resp) class DSSManagedFolderSettings(DSSTaggableObjectSettings): """ Base settings class for a DSS managed folder. Do not instantiate this class directly, use :meth:`DSSDSSManagedFolderSettings.get_settings` Use :meth:`save` to save your changes """ def __init__(self, folder, settings): super(DSSManagedFolderSettings, self).__init__(settings) self.folder = folder self.settings = settings def get_raw(self): """ Get the managef folder settings as a dict :returns: the settings. Top-level fields are: * **name** : the label of the managed folder * **type** : the type of the filesystem underlying the managed folder (S3, HDFS, GCS, ...) * **params** : the type-specific parameters, like the connection to use or the root path of the managed folder inside the connection * **contentType** : an optional semantic type describing the files in the folder * **partitioning** : definition of the partitioning of the managed folder * **selection** : when partitioned, the partitions to show when viewing the managed folder * **flowOptions** : build options * **metrics** : probes for computing metrics on the managed folder * **checks** : definitions of the checks on the managed folder :rtype: dict """ return self.settings def get_raw_params(self): """ Get the type-specific (S3/ filesystem/ HDFS/ ...) params as a dict. :returns: the type-specific patams. Each type defines a set of fields; commonly found fields are : * **connection** : name of the connection used by the managed folder * **path** : root of the managed folder within the connection * **bucket** or **container** : the bucket/container name on cloud storages :rtype: dict """ return self.settings["params"] @property def type(self): """ Get the type of filesystem that the managed folder uses. :rtype: string """ return self.settings["type"] def save(self): """ Save the changes to the settings on the managed folder. Usage example: .. code-block:: python folder = project.get_managed_folder("my_folder_id") settings = folder.get_settings() settings.set_connection_and_path("some_S3_connection", None) settings.get_raw_params()["bucket"] = "some_S3_bucket" settings.save() """ self.folder.client._perform_empty( "PUT", "/projects/%s/managedfolders/%s" % (self.folder.project_key, self.folder.odb_id), body=self.settings) ######################################################## # Partitioning ######################################################## def remove_partitioning(self): """ Make the managed folder non-partitioned. """ self.settings["partitioning"] = {"dimensions" : []} def add_discrete_partitioning_dimension(self, dim_name): """ Add a discrete partitioning dimension. :param string dim_name: name of the partitioning dimension """ self.settings["partitioning"]["dimensions"].append({"name": dim_name, "type": "value"}) def add_time_partitioning_dimension(self, dim_name, period="DAY"): """ Add a time partitioning dimension. :param string dim_name: name of the partitioning dimension :param string period: granularity of the partitioning dimension (YEAR, MONTH, DAY (default), HOUR) """ self.settings["partitioning"]["dimensions"].append({"name": dim_name, "type": "time", "params":{"period": period}}) def set_partitioning_file_pattern(self, pattern): """ Set the partitioning pattern of the folder. The pattern indicates which paths inside the folder belong to which partition. Partition dimensions are written with: * `%{dim_name}` for discrete dimensions * `%Y` (=year), `%M` (=month), `%D` (=day) and `%H` (=hour) for time dimensions Besides the `%...` variables for injecting the partition dimensions, the pattern is a regular expression. Usage example: .. code-block:: python # partition a managed folder by month folder = project.get_managed_folder("my_folder_id") settings = folder.get_settings() settings.add_time_partitioning_dimension("my_date", "MONTH") settings.set_partitioning_file_pattern("/year=%Y/month=%M/.*") settings.save() :param string pattern: the partitioning pattern """ self.settings["partitioning"]["filePathPattern"] = pattern ######################################################## # Basic ######################################################## def set_connection_and_path(self, connection, path): """ Change the managed folder connection and/or path. .. note:: When changing the connection or path, the folder's files aren't moved or copied to the new location .. attention:: When changing the connection for a connection with a different type, for example going from a S3 connection to an Azure Blob Storage connection, only the managed folder type is changed. Type-specific fields are not converted. In the example of a S3 to Azure conversion, the S3 bucket isn't converted to a storage account container. :param string connection: the name of a file-based connection. If `None`, the connection of the managed folder is left unchanged :param string path: a path relative to the connection root. If `None`, the path of the managed folder is left unchanged """ if connection is not None: if connection != self.settings["params"]["connection"]: # get the actual connection type (and check that it exists) connection_info = self.folder.client.get_connection(connection).get_info(self.folder.project_key) connection_type = connection_info["type"] if connection_type == 'EC2': self.settings["type"] = 'S3' # the fsprovider type is different elif connection_type == 'SSH': # can be SCP or SFTP, default to SCP if connection type changed if self.settings["type"] not in ['SCP', 'SFTP']: self.settings["type"] = 'SCP' else: self.settings["type"] = connection_type self.settings["params"]["connection"] = connection if path is not None: self.settings["params"]["path"] = path dataiku-api-client-python-11.4.0/dataikuapi/dss/meaning.py000066400000000000000000000022471440464736400234770ustar00rootroot00000000000000 class DSSMeaning(object): """ A user-defined meaning on the DSS instance """ def __init__(self, client, id): self.client = client self.id = id ######################################################## # Meaning definition ######################################################## def get_definition(self): """ Get the meaning's definition. .. attention:: This call requires an API key with admin rights :returns: the meaning definition. The precise structure of the dict depends on the meaning type and is not documented. :rtype: dict """ return self.client._perform_json("GET", "/meanings/%s" % self.id) def set_definition(self, definition): """ Set the meaning's definition. .. attention:: This call requires an API key with admin rights :param dict definition: the definition for the meaning, as a dict. You should only ``set_definition`` on a modified version of a dict you retrieved via :py:meth:`get_definition` """ return self.client._perform_json("PUT", "/meanings/%s" % self.id, body=definition) dataiku-api-client-python-11.4.0/dataikuapi/dss/metrics.py000066400000000000000000000171261440464736400235310ustar00rootroot00000000000000 class ComputedMetrics(object): """ Handle to the metrics of a DSS object and their last computed value .. important:: Do not create this class directly, instead use :meth:`.DSSDataset.get_last_metric_values`, :meth:`.DSSSavedModel.get_metric_values`, :meth:`.DSSManagedFolder.get_last_metric_values`. """ def __init__(self, raw): self.raw = raw def get_raw(self): """ Get the raw metrics data :rtype: dict """ return self.raw def get_metric_by_id(self, id): """ Retrieve the info for a given metric Usage example .. code-block:: python dataset = project.get_ataset("my_dataset") metrics = dataset.get_last_metric_values() count_files_metric = metrics.get_metric_by_id("basic:COUNT_FILES") for value in count_files_metric['lastValues']: print("partition=%s -> count of files=%s" % (value['partition'], value['value'])) :param string metric_id: identifier of the metric :return: information about the metric and its values. Top-level fields are * **metric** : definition of the metric * **meta** : display metadata, as a dict of **name** and **fullName** * **computingProbe** : name of the probe that computes the metric * **displayedAsMetric** : whether the metric is among the metrics displayed on the "Status" tab of the object * **notExistingViews** : list of the possible types of metrics datasets not yet created on the object * **partitionsWithValue** : list of the partition identifiers for which some value of the metric exists * **lastValues** : list of the last computed value, per partition. Each list element has * **partition** : the partition identifier, as a string. * **value** : the metric value, as a string * **dataType** : expected type of **value** (one of BIGINT, DOUBLE, STRING, BOOLEAN) * **computed** : timestamp of computation, in milliseconds since epoch :rtype: dict """ all_ids = [] for metric in self.raw["metrics"]: all_ids.append(metric["metric"]["id"]) if metric["metric"]["id"] == id: return metric raise Exception("Metric %s not found among: %s" % (id, all_ids)) def get_global_data(self, metric_id): """ Get the global value point of a given metric, or throws. For a partitioned dataset, the global value is the value of the metric computed on the whole dataset (coded as partition 'ALL'). :param string metric_id: identifier of the metric :returns: the metric data, as a dict. Fields are * **partition** : the partition identifier, as a string. * **value** : the metric value, as a string * **dataType** : expected type of **value** (one of BIGINT, DOUBLE, STRING, BOOLEAN) * **computed** : timestamp of computation, in milliseconds since epoch :rtype: dict """ for partition_data in self.get_metric_by_id(metric_id)["lastValues"]: if partition_data["partition"]== "NP" or partition_data["partition"] == "ALL": return partition_data raise Exception("No data found for global partition for metric %s" % metric_id) def get_global_value(self, metric_id): """ Get the global value of a given metric, or throws. For a partitioned dataset, the global value is the value of the metric computed on the whole dataset (coded as partition 'ALL'). Usage example .. code-block:: python dataset = project.get_ataset("my_dataset") metrics = dataset.get_last_metric_values() print("record count = %s" % metrics.get_global_value('records:COUNT_RECORDS')) :param string metric_id: identifier of the metric :returns: the value of the metric for the partition :rtype: str, int or float """ return ComputedMetrics.get_value_from_data(self.get_global_data(metric_id)) def get_partition_data(self, metric_id, partition): """ Get the value point of a given metric for a given partition, or throws. :param string metric_id: identifier of the metric :param string partition: partition identifier :returns: the metric data, as a dict. Fields are * **partition** : the partition identifier, as a string. * **value** : the metric value, as a string * **dataType** : expected type of **value** (one of BIGINT, DOUBLE, STRING, BOOLEAN) * **computed** : timestamp of computation, in milliseconds since epoch :rtype: dict """ for partition_data in self.get_metric_by_id(metric_id)["lastValues"]: if partition_data["partition"] == partition: return partition_data def get_partition_value(self, metric_id, partition): """ Get the value of a given metric for a given partition, or throws. :param string metric_id: identifier of the metric :param string partition: partition identifier :returns: the value of the metric for the partition :rtype: str, int or float """ return ComputedMetrics.get_value_from_data(self.get_partition_data(metric_id, partition)) def get_first_partition_data(self, metric_id): """ Get a value point of a given metric, or throws. The first value encountered is returned. :param string metric_id: identifier of the metric :returns: the metric data, as a dict. Fields are * **partition** : the partition identifier, as a string. * **value** : the metric value, as a string * **dataType** : expected type of **value** (one of BIGINT, DOUBLE, STRING, BOOLEAN) * **computed** : timestamp of computation, in milliseconds since epoch :rtype: dict """ for partition_data in self.get_metric_by_id(metric_id)["lastValues"]: return partition_data raise Exception("No partition data for metric %s" % metric_id) def get_all_ids(self): """ Get the identifiers of all metrics defined in this object :returns: list of metric identifiers :rtype: list[string] """ all_ids = [] for metric in self.raw["metrics"]: all_ids.append(metric["metric"]["id"]) return all_ids @staticmethod def get_value_from_data(data): """ Retrieves the value from a metric point, cast in the appropriate type (str, int or float). For other types, the value is not cast and left as a string. :param dict data: a value point for a metric, retrieved with :meth:`~get_global_data`, :meth:`~get_partition_data` or :meth:`~get_first_partition_data` :returns: the value, cast to the appropriate Python type :rtype: str, int or float """ dtype = data.get("dataType", "STRING") if dtype in ["BIGINT", "INT"]: return int(data["value"]) elif dtype in ["FLOAT", "DOUBLE"]: return float(data["value"]) else: return data["value"] dataiku-api-client-python-11.4.0/dataikuapi/dss/ml.py000066400000000000000000005732051440464736400225000ustar00rootroot00000000000000import json import logging import re import time import warnings from six import string_types from .future import DSSFuture from .utils import DSSDatasetSelectionBuilder from .utils import DSSFilterBuilder from ..utils import DataikuException from ..utils import _write_response_content_to_file logger = logging.getLogger("ML") logger.setLevel(logging.INFO) class PredictionSplitParamsHandler(object): """Object to modify the train/test splitting params.""" SPLIT_PARAMS_KEY = 'splitParams' def __init__(self, mltask_settings): """Do not call directly, use :meth:`DSSMLTaskSettings.get_split_params`""" self.mltask_settings = mltask_settings def get_raw(self): """Gets the raw settings of the prediction split configuration. This returns a reference to the raw settings, not a copy, so changes made to the returned object will be reflected when saving. :rtype: dict """ return self.mltask_settings[PredictionSplitParamsHandler.SPLIT_PARAMS_KEY] def set_split_random(self, train_ratio = 0.8, selection = None, dataset_name=None): """ Sets the train/test split to random splitting of an extract of a single dataset :param float train_ratio: Ratio of rows to use for train set. Must be between 0 and 1 :param object selection: A :class:`~dataikuapi.dss.utils.DSSDatasetSelectionBuilder` to build the settings of the extract of the dataset. May be None (won't be changed) :param str dataset_name: Name of dataset to split. If None, the main dataset used to create the visual analysis will be used. """ sp = self.mltask_settings[PredictionSplitParamsHandler.SPLIT_PARAMS_KEY] sp["ttPolicy"] = "SPLIT_SINGLE_DATASET" if selection is not None: if isinstance(selection, DSSDatasetSelectionBuilder): sp["ssdSelection"] = selection.build() else: sp["ssdSelection"] = selection sp["ssdTrainingRatio"] = train_ratio sp["kfold"] = False if dataset_name is not None: sp["ssdDatasetSmartName"] = dataset_name return self def set_split_kfold(self, n_folds = 5, selection = None, dataset_name=None): """ Sets the train/test split to k-fold splitting of an extract of a single dataset :param int n_folds: number of folds. Must be greater than 0 :param object selection: A :class:`~dataikuapi.dss.utils.DSSDatasetSelectionBuilder` to build the settings of the extract of the dataset. May be None (won't be changed) :param str dataset_name: Name of dataset to split. If None, the main dataset used to create the visual analysis will be used. """ sp = self.mltask_settings[PredictionSplitParamsHandler.SPLIT_PARAMS_KEY] sp["ttPolicy"] = "SPLIT_SINGLE_DATASET" if selection is not None: if isinstance(selection, DSSDatasetSelectionBuilder): sp["ssdSelection"] = selection.build() else: sp["ssdSelection"] = selection sp["kfold"] = True sp["nFolds"] = n_folds if dataset_name is not None: sp["ssdDatasetSmartName"] = dataset_name return self def set_split_explicit(self, train_selection, test_selection, dataset_name=None, test_dataset_name=None, train_filter=None, test_filter=None): """ Sets the train/test split to explicit extract of one or two dataset(s) :param object train_selection: A :class:`~dataikuapi.dss.utils.DSSDatasetSelectionBuilder` to build the settings of the extract of the train dataset. May be None (won't be changed) :param object test_selection: A :class:`~dataikuapi.dss.utils.DSSDatasetSelectionBuilder` to build the settings of the extract of the test dataset. May be None (won't be changed) :param str dataset_name: Name of dataset to use for the extracts. If None, the main dataset used to create the ML Task will be used. :param str test_dataset_name: Name of a second dataset to use for the test data extract. If None, both extracts are done from dataset_name :param object train_filter: A :class:`~dataikuapi.dss.utils.DSSFilterBuilder` to build the settings of the filter of the train dataset. May be None (won't be changed) :param object test_filter: A :class:`~dataikuapi.dss.utils.DSSFilterBuilder` to build the settings of the filter of the test dataset. May be None (won't be changed) """ sp = self.mltask_settings[PredictionSplitParamsHandler.SPLIT_PARAMS_KEY] if dataset_name is None: raise Exception("For explicit splitting a dataset_name is mandatory") if test_dataset_name is None or test_dataset_name == dataset_name: sp["ttPolicy"] = "EXPLICIT_FILTERING_SINGLE_DATASET" train_split ={} test_split = {} sp['efsdDatasetSmartName'] = dataset_name sp['efsdTrain'] = train_split sp['efsdTest'] = test_split else: sp["ttPolicy"] = "EXPLICIT_FILTERING_TWO_DATASETS" train_split ={'datasetSmartName' : dataset_name} test_split = {'datasetSmartName' : test_dataset_name} sp['eftdTrain'] = train_split sp['eftdTest'] = test_split if train_selection is not None: if isinstance(train_selection, DSSDatasetSelectionBuilder): train_split["selection"] = train_selection.build() else: train_split["selection"] = train_selection if test_selection is not None: if isinstance(test_selection, DSSDatasetSelectionBuilder): test_split["selection"] = test_selection.build() else: test_split["selection"] = test_selection if train_filter is not None: if isinstance(train_filter, DSSFilterBuilder): train_split["filter"] = train_filter.build() else: train_split["filter"] = train_filter if test_filter is not None: if isinstance(test_filter, DSSFilterBuilder): test_split["filter"] = test_filter.build() else: test_split["filter"] = test_filter return self def set_time_ordering(self, feature_name, ascending=True): """ Uses a variable to sort the data for train/test split and hyperparameter optimization by time :param str feature_name: Name of the variable to use :param bool ascending: True iff the test set is expected to have larger time values than the train set """ self.unset_time_ordering() if not feature_name in self.mltask_settings["preprocessing"]["per_feature"]: raise ValueError("Feature %s doesn't exist in this ML task, can't use as time" % feature_name) self.mltask_settings['time']['enabled'] = True self.mltask_settings['time']['timeVariable'] = feature_name self.mltask_settings['time']['ascending'] = ascending self.mltask_settings['preprocessing']['per_feature'][feature_name]['missing_handling'] = "DROP_ROW" if self.mltask_settings['splitParams']['ttPolicy'] == "SPLIT_SINGLE_DATASET": self.mltask_settings['splitParams']['ssdSplitMode'] = "SORTED" self.mltask_settings['splitParams']['ssdColumn'] = feature_name if self.mltask_settings['modeling']['gridSearchParams']['mode'] == "KFOLD": self.mltask_settings['modeling']['gridSearchParams']['mode'] = "TIME_SERIES_KFOLD" elif self.mltask_settings['modeling']['gridSearchParams']['mode'] == "SHUFFLE": self.mltask_settings['modeling']['gridSearchParams']['mode'] = "TIME_SERIES_SINGLE_SPLIT" return self def unset_time_ordering(self): """ Remove time-based ordering for train/test split and hyperparameter optimization """ self.mltask_settings['time']['enabled'] = False self.mltask_settings['time']['timeVariable'] = None if self.mltask_settings['splitParams']['ttPolicy'] == "SPLIT_SINGLE_DATASET": self.mltask_settings['splitParams']['ssdSplitMode'] = "RANDOM" self.mltask_settings['splitParams']['ssdColumn'] = None if self.mltask_settings['modeling']['gridSearchParams']['mode'] == "TIME_SERIES_KFOLD": self.mltask_settings['modeling']['gridSearchParams']['mode'] = "KFOLD" elif self.mltask_settings['modeling']['gridSearchParams']['mode'] == "TIME_SERIES_SINGLE_SPLIT": self.mltask_settings['modeling']['gridSearchParams']['mode'] = "SHUFFLE" return self def has_time_ordering(self): """ :return: whether the splitting uses time ordering :rtype: bool """ return 'time' in self.mltask_settings and self.mltask_settings['time']['enabled'] def get_time_ordering_variable(self): """ :return: the name of the variable :rtype: str """ if self.has_time_ordering(): return self.mltask_settings['time']['timeVariable'] else: warnings.warn("Time-based ordering is disabled in the current MLTask") def is_time_ordering_ascending(self): """ :return: True if the ordering is set to be ascending with respect to the time-ordering variable :rtype: bool """ if self.has_time_ordering(): return self.mltask_settings['time']['ascending'] else: warnings.warn("Time-based ordering is disabled in the current MLTask") class DSSMLTaskSettings(object): """ Object to read and modify the settings of a ML task. Do not create this object directly, use :meth:`DSSMLTask.get_settings()` instead """ def __init__(self, client, project_key, analysis_id, mltask_id, mltask_settings): self.client = client self.project_key = project_key self.analysis_id = analysis_id self.mltask_id = mltask_id self.mltask_settings = mltask_settings def get_raw(self): """ Gets the raw settings of this ML Task. This returns a reference to the raw settings, not a copy, so changes made to the returned object will be reflected when saving. :rtype: dict """ return self.mltask_settings def get_feature_preprocessing(self, feature_name): """ Gets the feature preprocessing params for a particular feature. This returns a reference to the feature's settings, not a copy, so changes made to the returned object will be reflected when saving :return: A dict of the preprocessing settings for a feature :rtype: dict """ return self.mltask_settings["preprocessing"]["per_feature"][feature_name] def foreach_feature(self, fn, only_of_type = None): """ Applies a function to all features (except target) :param function fn: Function that takes 2 parameters: feature_name and feature_params and returns modified feature_params :param str only_of_type: if not None, only applies to feature of the given type. Can be one of ``CATEGORY``, ``NUMERIC``, ``TEXT`` or ``VECTOR`` """ import copy new_per_feature = {} for (k, v) in self.mltask_settings["preprocessing"]["per_feature"].items(): if v["role"] == "TARGET" or (only_of_type is not None and v["type"] != only_of_type): new_per_feature[k] = v else: new_per_feature[k] = fn(k, copy.deepcopy(v)) self.mltask_settings["preprocessing"]["per_feature"] = new_per_feature def reject_feature(self, feature_name): """ Marks a feature as rejected and not used for training :param str feature_name: Name of the feature to reject """ self.get_feature_preprocessing(feature_name)["role"] = "REJECT" def use_feature(self, feature_name): """ Marks a feature as input for training :param str feature_name: Name of the feature to reject """ self.get_feature_preprocessing(feature_name)["role"] = "INPUT" def get_algorithm_settings(self, algorithm_name): raise NotImplementedError() def _get_custom_algorithm_settings(self, algorithm_name): # returns the first algorithm with this name for algo in self.mltask_settings["modeling"]["custom_mllib"]: if algorithm_name == algo["name"]: return algo for algo in self.mltask_settings["modeling"]["custom_python"]: if algorithm_name == algo["name"]: return algo raise ValueError("Unknown algorithm: {}".format(algorithm_name)) def get_diagnostics_settings(self): """ Gets the diagnostics settings for a mltask. This returns a reference to the diagnostics' settings, not a copy, so changes made to the returned object will be reflected when saving. This method returns a dictionary of the settings with: - 'enabled': indicates if the diagnostics are enabled globally, if False, all diagnostics will be disabled - 'settings': a list of dict comprised of: - 'type': the diagnostic type - 'enabled': indicates if the diagnostic type is enabled, if False, all diagnostics of that type will be disabled Please refer to the documentation for details on available diagnostics. :return: A dict of diagnostics settings :rtype: dict """ return self.mltask_settings["diagnosticsSettings"] def set_diagnostics_enabled(self, enabled): """ Globally enables or disables all diagnostics. :param bool enabled: if the diagnostics should be enabled or not """ settings = self.get_diagnostics_settings() settings["enabled"] = enabled def set_diagnostic_type_enabled(self, diagnostic_type, enabled): """ Enables or disables a diagnostic based on its type. Please refer to the documentation for details on available diagnostics. :param str diagnostic_type: Name (in capitals) of the diagnostic type. :param bool enabled: if the diagnostic should be enabled or not """ settings = self.get_diagnostics_settings()["settings"] diagnostic = [h for h in settings if h["type"] == diagnostic_type] if len(diagnostic) == 0: raise ValueError("Diagnostic type '{}' not found in settings".format(diagnostic_type)) if len(diagnostic) > 1: raise ValueError("Should not happen: multiple diagnostic types '{}' found in settings".format(diagnostic_type)) diagnostic[0]["enabled"] = enabled def set_algorithm_enabled(self, algorithm_name, enabled): """ Enables or disables an algorithm based on its name. Please refer to the documentation for details on available algorithms. :param str algorithm_name: Name (in capitals) of the algorithm. """ self.get_algorithm_settings(algorithm_name)["enabled"] = enabled def disable_all_algorithms(self): """Disables all algorithms""" for algorithm_key in self.__class__._algorithm_remap.keys(): algorithm_meta = self.__class__._algorithm_remap[algorithm_key] if isinstance(algorithm_meta, PredictionAlgorithmMeta): key = algorithm_meta.algorithm_name else: key = algorithm_meta if key in self.mltask_settings["modeling"]: self.mltask_settings["modeling"][key]["enabled"] = False for custom_mllib in self.mltask_settings["modeling"]["custom_mllib"]: custom_mllib["enabled"] = False for custom_python in self.mltask_settings["modeling"]["custom_python"]: custom_python["enabled"] = False for plugin in self.mltask_settings["modeling"].get("plugin_python", {}).values(): plugin["enabled"] = False def get_all_possible_algorithm_names(self): """ Returns the list of possible algorithm names, i.e. the list of valid identifiers for :meth:`set_algorithm_enabled` and :meth:`get_algorithm_settings` This includes all possible algorithms, regardless of the prediction kind (regression/classification) or engine, so some algorithms may be irrelevant :returns: the list of algorithm names as a list of strings :rtype: list of string """ return list(self.__class__._algorithm_remap.keys()) + self._get_custom_algorithm_names() def _get_custom_algorithm_names(self): """ Returns the list of names of defined custom models (Python & MLlib backends) :returns: the list of custom models names :rtype: list of string """ return ([algo["name"] for algo in self.mltask_settings["modeling"]["custom_mllib"]] + [algo["name"] for algo in self.mltask_settings["modeling"]["custom_python"]]) def get_enabled_algorithm_names(self): """ :returns: the list of enabled algorithm names as a list of strings :rtype: list of string """ return [algo_name for algo_name in self.get_all_possible_algorithm_names() if self.get_algorithm_settings(algo_name).get("enabled", False)] def get_enabled_algorithm_settings(self): """ :returns: the map of enabled algorithm names with their settings :rtype: dict """ return {key: self.get_algorithm_settings(key) for key in self.get_enabled_algorithm_names()} def set_metric(self, metric=None, custom_metric=None, custom_metric_greater_is_better=True, custom_metric_use_probas=False): """ Sets the score metric to optimize for a prediction ML Task :param str metric: metric to use. Leave empty to use a custom metric. You need to set the ``custom_metric`` value in that case :param str custom_metric: code of the custom metric :param bool custom_metric_greater_is_better: whether the custom metric is a score or a loss :param bool custom_metric_use_probas: whether to use the classes' probas or the predicted value (for classification) """ if custom_metric is None and metric is None: raise ValueError("Either metric or custom_metric must be defined") self.mltask_settings["modeling"]["metrics"]["evaluationMetric"] = metric if custom_metric is None else 'CUSTOM' self.mltask_settings["modeling"]["metrics"]["customEvaluationMetricCode"] = custom_metric self.mltask_settings["modeling"]["metrics"]["customEvaluationMetricGIB"] = custom_metric_greater_is_better self.mltask_settings["modeling"]["metrics"]["customEvaluationMetricNeedsProba"] = custom_metric_use_probas def add_custom_python_model(self, name="Custom Python Model", code=""): """ Adds a new custom python model :param str name: name of the custom model :param str code: code of the custom model """ self.mltask_settings["modeling"]["custom_python"].append({ "name": name, "code": code, "enabled": True }) def add_custom_mllib_model(self, name="Custom MLlib Model", code=""): """ Adds a new custom MLlib model :param str name: name of the custom model :param str code: code of the custom model """ self.mltask_settings["modeling"]["custom_mllib"].append({ "name": name, "initializationCode": code, "enabled": True }) def save(self): """Saves back these settings to the ML Task""" self.client._perform_empty( "POST", "/projects/%s/models/lab/%s/%s/settings" % (self.project_key, self.analysis_id, self.mltask_id), body = self.mltask_settings) class HyperparameterSearchSettings(object): def __init__(self, raw_settings): self._raw_settings = raw_settings def _key_repr(self, key): if isinstance(self._raw_settings[key], string_types): return " \"{}\"=\"{}\"\n".format(key, self._raw_settings[key]) else: return " \"{}\"={}\n".format(key, self._raw_settings[key]) def _repr_html_(self): res = "
"
        res += self.__class__.__name__ + "(\n"
        res += "Search Strategy:\n"
        res += self._key_repr("strategy")
        if self._raw_settings["strategy"] == "BAYESIAN":
            res += self._key_repr("bayesianOptimizer")

        res += "Search Validation:\n"
        res += self._key_repr("mode")
        if self._raw_settings["mode"] in {"SHUFFLE", "TIME_SERIES_SINGLE_SPLIT"}:
            res += self._key_repr("splitRatio")
        elif self._raw_settings["mode"] in {"KFOLD", "TIME_SERIES_KFOLD"}:
            res += self._key_repr("nFolds")
        res += self._key_repr("cvSeed")
        res += self._key_repr("stratified")

        res += "Execution Settings:\n"
        if self._raw_settings.get("timeout", 0) > 0:
            res += self._key_repr("timeout")
        if self._raw_settings["strategy"] == "GRID":
            res += self._key_repr("nIter")
            res += self._key_repr("randomized")
            if self._raw_settings.get("randomized", False):
                res += self._key_repr("seed")
        else:
            # RANDOM and BAYESIAN search strategies
            res += self._key_repr("nIterRandom")
            res += self._key_repr("seed")

        res += "Parallelism Settings:\n"
        res += self._key_repr("nJobs")
        res += self._key_repr("distributed")
        if self._raw_settings.get("distributed", False):
            res += self._key_repr("nContainers")
        res += ")
" res += "
{}
".format(self.__repr__()) return res def __repr__(self): return self.__class__.__name__ + "(settings={})".format(self._raw_settings) __str__ = __repr__ def _set_seed(self, seed): if seed is not None: if not isinstance(seed, int): warnings.warn("HyperparameterSearchSettings ignoring invalid input: seed") warnings.warn("seed must be an integer") else: self._raw_settings["seed"] = seed def _set_cv_seed(self, seed): if seed is not None: assert isinstance(seed, int), "HyperparameterSearchSettings invalid input: cvSeed must be an integer" self._raw_settings["cvSeed"] = seed @property def strategy(self): """ :return: strategy: "GRID" | "RANDOM" | "BAYESIAN" :rtype: str """ return self._raw_settings["strategy"] @strategy.setter def strategy(self, strategy): """ :param strategy: "GRID" | "RANDOM" | "BAYESIAN" :type strategy: str """ assert strategy in {"GRID", "RANDOM", "BAYESIAN"} self._raw_settings["strategy"] = strategy def set_grid_search(self, shuffle=True, seed=1337): """ Sets the search strategy to "GRID" to perform a grid-search on the hyperparameters. :param shuffle: if True, iterate over a shuffled grid as opposed to the lexicographical iteration over the cartesian product of the hyperparameters. :type shuffle: bool :param seed: :type seed: int """ self._raw_settings["strategy"] = "GRID" if shuffle is not None: if not isinstance(shuffle, bool): warnings.warn("HyperparameterSearchSettings.set_grid_search ignoring invalid input: shuffle") warnings.warn("shuffle must be a boolean") else: self._raw_settings["randomized"] = shuffle self._set_seed(seed) def set_random_search(self, seed=1337): """ Sets the search strategy to "RANDOM" to perform a random search on the hyperparameters. :param seed: defaults to 0 :type seed: int """ self._raw_settings["strategy"] = "RANDOM" self._set_seed(seed) def set_bayesian_search(self, seed=1337): """ Sets the search strategy to "BAYESIAN" to perform a Bayesian search on the hyperparameters. :param seed: defaults to 0 :type seed: int """ self._raw_settings["strategy"] = "BAYESIAN" self._set_seed(seed) @property def validation_mode(self): """ :return: mode: "KFOLD" | "SHUFFLE" | "TIME_SERIES_KFOLD" | "TIME_SERIES_SINGLE_SPLIT" | "CUSTOM" :rtype: str """ return self._raw_settings["mode"] @validation_mode.setter def validation_mode(self, mode): """ :param mode: "KFOLD" | "SHUFFLE" | "TIME_SERIES_KFOLD" | "TIME_SERIES_SINGLE_SPLIT" | "CUSTOM" :type mode: str """ assert mode in {"KFOLD", "SHUFFLE", "TIME_SERIES_KFOLD", "TIME_SERIES_SINGLE_SPLIT", "CUSTOM"} self._raw_settings["mode"] = mode @property def fold_offset(self): """ :return: Whether there is an offset between validation sets, to avoid overlap between cross-test sets (model evaluation) and cross-validation sets (hyperparameter search), if both are using k-fold. Only relevant for time series forecasting :rtype: bool """ return self._raw_settings["foldOffset"] @fold_offset.setter def fold_offset(self, value): """ :param value: Whether there is an offset between validation sets, to avoid overlap between cross-test sets (model evaluation) and cross-validation sets (hyperparameter search), if both are using k-fold. Only relevant for time series forecasting :type value: bool """ assert isinstance(value, bool) self._raw_settings["foldOffset"] = value @property def cv_seed(self): """ :return: cross-validation seed for splitting the data during hyperparameter search :rtype: int """ return self._raw_settings["cvSeed"] @cv_seed.setter def cv_seed(self, seed): """ :param seed: cross-validation seed for splitting the data during hyperparameter search :type seed: int """ self._set_cv_seed(seed) def set_kfold_validation(self, n_folds=5, stratified=True, cv_seed=1337): """ Sets the validation mode to k-fold cross-validation (either "KFOLD" or "TIME_SERIES_KFOLD" if time-based ordering is enabled). :param n_folds: the number of folds used for the hyperparameter search, defaults to 5 :type n_folds: int :param stratified: if True, keep the same proportion of each target classes in all folds, defaults to True :type stratified: bool """ if self._raw_settings["mode"] == "TIME_SERIES_SINGLE_SPLIT": self._raw_settings["mode"] = "TIME_SERIES_KFOLD" else: self._raw_settings["mode"] = "KFOLD" if n_folds is not None: if not (isinstance(n_folds, int) and n_folds > 0): warnings.warn("HyperparameterSearchSettings.set_kfold_validation ignoring invalid input: n_folds") warnings.warn("n_folds must be a positive integer") else: self._raw_settings["nFolds"] = n_folds if stratified is not None: if not isinstance(stratified, bool): warnings.warn("HyperparameterSearchSettings.set_validation_mode_to_kfold ignoring invalid input: stratified") warnings.warn("stratified must be a boolean") else: self._raw_settings["stratified"] = stratified self._set_cv_seed(cv_seed) def set_single_split_validation(self, split_ratio=0.8, stratified=True, cv_seed=1337): """ Sets the validation mode to single split (either "SHUFFLE" or "TIME_SERIES_SINGLE_SPLIT" if time-based ordering is enabled). :param split_ratio: ratio of the data used for the train during hyperparameter search, defaults to 0.8 :type split_ratio: float :param stratified: if True, keep the same proportion of each target classes in both splits, defaults to True :type stratified: bool """ if self._raw_settings["mode"] == "TIME_SERIES_KFOLD": self._raw_settings["mode"] = "TIME_SERIES_SINGLE_SPLIT" else: self._raw_settings["mode"] = "SHUFFLE" if split_ratio is not None: if not (isinstance(split_ratio, float) and split_ratio > 0 and split_ratio < 1): warnings.warn("HyperparameterSearchSettings.set_single_split_validation ignoring invalid input: split_ratio") warnings.warn(" split_ratio must be float between 0 and 1") else: self._raw_settings["splitRatio"] = split_ratio if stratified is not None: if not isinstance(stratified, bool): warnings.warn("HyperparameterSearchSettings.set_single_split_validation ignoring invalid input: stratified") warnings.warn("stratified must be a boolean") else: self._raw_settings["stratified"] = stratified self._set_cv_seed(cv_seed) def set_custom_validation(self, code=None): """ Sets the validation mode to "CUSTOM". :param code: definition of the validation :type code: str """ self._raw_settings["mode"] = "CUSTOM" if code is not None: if not isinstance(code, string_types): warnings.warn("HyperparameterSearchSettings.set_custom_validation ignoring invalid input: code") warnings.warn("code must be a Python interpretable string") else: self._raw_settings["code"] = code def set_search_distribution(self, distributed=False, n_containers=4): """ Sets the distribution parameters for the hyperparameter search execution. :param distributed: if True, distribute search in the Kubernetes cluster selected in the runtime environment's containerized execution configuration, defaults to False :type distributed: bool :param n_containers: number of containers to use for the distributed search, defaults to 4 :type n_containers: int """ assert isinstance(distributed, bool) if n_containers is not None: assert isinstance(n_containers, int) self._raw_settings["nContainers"] = n_containers self._raw_settings["distributed"] = distributed @property def distributed(self): return self._raw_settings["distributed"] @distributed.setter def distributed(self, distributed): assert isinstance(distributed, bool) self._raw_settings["distributed"] = distributed @property def timeout(self): return self._raw_settings["timeout"] @timeout.setter def timeout(self, timeout): assert isinstance(timeout, int) self._raw_settings["timeout"] = timeout @property def n_iter(self): if self._raw_settings["strategy"] == "GRID": return self._raw_settings["nIter"] else: # RANDOM and BAYESIAN search strategies return self._raw_settings["nIterRandom"] @n_iter.setter def n_iter(self, n_iter): assert isinstance(n_iter, int) if self._raw_settings["strategy"] == "GRID": self._raw_settings["nIter"] = n_iter else: self._raw_settings["nIterRandom"] = n_iter @property def parallelism(self): return self._raw_settings["nJobs"] @parallelism.setter def parallelism(self, n_jobs): assert isinstance(n_jobs, int) self._raw_settings["nJobs"] = n_jobs class HyperparameterSettings(object): def __init__(self, name, algo_settings): self.name = name self._algo_settings = algo_settings def _repr_html_(self): return "
" + self._pretty_repr() + "
{}
".format(self.__repr__()) def _pretty_repr(self): raise NotImplementedError() class NumericalHyperparameterSettings(HyperparameterSettings): def _pretty_repr(self): raw_hyperparam = self._algo_settings[self.name] pretty_hyperparam = dict() if self._algo_settings.strategy == "GRID": pretty_hyperparam["definition_mode"] = raw_hyperparam["gridMode"] else: # RANDOM and BAYESIAN strategies pretty_hyperparam["definition_mode"] = raw_hyperparam["randomMode"] if self.definition_mode == "EXPLICIT": pretty_hyperparam["values"] = raw_hyperparam["values"] else: pretty_hyperparam["range"] = raw_hyperparam["range"] return self.__class__.__name__ + "(hyperparameter=\"{}\", settings={})".format(self.name, json.dumps(pretty_hyperparam, indent=4)) def __repr__(self): raw_dict = self._algo_settings[self.name] return self.__class__.__name__ + "(hyperparameter=\"{}\", settings={})".format(self.name, json.dumps(raw_dict)) __str__ = __repr__ @property def definition_mode(self): """ "EXPLICIT" means that the hyperparameter search is performed over a given set of values (default for grid search) "RANGE" means that the hyperparameter search is performed over a range of values (default for random and Bayesian searches) :return: str mode: "EXPLICIT" | "RANGE" """ if self._algo_settings.strategy == "GRID": return self._algo_settings[self.name]["gridMode"] else: # RANDOM and BAYESIAN search strategies return self._algo_settings[self.name]["randomMode"] @definition_mode.setter def definition_mode(self, mode): """ :param mode: "EXPLICIT" | "RANGE" :type mode: str """ assert mode in ["EXPLICIT", "RANGE"], "Hyperparameter definition mode must be either \"EXPLICIT\" or \"RANGE\"" if self._algo_settings.strategy == "GRID": self._algo_settings[self.name]["gridMode"] = mode else: # RANDOM and BAYESIAN search strategies self._algo_settings[self.name]["randomMode"] = mode def set_explicit_values(self, values): """ Sets both: - the explicit values to search over for the current numerical hyperparameter - the definition mode of the current numerical hyperparameter to "EXPLICIT" :param values: the explicit list of numerical values considered for this hyperparameter in the search :type values: list of float | list of int """ self.values = values self.definition_mode = "EXPLICIT" @property def values(self): """ :return: the explicit list of numerical values considered for this hyperparameter in the search :rtype: list """ return self._algo_settings[self.name]["values"] @values.setter def values(self, values): """ :param values: the explicit list of numerical values considered for this hyperparameter in the search :type values: list of float | list of int """ error_message = "Invalid values input type for hyperparameter " \ "\"{}\": ".format(self.name) + \ " expecting a non-empty list of numbers" assert values is not None and isinstance(values, list) and len(values) > 0, error_message for val in values: assert isinstance(val, int) or isinstance(val, float), error_message limit_min = self._algo_settings[self.name]["limit"].get("min") if limit_min is not None: assert all(limit_min <= val for val in values), "Value(s) below hyperparameter \"{}\" limit {}".format(self.name, limit_min) limit_max = self._algo_settings[self.name]["limit"].get("max") if limit_max is not None: assert all(val <= limit_max for val in values), "Value(s) above hyperparameter \"{}\" limit {}".format(self.name, limit_max) if len(set(values)) < len(values): warnings.warn("Detected duplicates in provided values: " + str(sorted(values))) self._algo_settings[self.name]["values"] = values def _check_number_input(self, input): assert isinstance(input, int) or isinstance(input, float), \ "Invalid input type for hyperparameter \"{}\": ".format(self.name) + \ "range bounds must be numbers" def _set_range(self, min=None, max=None, nb_values=None): if min is None and max is None and nb_values is None: warnings.warn("Numerical range for hyperparameter \"{}\" not modified".format(self.name)) else: # Check all the Range parameters input before setting any of them if min is not None: self._check_number_input(min) limit_min = self._algo_settings[self.name]["limit"].get("min") if limit_min is not None: assert limit_min <= min, "Range min {} is below hyperparameter \"{}\" limit {}".format(min, self.name, limit_min) if max is not None: self._check_number_input(max) limit_max = self._algo_settings[self.name]["limit"].get("max") if limit_max is not None: assert max <= limit_max, "Range max {} is above hyperparameter \"{}\" limit {}".format(max, self.name, limit_max) if min is not None and max is not None: assert min <= max, "Invalid Range: min {} is greater max {}".format(min, max) if nb_values is not None: assert isinstance(nb_values, int) and nb_values >= 2, "Range number of values for hyperparameter \"{}\" must be an integer and >= 2".format(self.name) # Set the Range parameters after they have been checked if min is not None: self._algo_settings[self.name]["range"]["min"] = min if max is not None: self._algo_settings[self.name]["range"]["max"] = max if nb_values is not None: self._algo_settings[self.name]["range"]["nbValues"] = nb_values class RangeSettings(object): """ [Internal] Range of a numerical hyperparameter (points to the algorithm settings) Should not be used directly by end users of the API """ def __init__(self, numerical_hyperparameter_settings): self._numerical_hyperparameter_settings = numerical_hyperparameter_settings self._range_dict = self._numerical_hyperparameter_settings._algo_settings[numerical_hyperparameter_settings.name]["range"] def __repr__(self): return "RangeSettings(min={}, max={}, nb_values={})".format(self.min, self.max, self.nb_values) @property def min(self): """ :return: the lower bound of the range for this hyperparameter :rtype: float | int """ return self._range_dict["min"] @min.setter def min(self, value): """ :param value: the lower bound of the range for this hyperparameter :type value: float | int """ self._numerical_hyperparameter_settings._set_range(min=value) @property def max(self): """ :return: the upper bound of the range for this hyperparameter :rtype: float | int """ return self._range_dict["max"] @max.setter def max(self, value): """ :param value: the upper bound of the range for this hyperparameter :type value: float | int """ self._numerical_hyperparameter_settings._set_range(max=value) @property def nb_values(self): """ :return: for grid-search ("GRID" strategy) only, the number of values between min and max to consider :rtype: int """ return self._range_dict["nbValues"] @nb_values.setter def nb_values(self, value): """ :param value: for grid-search ("GRID" strategy) only, the number of values between min and max to consider :type value: int """ self._numerical_hyperparameter_settings._set_range(nb_values=value) def set_range(self, min=None, max=None, nb_values=None): """ Sets both: - the range parameters to search over for the current numerical hyperparameter - the definition mode of the current numerical hyperparameter to "RANGE" :param min: the lower bound of the range for this hyperparameter :type min: float | int :param max: the upper bound of the range for this hyperparameter :type max: float | int :param nb_values: for grid-search ("GRID" strategy) only, the number of values between min and max to consider :type nb_values: int """ self._set_range(min=min, max=max, nb_values=nb_values) self.definition_mode = "RANGE" @property def range(self): return NumericalHyperparameterSettings.RangeSettings(self) class Range(object): """ Range of a numerical hyperparameter (min, max, nb_values) Use this class to define explicitly the parameters of the range of a numerical hyperparameter """ def _check_input(self, value): assert isinstance(value, (int, float)), "Invalid input type for Range: {}".format(type(value)) def __init__(self, min, max, nb_values=None): self._check_input(min) self._check_input(max) assert min <= max, "Invalid Range: min must be lower than max" self.min = min self.max = max self.nb_values = nb_values def __repr__(self): return "Range(min={}, max={}, nb_values={})".format(self.min, self.max, self.nb_values) class CategoricalHyperparameterSettings(HyperparameterSettings): def __repr__(self): return self.__class__.__name__ + "(hyperparameter=\"{}\", settings={})".format(self.name, json.dumps(self._algo_settings[self.name])) __str__ = __repr__ def _pretty_repr(self): return self.__class__.__name__ + "(hyperparameter=\"{}\", settings={})".format(self.name, json.dumps(self._algo_settings[self.name], indent=4)) def set_values(self, values): """ Enables the search over listed values (categories). :param values: list of values to enable, all other values will be disabled :type values: list of str """ assert isinstance(values, list), \ "Invalid input type {} for categorical hyperparameter {}: must be a list of strings".format(type(values), self.name) all_possible_values = self.get_all_possible_values() for category in values: assert isinstance(category, string_types), \ "Invalid input type {} for categorical hyperparameter {}: must be a string".format(type(category), self.name) assert category in all_possible_values, \ "Invalid input value \"{}\" for categorical hyperparameter {}: must be a member of {}".format(category, self.name, all_possible_values) for category in all_possible_values: if category in values: self._algo_settings[self.name]["values"][category] = {"enabled": True} else: self._algo_settings[self.name]["values"][category] = {"enabled": False} def get_values(self): """ :return: list of enabled categories for this hyperparameter :rtype: list of str """ values_dict = self._algo_settings[self.name]["values"] return [value for value in values_dict.keys() if values_dict[value]["enabled"]] def get_all_possible_values(self): """ :return: list of possible values for this hyperparameter :rtype: list of str """ return list(self._algo_settings[self.name]["values"].keys()) class SingleValueHyperparameterSettings(HyperparameterSettings): def __init__(self, name, algo_settings, accepted_types=None): super(SingleValueHyperparameterSettings, self).__init__(name, algo_settings) self.accepted_types = accepted_types def __repr__(self): return self.__class__.__name__ + "(hyperparameter=\"{}\", value={})".format(self.name, self._algo_settings[self.name]) __str__ = __repr__ _pretty_repr = __repr__ def set_value(self, value): """ :param value: :type value: bool | int | float | NoneType """ if self.accepted_types is not None: assert isinstance(value, tuple(self.accepted_types)), "Invalid type for hyperparameter {}. Must be one of: {}".format(self.name, self.accepted_types) self._algo_settings[self.name] = value def get_value(self): """ :return: current value :rtype: bool | int | float | NoneType """ return self._algo_settings[self.name] def get_accepted_types(self): """ :return: valid types for this hyperparameter """ return self.accepted_types class SingleCategoryHyperparameterSettings(HyperparameterSettings): def __init__(self, name, algo_settings, accepted_values=None): super(SingleCategoryHyperparameterSettings, self).__init__(name, algo_settings) self.accepted_values = accepted_values def __repr__(self): if self.accepted_values is not None: return self.__class__.__name__ + "(hyperparameter=\"{}\", value=\"{}\", accepted_values={})".format(self.name, self._algo_settings[self.name], self.accepted_values) else: return self.__class__.__name__ + "(hyperparameter=\"{}\", value=\"{}\")".format(self.name, self._algo_settings[self.name]) __str__ = __repr__ _pretty_repr = __repr__ def set_value(self, value): """ :param value: :type value: str """ if self.accepted_values is not None: assert value in self.accepted_values, "Invalid value for hyperparameter {}. Must be in {}".format(self.name, json.dumps(self.accepted_values)) self._algo_settings[self.name] = value def get_value(self): """ :return: current value :rtype: str """ return self._algo_settings[self.name] def get_all_possible_values(self): """ :return: list of possible values for this hyperparameter :rtype: list of str """ return self.accepted_values class PredictionAlgorithmSettings(dict): """ Object to read and modify the settings of a prediction ML algorithm. Do not create this object directly, use :meth:`DSSMLTask.get_algorithm_settings` instead """ def __init__(self, raw_settings, hyperparameter_search_params): super(PredictionAlgorithmSettings, self).__init__(raw_settings) self._hyperparameter_search_params = hyperparameter_search_params self._hyperparameters_registry = dict() self._attr_to_json_remapping = dict() def __setattr__(self, attr_name, value): if not hasattr(self, attr_name): # call from __init__ super(PredictionAlgorithmSettings, self).__setattr__(attr_name, value) else: if attr_name in self._attr_to_json_remapping: # attribute name and json key mismatch (e.g. "lambda", "alphaMode") json_key = self._attr_to_json_remapping[attr_name] else: json_key = attr_name if json_key in self._hyperparameters_registry: # syntactic sugars target = self._hyperparameters_registry[json_key] if isinstance(target, (SingleValueHyperparameterSettings, SingleCategoryHyperparameterSettings)): target.set_value(value) elif isinstance(target, CategoricalHyperparameterSettings): target.set_values(value) elif isinstance(target, NumericalHyperparameterSettings): if isinstance(value, list): # algo.hyperparam = [x, y, z] target.set_explicit_values(values=value) elif isinstance(value, Range): # algo.hyperparam = Range(min=x, max=y, nb_values=z) target.set_range(min=value.min, max=value.max, nb_values=value.nb_values) elif isinstance(value, NumericalHyperparameterSettings): # algo.hyperparam = other_algo.other_hyperparam target.set_range(min=value.range.min, max=value.range.max, nb_values=value.range.nb_values) target.set_explicit_values(values=list(value.values)) target.definition_mode = value.definition_mode else: raise TypeError(("Invalid type for NumericalHyperparameterSettings {}\n" + "Expecting either list, Range or NumericalHyperparameterSettings").format(attr_name)) else: # simple parameter assert isinstance(value, type(target)), "Invalid type {} for parameter {}: expected {}".format(type(value), attr_name, type(target)) super(PredictionAlgorithmSettings, self).__setattr__(attr_name, value) # update attribute value self[json_key] = value # update underlying dict value for key json_key self._hyperparameters_registry[json_key] = value else: # other cases (properties setter, ...) super(PredictionAlgorithmSettings, self).__setattr__(attr_name, value) def _register_numerical_hyperparameter(self, json_key, attr_name=None): if attr_name is not None: self._attr_to_json_remapping[attr_name] = json_key self._hyperparameters_registry[json_key] = NumericalHyperparameterSettings(json_key, self) return self._hyperparameters_registry[json_key] def _register_categorical_hyperparameter(self, json_key, attr_name=None): if attr_name is not None: self._attr_to_json_remapping[attr_name] = json_key self._hyperparameters_registry[json_key] = CategoricalHyperparameterSettings(json_key, self) return self._hyperparameters_registry[json_key] def _register_single_category_hyperparameter(self, json_key, accepted_values=None, attr_name=None): if attr_name is not None: self._attr_to_json_remapping[attr_name] = json_key self._hyperparameters_registry[json_key] = SingleCategoryHyperparameterSettings(json_key, self, accepted_values=accepted_values) return self._hyperparameters_registry[json_key] def _register_single_value_hyperparameter(self, json_key, accepted_types=None, attr_name=None): if attr_name is not None: self._attr_to_json_remapping[attr_name] = json_key self._hyperparameters_registry[json_key] = SingleValueHyperparameterSettings(json_key, self, accepted_types=accepted_types) return self._hyperparameters_registry[json_key] def _register_simple_parameter(self, json_key, attr_name=None): if attr_name is not None: self._attr_to_json_remapping[attr_name] = json_key self._hyperparameters_registry[json_key] = self[json_key] return self._hyperparameters_registry[json_key] def _repr_html_(self): res = "
" + self.__class__.__name__ + "(\n"
        res += "    \"enabled\": {}".format(self.enabled) + "\n"
        for name, hyperparam_settings in self._hyperparameters_registry.items():
            if isinstance(hyperparam_settings, HyperparameterSettings):
                res += "    \"{}\": {}".format(name, hyperparam_settings._pretty_repr()) + "\n"
            else:
                res += "    \"{}\": {}".format(name, hyperparam_settings) + "\n"
        res += ")
" return res + "
{}
".format(self.__repr__()) def __repr__(self): return self.__class__.__name__ + "(values={})".format(dict.__repr__(self)) __str__ = __repr__ def _get_all_hyperparameter_names(self): return list(self._hyperparameters_registry.keys()) @property def enabled(self): """ :rtype: bool """ return self["enabled"] @enabled.setter def enabled(self, enabled): """ :param enabled: :type enabled: bool """ assert isinstance(enabled, bool), "enabled property must be a boolean" self["enabled"] = enabled @property def strategy(self): return self._hyperparameter_search_params["strategy"] @strategy.setter def strategy(self, _): raise AttributeError("The strategy must be set at the MLTask settings level.\n" "To update the search strategy, use .strategy = ..., " "obtained with .get_hyperparameter_search_settings()") class RandomForestSettings(PredictionAlgorithmSettings): def __init__(self, raw_settings, hyperparameter_search_params): super(RandomForestSettings, self).__init__(raw_settings, hyperparameter_search_params) self.n_estimators = self._register_numerical_hyperparameter("n_estimators") self.min_samples_leaf = self._register_numerical_hyperparameter("min_samples_leaf") self.max_tree_depth = self._register_numerical_hyperparameter("max_tree_depth") self.max_feature_prop = self._register_numerical_hyperparameter("max_feature_prop") self.max_features = self._register_numerical_hyperparameter("max_features") self.n_jobs = self._register_simple_parameter("n_jobs") self.selection_mode = self._register_single_category_hyperparameter("selection_mode", accepted_values=["auto", "sqrt", "log2", "number", "prop"]) class LightGBMSettings(PredictionAlgorithmSettings): def __init__(self, raw_settings, hyperparameter_search_params): super(LightGBMSettings, self).__init__(raw_settings, hyperparameter_search_params) self.boosting_type = self._register_categorical_hyperparameter("boosting_type") self.num_leaves = self._register_numerical_hyperparameter("num_leaves") self.learning_rate = self._register_numerical_hyperparameter("learning_rate") self.n_estimators = self._register_numerical_hyperparameter("n_estimators") self.min_split_gain = self._register_numerical_hyperparameter("min_split_gain") self.min_child_weight = self._register_numerical_hyperparameter("min_child_weight") self.min_child_samples = self._register_numerical_hyperparameter("min_child_samples") self.colsample_bytree = self._register_numerical_hyperparameter("colsample_bytree") self.reg_alpha = self._register_numerical_hyperparameter("reg_alpha") self.reg_lambda = self._register_numerical_hyperparameter("reg_lambda") self.early_stopping = self._register_single_value_hyperparameter("early_stopping", accepted_types=[bool]) self.early_stopping_rounds = self._register_single_value_hyperparameter("early_stopping_rounds", accepted_types=[int]) self.random_state = self._register_single_value_hyperparameter("random_state", accepted_types=[int]) self.n_jobs = self._register_single_value_hyperparameter("n_jobs", accepted_types=[int]) self.max_depth = self._register_single_value_hyperparameter("max_depth", accepted_types=[int]) self.subsample = self._register_single_value_hyperparameter("subsample", accepted_types=[float]) self.subsample_freq = self._register_single_value_hyperparameter("subsample_freq", accepted_types=[int]) self.use_bagging = self._register_single_value_hyperparameter("use_bagging", accepted_types=[bool]) class XGBoostSettings(PredictionAlgorithmSettings): def __init__(self, raw_settings, hyperparameter_search_params): super(XGBoostSettings, self).__init__(raw_settings, hyperparameter_search_params) self.max_depth = self._register_numerical_hyperparameter("max_depth") self.learning_rate = self._register_numerical_hyperparameter("learning_rate") self.gamma = self._register_numerical_hyperparameter("gamma") self.min_child_weight = self._register_numerical_hyperparameter("min_child_weight") self.max_delta_step = self._register_numerical_hyperparameter("max_delta_step") self.subsample = self._register_numerical_hyperparameter("subsample") self.colsample_bytree = self._register_numerical_hyperparameter("colsample_bytree") self.colsample_bylevel = self._register_numerical_hyperparameter("colsample_bylevel") self.alpha = self._register_numerical_hyperparameter("alpha") self.lambda_ = self._register_numerical_hyperparameter("lambda", attr_name="lambda_") self.booster = self._register_categorical_hyperparameter("booster") self.objective = self._register_categorical_hyperparameter("objective") self.n_estimators = self._register_single_value_hyperparameter("n_estimators", accepted_types=[int]) self.nthread = self._register_simple_parameter("nthread") self.scale_pos_weight = self._register_single_value_hyperparameter("scale_pos_weight", accepted_types=[int, float]) self.base_score = self._register_single_value_hyperparameter("base_score", accepted_types=[int, float]) self.impute_missing = self._register_single_value_hyperparameter("impute_missing", accepted_types=[bool]) self.missing = self._register_single_value_hyperparameter("missing", accepted_types=[int, float]) self.cpu_tree_method = self._register_single_category_hyperparameter("cpu_tree_method", accepted_values=["auto", "exact", "approx", "hist"]) self.gpu_tree_method = self._register_single_category_hyperparameter("gpu_tree_method", accepted_values=["gpu_exact", "gpu_hist"]) self.enable_cuda = self._register_simple_parameter("enable_cuda") self.seed = self._register_single_value_hyperparameter("seed", accepted_types=[int]) self.enable_early_stopping = self._register_single_value_hyperparameter("enable_early_stopping", accepted_types=[bool]) self.early_stopping_rounds = self._register_single_value_hyperparameter("early_stopping_rounds", accepted_types=[int]) class GradientBoostedTreesSettings(PredictionAlgorithmSettings): def __init__(self, raw_settings, hyperparameter_search_params): super(GradientBoostedTreesSettings, self).__init__(raw_settings, hyperparameter_search_params) self.n_estimators = self._register_numerical_hyperparameter("n_estimators") self.max_depth = self._register_numerical_hyperparameter("max_depth") self.min_samples_leaf = self._register_numerical_hyperparameter("min_samples_leaf") self.max_features = self._register_numerical_hyperparameter("max_features") self.max_feature_prop = self._register_numerical_hyperparameter("max_feature_prop") self.learning_rate = self._register_numerical_hyperparameter("learning_rate") self.loss = self._register_categorical_hyperparameter("loss") self.selection_mode = self._register_single_category_hyperparameter("selection_mode", accepted_values=["auto", "sqrt", "log2", "number", "prop"]) class DecisionTreeSettings(PredictionAlgorithmSettings): def __init__(self, raw_settings, hyperparameter_search_params): super(DecisionTreeSettings, self).__init__(raw_settings, hyperparameter_search_params) self.max_depth = self._register_numerical_hyperparameter("max_depth") self.min_samples_leaf = self._register_numerical_hyperparameter("min_samples_leaf") self.criterion = self._register_categorical_hyperparameter("criterion") self.splitter = self._register_categorical_hyperparameter("splitter") class LogitSettings(PredictionAlgorithmSettings): def __init__(self, raw_settings, hyperparameter_search_params): super(LogitSettings, self).__init__(raw_settings, hyperparameter_search_params) self.C = self._register_numerical_hyperparameter("C") self.penalty = self._register_categorical_hyperparameter("penalty") self.multi_class = self._register_single_category_hyperparameter("multi_class", accepted_values=["multinomial", "ovr"]) self.n_jobs = self._register_simple_parameter("n_jobs") class RidgeRegressionSettings(PredictionAlgorithmSettings): def __init__(self, raw_settings, hyperparameter_search_params): super(RidgeRegressionSettings, self).__init__(raw_settings, hyperparameter_search_params) self.alpha = self._register_numerical_hyperparameter("alpha") self.alpha_mode = self._register_single_category_hyperparameter("alphaMode", accepted_values=["MANUAL", "AUTO"], attr_name="alpha_mode") class LassoRegressionSettings(PredictionAlgorithmSettings): def __init__(self, raw_settings, hyperparameter_search_params): super(LassoRegressionSettings, self).__init__(raw_settings, hyperparameter_search_params) self.alpha = self._register_numerical_hyperparameter("alpha") self.alpha_mode = self._register_single_category_hyperparameter("alphaMode", accepted_values=["MANUAL", "AUTO_CV", "AUTO_IC"], attr_name="alpha_mode") class OLSSettings(PredictionAlgorithmSettings): def __init__(self, raw_settings, hyperparameter_search_params): super(OLSSettings, self).__init__(raw_settings, hyperparameter_search_params) self.n_jobs = self._register_simple_parameter("n_jobs") class LARSSettings(PredictionAlgorithmSettings): def __init__(self, raw_settings, hyperparameter_search_params): super(LARSSettings, self).__init__(raw_settings, hyperparameter_search_params) self.max_features = self._register_single_value_hyperparameter("max_features", accepted_types=[int]) self.K = self._register_single_value_hyperparameter("K", accepted_types=[int]) class SGDSettings(PredictionAlgorithmSettings): def __init__(self, raw_settings, hyperparameter_search_params): super(SGDSettings, self).__init__(raw_settings, hyperparameter_search_params) self.alpha = self._register_numerical_hyperparameter("alpha") self.loss = self._register_categorical_hyperparameter("loss") self.penalty = self._register_categorical_hyperparameter("penalty") self.l1_ratio = self._register_single_value_hyperparameter("l1_ratio", accepted_types=[int, float]) self.max_iter = self._register_single_value_hyperparameter("max_iter", accepted_types=[int]) self.tol = self._register_single_value_hyperparameter("tol", accepted_types=[int, float]) self.n_jobs = self._register_simple_parameter("n_jobs") class KNNSettings(PredictionAlgorithmSettings): def __init__(self, raw_settings, hyperparameter_search_params): super(KNNSettings, self).__init__(raw_settings, hyperparameter_search_params) self.k = self._register_numerical_hyperparameter("k") self.algorithm = self._register_single_category_hyperparameter("algorithm", accepted_values=["auto", "kd_tree", "ball_tree", "brute"]) self.distance_weighting = self._register_single_value_hyperparameter("distance_weighting", accepted_types=[bool]) self.p = self._register_single_value_hyperparameter("p", accepted_types=[int]) self.leaf_size = self._register_single_value_hyperparameter("leaf_size", accepted_types=[int]) class SVMSettings(PredictionAlgorithmSettings): def __init__(self, raw_settings, hyperparameter_search_params): super(SVMSettings, self).__init__(raw_settings, hyperparameter_search_params) self.custom_gamma = self._register_numerical_hyperparameter("custom_gamma") self.C = self._register_numerical_hyperparameter("C") self.gamma = self._register_categorical_hyperparameter("gamma") self.kernel = self._register_categorical_hyperparameter("kernel") self.coef0 = self._register_single_value_hyperparameter("coef0", accepted_types=[int, float]) self.tol = self._register_single_value_hyperparameter("tol", accepted_types=[int, float]) self.max_iter = self._register_single_value_hyperparameter("max_iter", accepted_types=[int]) class SingleLayerPerceptronSettings(PredictionAlgorithmSettings): def __init__(self, raw_settings, hyperparameter_search_params): super(SingleLayerPerceptronSettings, self).__init__(raw_settings, hyperparameter_search_params) self.layer_sizes = self._register_numerical_hyperparameter("layer_sizes") self.activation = self._register_single_category_hyperparameter("activation", accepted_values=["relu", "identity", "logistic", "tanh"]) self.solver = self._register_single_category_hyperparameter("solver", accepted_values=["adam", "sgd"]) self.alpha = self._register_single_value_hyperparameter("alpha", accepted_types=[int, float]) self.batch_size = self._register_single_value_hyperparameter("batch_size", accepted_types=[int]) self.auto_batch = self._register_single_value_hyperparameter("auto_batch", accepted_types=[bool]) self.max_iter = self._register_single_value_hyperparameter("max_iter", accepted_types=[int]) self.seed = self._register_single_value_hyperparameter("seed", accepted_types=[int]) self.tol = self._register_single_value_hyperparameter("tol", accepted_types=[int, float]) self.early_stopping = self._register_single_value_hyperparameter("early_stopping", accepted_types=[bool]) self.validation_fraction = self._register_single_value_hyperparameter("validation_fraction", accepted_types=[int, float]) self.beta_1 = self._register_single_value_hyperparameter("beta_1", accepted_types=[int, float]) self.beta_2 = self._register_single_value_hyperparameter("beta_2", accepted_types=[int, float]) self.epsilon = self._register_single_value_hyperparameter("epsilon", accepted_types=[int, float]) self.learning_rate = self._register_single_category_hyperparameter("learning_rate", accepted_values=["constant", "invscaling", "adaptive"]) self.power_t = self._register_single_value_hyperparameter("power_t", accepted_types=[int, float]) self.momentum = self._register_single_value_hyperparameter("momentum", accepted_types=[int, float]) self.nesterovs_momentum = self._register_single_value_hyperparameter("nesterovs_momentum", accepted_types=[bool]) self.shuffle = self._register_single_value_hyperparameter("shuffle", accepted_types=[bool]) self.learning_rate_init = self._register_single_value_hyperparameter("learning_rate_init", accepted_types=[int, float]) class DeepNeuralNetworkSettings(PredictionAlgorithmSettings): def __init__(self, raw_settings, hyperparameter_search_params): super(DeepNeuralNetworkSettings, self).__init__(raw_settings, hyperparameter_search_params) self.learning_rate = self._register_numerical_hyperparameter("learning_rate") self.hidden_layers = self._register_numerical_hyperparameter("hidden_layers") self.units = self._register_numerical_hyperparameter("units") self.batch_size = self._register_single_value_hyperparameter("batch_size", accepted_types=[int]) self.max_epochs = self._register_single_value_hyperparameter("max_epochs", accepted_types=[int]) self.device = self._register_single_category_hyperparameter("device", accepted_values=["cpu", "cuda"]) self.early_stopping_enabled = self._register_single_value_hyperparameter("early_stopping_enabled", accepted_types=[bool]) self.early_stopping_patience = self._register_single_value_hyperparameter("early_stopping_patience", accepted_types=[int]) self.early_stopping_threshold = self._register_single_value_hyperparameter("early_stopping_threshold", accepted_types=[float]) class MLLibLogitSettings(PredictionAlgorithmSettings): def __init__(self, raw_settings, hyperparameter_search_params): super(MLLibLogitSettings, self).__init__(raw_settings, hyperparameter_search_params) self.reg_param = self._register_numerical_hyperparameter("reg_param") self.enet_param = self._register_numerical_hyperparameter("enet_param") self.max_iter = self._register_single_value_hyperparameter("max_iter", accepted_types=[int]) class MLLibNaiveBayesSettings(PredictionAlgorithmSettings): def __init__(self, raw_settings, hyperparameter_search_params): super(MLLibNaiveBayesSettings, self).__init__(raw_settings, hyperparameter_search_params) self.lambda_ = self._register_numerical_hyperparameter("lambda", attr_name="lambda_") class MLLibLinearRegressionSettings(PredictionAlgorithmSettings): def __init__(self, raw_settings, hyperparameter_search_params): super(MLLibLinearRegressionSettings, self).__init__(raw_settings, hyperparameter_search_params) self.reg_param = self._register_numerical_hyperparameter("reg_param") self.enet_param = self._register_numerical_hyperparameter("enet_param") self.max_iter = self._register_single_value_hyperparameter("max_iter", accepted_types=[int]) class MLLibDecisionTreeSettings(PredictionAlgorithmSettings): def __init__(self, raw_settings, hyperparameter_search_params): super(MLLibDecisionTreeSettings, self).__init__(raw_settings, hyperparameter_search_params) self.max_depth = self._register_numerical_hyperparameter("max_depth") self.cache_node_ids = self._register_simple_parameter("cache_node_ids") self.checkpoint_interval = self._register_single_value_hyperparameter("checkpoint_interval", accepted_types=[int]) self.max_bins = self._register_single_value_hyperparameter("max_bins", accepted_types=[int]) self.max_memory_mb = self._register_simple_parameter("max_memory_mb") self.min_info_gain = self._register_single_value_hyperparameter("min_info_gain", accepted_types=[int, float]) self.min_instance_per_node = self._register_single_value_hyperparameter("min_instance_per_node", accepted_types=[int]) class _MLLibTreeEnsembleSettings(PredictionAlgorithmSettings): def __init__(self, raw_settings, hyperparameter_search_params): super(_MLLibTreeEnsembleSettings, self).__init__(raw_settings, hyperparameter_search_params) self.max_depth = self._register_numerical_hyperparameter("max_depth") self.num_trees = self._register_numerical_hyperparameter("num_trees") self.cache_node_ids = self._register_simple_parameter("cache_node_ids") self.checkpoint_interval = self._register_single_value_hyperparameter("checkpoint_interval", accepted_types=[int]) self.max_bins = self._register_single_value_hyperparameter("max_bins", accepted_types=[int]) self.max_memory_mb = self._register_simple_parameter("max_memory_mb") self.min_info_gain = self._register_single_value_hyperparameter("min_info_gain", accepted_types=[int, float]) self.min_instance_per_node = self._register_single_value_hyperparameter("min_instance_per_node", accepted_types=[int]) self.seed = self._register_single_value_hyperparameter("seed", accepted_types=[int]) self.subsampling_rate = self._register_single_value_hyperparameter("subsampling_rate", accepted_types=[int, float]) class MLLibRandomForestSettings(_MLLibTreeEnsembleSettings): def __init__(self, raw_settings, hyperparameter_search_params): super(MLLibRandomForestSettings, self).__init__(raw_settings, hyperparameter_search_params) self.impurity = self._register_single_category_hyperparameter("impurity", accepted_values=["gini", "entropy", "variance"]) self.subset_strategy = self._register_single_category_hyperparameter("subset_strategy", accepted_values=["auto", "all", "onethird", "sqrt", "log2"]) class MLLibGBTSettings(_MLLibTreeEnsembleSettings): def __init__(self, raw_settings, hyperparameter_search_params): super(MLLibGBTSettings, self).__init__(raw_settings, hyperparameter_search_params) self.step_size = self._register_numerical_hyperparameter("step_size") class SeasonalNaiveSettings(PredictionAlgorithmSettings): def __init__(self, raw_settings, hyperparameter_search_params): super(SeasonalNaiveSettings, self).__init__(raw_settings, hyperparameter_search_params) self.season_length = self._register_numerical_hyperparameter("season_length") class AutoArimaSettings(PredictionAlgorithmSettings): def __init__(self, raw_settings, hyperparameter_search_params): super(AutoArimaSettings, self).__init__(raw_settings, hyperparameter_search_params) self.m = self._register_numerical_hyperparameter("m") self.information_criterion = self._register_categorical_hyperparameter("information_criterion") self.test = self._register_categorical_hyperparameter("test") self.seasonal_test = self._register_categorical_hyperparameter("seasonal_test") self.method = self._register_categorical_hyperparameter("method") self.start_p = self._register_single_value_hyperparameter("start_p", accepted_types=[int]) self.max_p = self._register_single_value_hyperparameter("max_p", accepted_types=[int]) self.d = self._register_single_value_hyperparameter("d", accepted_types=[int, type(None)]) self.max_d = self._register_single_value_hyperparameter("max_d", accepted_types=[int]) self.start_q = self._register_single_value_hyperparameter("start_q", accepted_types=[int]) self.max_q = self._register_single_value_hyperparameter("max_q", accepted_types=[int]) self.start_P = self._register_single_value_hyperparameter("start_P", accepted_types=[int]) self.max_P = self._register_single_value_hyperparameter("max_P", accepted_types=[int]) self.D = self._register_single_value_hyperparameter("D", accepted_types=[int, type(None)]) self.max_D = self._register_single_value_hyperparameter("max_D", accepted_types=[int]) self.start_Q = self._register_single_value_hyperparameter("start_Q", accepted_types=[int]) self.max_Q = self._register_single_value_hyperparameter("max_Q", accepted_types=[int]) self.max_order = self._register_single_value_hyperparameter("max_order", accepted_types=[int]) self.stationary = self._register_single_value_hyperparameter("stationary", accepted_types=[bool]) self.maxiter = self._register_single_value_hyperparameter("maxiter", accepted_types=[int]) class SeasonalLoessSettings(PredictionAlgorithmSettings): def __init__(self, raw_settings, hyperparameter_search_params): super(SeasonalLoessSettings, self).__init__(raw_settings, hyperparameter_search_params) self.period = self._register_numerical_hyperparameter("period") self.seasonal = self._register_numerical_hyperparameter("seasonal") self.trend = self._register_numerical_hyperparameter("trend") self.low_pass = self._register_numerical_hyperparameter("low_pass") self.seasonal_deg = self._register_categorical_hyperparameter("seasonal_deg") self.trend_deg = self._register_categorical_hyperparameter("trend_deg") self.low_pass_deg = self._register_categorical_hyperparameter("low_pass_deg") self.seasonal_jump = self._register_numerical_hyperparameter("seasonal_jump") self.trend_jump = self._register_numerical_hyperparameter("trend_jump") self.low_pass_jump = self._register_numerical_hyperparameter("low_pass_jump") self.auto_trend = self._register_single_value_hyperparameter("auto_trend", accepted_types=[bool]) self.auto_low_pass = self._register_single_value_hyperparameter("auto_low_pass", accepted_types=[bool]) class GluonTSNPTSForecasterSettings(PredictionAlgorithmSettings): def __init__(self, raw_settings, hyperparameter_search_params): super(GluonTSNPTSForecasterSettings, self).__init__(raw_settings, hyperparameter_search_params) self.context_length = self._register_numerical_hyperparameter("context_length") self.kernel_type = self._register_categorical_hyperparameter("kernel_type") self.exp_kernel_weights = self._register_numerical_hyperparameter("exp_kernel_weights") self.feature_scale = self._register_numerical_hyperparameter("feature_scale") self.full_context = self._register_single_value_hyperparameter("full_context", accepted_types=[bool]) self.use_seasonal_model = self._register_single_value_hyperparameter("use_seasonal_model", accepted_types=[bool]) self.use_default_time_features = self._register_single_value_hyperparameter("use_default_time_features", accepted_types=[bool]) self.seed = self._register_single_value_hyperparameter("seed", accepted_types=[int]) class GluonTSSimpleFeedForwardSettings(PredictionAlgorithmSettings): def __init__(self, raw_settings, hyperparameter_search_params): super(GluonTSSimpleFeedForwardSettings, self).__init__(raw_settings, hyperparameter_search_params) self.context_length = self._register_numerical_hyperparameter("context_length") self.distr_output = self._register_categorical_hyperparameter("distr_output") self.batch_normalization = self._register_categorical_hyperparameter("batch_normalization") self.mean_scaling = self._register_categorical_hyperparameter("mean_scaling") self.num_hidden_dimensions = self._register_single_value_hyperparameter("num_hidden_dimensions", accepted_types=[list]) self.full_context = self._register_single_value_hyperparameter("full_context", accepted_types=[bool]) self.num_parallel_samples = self._register_single_value_hyperparameter("num_parallel_samples", accepted_types=[int]) self.batch_size = self._register_single_value_hyperparameter("batch_size", accepted_types=[int]) self.epochs = self._register_single_value_hyperparameter("epochs", accepted_types=[int]) self.auto_num_batches_per_epoch = self._register_single_value_hyperparameter("auto_num_batches_per_epoch", accepted_types=[bool]) self.num_batches_per_epoch = self._register_single_value_hyperparameter("num_batches_per_epoch", accepted_types=[int]) self.seed = self._register_single_value_hyperparameter("seed", accepted_types=[int]) class GluonTSDeepARSettings(PredictionAlgorithmSettings): def __init__(self, raw_settings, hyperparameter_search_params): super(GluonTSDeepARSettings, self).__init__(raw_settings, hyperparameter_search_params) self.context_length = self._register_numerical_hyperparameter("context_length") self.num_layers = self._register_numerical_hyperparameter("num_layers") self.num_cells = self._register_numerical_hyperparameter("num_cells") self.cell_type = self._register_categorical_hyperparameter("cell_type") self.dropoutcell_type = self._register_categorical_hyperparameter("dropoutcell_type") self.dropout_rate = self._register_numerical_hyperparameter("dropout_rate") self.alpha = self._register_numerical_hyperparameter("alpha") self.beta = self._register_numerical_hyperparameter("beta") self.distr_output = self._register_categorical_hyperparameter("distr_output") self.full_context = self._register_single_value_hyperparameter("full_context", accepted_types=[bool]) self.scaling = self._register_single_value_hyperparameter("scaling", accepted_types=[bool]) self.num_parallel_samples = self._register_single_value_hyperparameter("num_parallel_samples", accepted_types=[int]) self.minimum_scale = self._register_single_value_hyperparameter("minimum_scale", accepted_types=[float]) self.batch_size = self._register_single_value_hyperparameter("batch_size", accepted_types=[int]) self.epochs = self._register_single_value_hyperparameter("epochs", accepted_types=[int]) self.auto_num_batches_per_epoch = self._register_single_value_hyperparameter("auto_num_batches_per_epoch", accepted_types=[bool]) self.num_batches_per_epoch = self._register_single_value_hyperparameter("num_batches_per_epoch", accepted_types=[int]) self.seed = self._register_single_value_hyperparameter("seed", accepted_types=[int]) class GluonTSTransformerSettings(PredictionAlgorithmSettings): def __init__(self, raw_settings, hyperparameter_search_params): super(GluonTSTransformerSettings, self).__init__(raw_settings, hyperparameter_search_params) self.context_length = self._register_numerical_hyperparameter("context_length") self.model_dim = self._register_numerical_hyperparameter("model_dim") self.inner_ff_dim_scale = self._register_numerical_hyperparameter("inner_ff_dim_scale") self.num_heads = self._register_numerical_hyperparameter("num_heads") self.dropout_rate = self._register_numerical_hyperparameter("dropout_rate") self.distr_output = self._register_categorical_hyperparameter("distr_output") self.full_context = self._register_single_value_hyperparameter("full_context", accepted_types=[bool]) self.num_parallel_samples = self._register_single_value_hyperparameter("num_parallel_samples", accepted_types=[int]) self.batch_size = self._register_single_value_hyperparameter("batch_size", accepted_types=[int]) self.epochs = self._register_single_value_hyperparameter("epochs", accepted_types=[int]) self.auto_num_batches_per_epoch = self._register_single_value_hyperparameter("auto_num_batches_per_epoch", accepted_types=[bool]) self.num_batches_per_epoch = self._register_single_value_hyperparameter("num_batches_per_epoch", accepted_types=[int]) self.seed = self._register_single_value_hyperparameter("seed", accepted_types=[int]) class GluonTSMQCNNSettings(PredictionAlgorithmSettings): def __init__(self, raw_settings, hyperparameter_search_params): super(GluonTSMQCNNSettings, self).__init__(raw_settings, hyperparameter_search_params) self.context_length = self._register_numerical_hyperparameter("context_length") self.full_context = self._register_single_value_hyperparameter("full_context", accepted_types=[bool]) self.seed = self._register_single_value_hyperparameter("seed", accepted_types=[int]) self.decoder_mlp_dim_seq = self._register_single_value_hyperparameter("decoder_mlp_dim_seq", accepted_types=[list]) self.channels_seq = self._register_single_value_hyperparameter("channels_seq", accepted_types=[list]) self.dilation_seq = self._register_single_value_hyperparameter("dilation_seq", accepted_types=[list]) self.kernel_size_seq = self._register_single_value_hyperparameter("kernel_size_seq", accepted_types=[list]) self.scaling = self._register_single_value_hyperparameter("scaling", accepted_types=[bool]) self.batch_size = self._register_single_value_hyperparameter("batch_size", accepted_types=[int]) self.epochs = self._register_single_value_hyperparameter("epochs", accepted_types=[int]) self.auto_num_batches_per_epoch = self._register_single_value_hyperparameter("auto_num_batches_per_epoch", accepted_types=[bool]) self.num_batches_per_epoch = self._register_single_value_hyperparameter("num_batches_per_epoch", accepted_types=[int]) class PredictionAlgorithmMeta: def __init__(self, algorithm_name, algorithm_settings_class=PredictionAlgorithmSettings): self.algorithm_name = algorithm_name self.algorithm_settings_class = algorithm_settings_class class AbstractTabularPredictionMLTaskSettings(DSSMLTaskSettings): def get_prediction_type(self): return self.mltask_settings['predictionType'] def get_algorithm_settings(self, algorithm_name): """ Gets the training settings for a particular algorithm. This returns a reference to the algorithm's settings, not a copy, so changes made to the returned object will be reflected when saving. This method returns the settings for this algorithm as an PredictionAlgorithmSettings (extended dict). All algorithm dicts have at least an "enabled" property/key in the settings. The "enabled" property/key indicates whether this algorithm will be trained. Other settings are algorithm-dependent and are the various hyperparameters of the algorithm. The precise properties/keys for each algorithm are not all documented. You can print the returned AlgorithmSettings to learn more about the settings of each particular algorithm. Please refer to the documentation for details on available algorithms. :param algorithm_name: Name (in capitals) of the algorithm. :type algorithm_name: str :return: A PredictionAlgorithmSettings (extended dict) for one of the built-in prediction algorithms :rtype: PredictionAlgorithmSettings """ if algorithm_name in self.__class__._algorithm_remap: algorithm_meta = self.__class__._algorithm_remap[algorithm_name] algorithm_name = algorithm_meta.algorithm_name algorithm_settings_class = algorithm_meta.algorithm_settings_class algorithm_settings = self.mltask_settings["modeling"][algorithm_name.lower()] if not isinstance(algorithm_settings, PredictionAlgorithmSettings): raw_hyperparameter_search_params = self.mltask_settings["modeling"]["gridSearchParams"] algorithm_settings = algorithm_settings_class(algorithm_settings, raw_hyperparameter_search_params) # Subsequent calls get the same object self.mltask_settings["modeling"][algorithm_name.lower()] = algorithm_settings return self.mltask_settings["modeling"][algorithm_name.lower()] elif algorithm_name in self._get_custom_algorithm_names(): return self._get_custom_algorithm_settings(algorithm_name) else: raise ValueError("Unknown algorithm: {}".format(algorithm_name)) def get_hyperparameter_search_settings(self): """ Gets the hyperparameter search parameters of the current DSSPredictionMLTaskSettings instance as a HyperparameterSearchSettings object. This object can be used to both get and set properties relevant to hyperparameter search, such as search strategy, cross-validation method, execution limits and parallelism. :return: A HyperparameterSearchSettings :rtype: :class:`HyperparameterSearchSettings` """ return HyperparameterSearchSettings(self.mltask_settings["modeling"]["gridSearchParams"]) @property def split_params(self): """ Gets a handle to modify train/test splitting params. :rtype: :class:`PredictionSplitParamsHandler` """ return self.get_split_params() def get_split_params(self): """ Gets a handle to modify train/test splitting params. :rtype: :class:`PredictionSplitParamsHandler` """ return PredictionSplitParamsHandler(self.mltask_settings) def get_assertions_params(self): """ Retrieves the assertions parameters for this ml task :rtype: :class:`DSSMLAssertionsParams` """ raise NotImplementedError("ML assertions are not available for {}".format(self.__class__.__name__)) class DSSPredictionMLTaskSettings(AbstractTabularPredictionMLTaskSettings): """ For classical prediction ML tasks """ __doc__ = [] _algorithm_remap = { "RANDOM_FOREST_CLASSIFICATION": PredictionAlgorithmMeta("random_forest_classification", RandomForestSettings), "RANDOM_FOREST_REGRESSION": PredictionAlgorithmMeta("random_forest_regression", RandomForestSettings), "EXTRA_TREES": PredictionAlgorithmMeta("extra_trees", RandomForestSettings), "GBT_CLASSIFICATION": PredictionAlgorithmMeta("gbt_classification", GradientBoostedTreesSettings), "GBT_REGRESSION": PredictionAlgorithmMeta("gbt_regression", GradientBoostedTreesSettings), "DECISION_TREE_CLASSIFICATION": PredictionAlgorithmMeta("decision_tree_classification", DecisionTreeSettings), "DECISION_TREE_REGRESSION": PredictionAlgorithmMeta("decision_tree_regression", DecisionTreeSettings), "RIDGE_REGRESSION": PredictionAlgorithmMeta("ridge_regression", RidgeRegressionSettings), "LASSO_REGRESSION": PredictionAlgorithmMeta("lasso_regression", LassoRegressionSettings), "LEASTSQUARE_REGRESSION": PredictionAlgorithmMeta("leastsquare_regression", OLSSettings), "SGD_REGRESSION": PredictionAlgorithmMeta("sgd_regression", SGDSettings), "KNN": PredictionAlgorithmMeta("knn", KNNSettings), "LOGISTIC_REGRESSION": PredictionAlgorithmMeta("logistic_regression", LogitSettings), "NEURAL_NETWORK": PredictionAlgorithmMeta("neural_network", SingleLayerPerceptronSettings), "DEEP_NEURAL_NETWORK_REGRESSION": PredictionAlgorithmMeta("deep_neural_network_regression", DeepNeuralNetworkSettings), "DEEP_NEURAL_NETWORK_CLASSIFICATION": PredictionAlgorithmMeta("deep_neural_network_classification", DeepNeuralNetworkSettings), "SVC_CLASSIFICATION": PredictionAlgorithmMeta("svc_classifier", SVMSettings), "SVM_REGRESSION": PredictionAlgorithmMeta("svm_regression", SVMSettings), "SGD_CLASSIFICATION": PredictionAlgorithmMeta("sgd_classifier", SGDSettings), "LARS": PredictionAlgorithmMeta("lars_params", LARSSettings), "LIGHTGBM_CLASSIFICATION": PredictionAlgorithmMeta("lightgbm_classification", LightGBMSettings), "LIGHTGBM_REGRESSION": PredictionAlgorithmMeta("lightgbm_regression", LightGBMSettings), "XGBOOST_CLASSIFICATION": PredictionAlgorithmMeta("xgboost", XGBoostSettings), "XGBOOST_REGRESSION": PredictionAlgorithmMeta("xgboost", XGBoostSettings), "SPARKLING_DEEP_LEARNING": PredictionAlgorithmMeta("deep_learning_sparkling"), "SPARKLING_GBM": PredictionAlgorithmMeta("gbm_sparkling"), "SPARKLING_RF": PredictionAlgorithmMeta("rf_sparkling"), "SPARKLING_GLM": PredictionAlgorithmMeta("glm_sparkling"), "SPARKLING_NB": PredictionAlgorithmMeta("nb_sparkling"), "MLLIB_LOGISTIC_REGRESSION": PredictionAlgorithmMeta("mllib_logit", MLLibLogitSettings), "MLLIB_NAIVE_BAYES": PredictionAlgorithmMeta("mllib_naive_bayes", MLLibNaiveBayesSettings), "MLLIB_LINEAR_REGRESSION": PredictionAlgorithmMeta("mllib_linreg", MLLibLinearRegressionSettings), "MLLIB_RANDOM_FOREST": PredictionAlgorithmMeta("mllib_rf", MLLibRandomForestSettings), "MLLIB_GBT": PredictionAlgorithmMeta("mllib_gbt", MLLibGBTSettings), "MLLIB_DECISION_TREE": PredictionAlgorithmMeta("mllib_dt", MLLibDecisionTreeSettings), "VERTICA_LINEAR_REGRESSION": PredictionAlgorithmMeta("vertica_linear_regression"), "VERTICA_LOGISTIC_REGRESSION": PredictionAlgorithmMeta("vertica_logistic_regression"), "KERAS_CODE": PredictionAlgorithmMeta("keras") } class PredictionTypes: BINARY = "BINARY_CLASSIFICATION" REGRESSION = "REGRESSION" MULTICLASS = "MULTICLASS" def __init__(self, client, project_key, analysis_id, mltask_id, mltask_settings): super(DSSPredictionMLTaskSettings, self).__init__(client, project_key, analysis_id, mltask_id, mltask_settings) prediction_type = self.get_prediction_type() if prediction_type not in [self.PredictionTypes.BINARY, self.PredictionTypes.REGRESSION, self.PredictionTypes.MULTICLASS]: raise ValueError("Unknown prediction type: {}".format(prediction_type)) self.classification_prediction_types = [self.PredictionTypes.BINARY, self.PredictionTypes.MULTICLASS] def get_all_possible_algorithm_names(self): """ Returns the list of possible algorithm names, i.e. the list of valid identifiers for :meth:`set_algorithm_enabled` and :meth:`get_algorithm_settings` This includes all possible algorithms, regardless of the prediction kind (regression/classification) or engine, so some algorithms may be irrelevant :returns: the list of algorithm names as a list of strings :rtype: list of string """ return super(DSSPredictionMLTaskSettings, self).get_all_possible_algorithm_names() + self._get_plugin_algorithm_names() def _get_plugin_algorithm_names(self): return list(self.mltask_settings["modeling"]["plugin_python"].keys()) def _get_plugin_algorithm_settings(self, algorithm_name): if algorithm_name in self.mltask_settings["modeling"]["plugin_python"]: return self.mltask_settings["modeling"]["plugin_python"][algorithm_name] raise ValueError("Unknown algorithm: {}".format(algorithm_name)) def get_enabled_algorithm_names(self): """ :returns: the list of enabled algorithm names as a list of strings :rtype: list of string """ algo_names = super(DSSPredictionMLTaskSettings, self).get_enabled_algorithm_names() # Hide either "XGBOOST_CLASSIFICATION" or "XGBOOST_REGRESSION" which point to the same key "xgboost" if self.mltask_settings["predictionType"] == "REGRESSION": excluded_names = {"XGBOOST_CLASSIFICATION"} else: excluded_names = {"XGBOOST_REGRESSION"} return [algo_name for algo_name in algo_names if algo_name not in excluded_names] def get_algorithm_settings(self, algorithm_name): if algorithm_name in self._get_plugin_algorithm_names(): return self._get_plugin_algorithm_settings(algorithm_name) else: return super(DSSPredictionMLTaskSettings, self).get_algorithm_settings(algorithm_name) def split_ordered_by(self, feature_name, ascending=True): """ Deprecated. Use split_params.set_time_ordering() """ warnings.warn("split_ordered_by() is deprecated, please use split_params.set_time_ordering() instead", DeprecationWarning) self.split_params.set_time_ordering(feature_name, ascending=ascending) return self def remove_ordered_split(self): """ Deprecated. Use split_params.unset_time_ordering() """ warnings.warn("remove_ordered_split() is deprecated, please use split_params.unset_time_ordering() instead", DeprecationWarning) self.split_params.unset_time_ordering() return self def use_sample_weighting(self, feature_name): """ Deprecated. use set_weighting() """ warnings.warn("use_sample_weighting() is deprecated, please use set_weighting() instead", DeprecationWarning) return self.set_weighting(method='SAMPLE_WEIGHT', feature_name=feature_name, ) def set_weighting(self, method, feature_name=None): """ Sets the method to weight samples. If there was a WEIGHT feature declared previously, it will be set back as an INPUT feature first. :param str method: Method to use. One of NO_WEIGHTING, SAMPLE_WEIGHT (must give a feature name), CLASS_WEIGHT or CLASS_AND_SAMPLE_WEIGHT (must give a feature name) :param str feature_name: Name of the feature to use as sample weight """ # First, if there was a WEIGHT feature, restore it as INPUT for other_feature_name in self.mltask_settings['preprocessing']['per_feature']: if self.mltask_settings['preprocessing']['per_feature'][other_feature_name]['role'] == 'WEIGHT': self.mltask_settings['preprocessing']['per_feature'][other_feature_name]['role'] = 'INPUT' if method == "NO_WEIGHTING": self.mltask_settings['weight']['weightMethod'] = method elif method == "SAMPLE_WEIGHT": if not feature_name in self.mltask_settings["preprocessing"]["per_feature"]: raise ValueError("Feature %s doesn't exist in this ML task, can't use as weight" % feature_name) self.mltask_settings['weight']['weightMethod'] = method self.mltask_settings['weight']['sampleWeightVariable'] = feature_name self.mltask_settings['preprocessing']['per_feature'][feature_name]['role'] = 'WEIGHT' elif method == "CLASS_WEIGHT": if self.get_prediction_type() not in self.classification_prediction_types: raise ValueError("Weighting method: {} not compatible with prediction type: {}, should be in {}".format(method, self.get_prediction_type(), self.classification_prediction_types)) self.mltask_settings['weight']['weightMethod'] = method elif method == "CLASS_AND_SAMPLE_WEIGHT": if self.get_prediction_type() not in self.classification_prediction_types: raise ValueError("Weighting method: {} not compatible with prediction type: {}, should be in {}".format(method, self.get_prediction_type(), self.classification_prediction_types)) if not feature_name in self.mltask_settings["preprocessing"]["per_feature"]: raise ValueError("Feature %s doesn't exist in this ML task, can't use as weight" % feature_name) self.mltask_settings['weight']['weightMethod'] = method self.mltask_settings['weight']['sampleWeightVariable'] = feature_name self.mltask_settings['preprocessing']['per_feature'][feature_name]['role'] = 'WEIGHT' else: raise ValueError("Unknown weighting method: {}".format(method)) return self def remove_sample_weighting(self): """ Deprecated. Use set_weighting(method=\"NO_WEIGHTING\") instead """ warnings.warn("remove_sample_weighting() is deprecated, please use set_weighting(method=\"NO_WEIGHTING\") instead", DeprecationWarning) return self.set_weighting(method="NO_WEIGHTING") def get_assertions_params(self): """ Retrieves the assertions parameters for this ml task :rtype: :class:`DSSMLAssertionsParams` """ return DSSMLAssertionsParams(self.mltask_settings["assertionsParams"]) class DSSClusteringMLTaskSettings(DSSMLTaskSettings): __doc__ = [] _algorithm_remap = { "DBSCAN": "db_scan_clustering", "SPECTRAL": "spectral_clustering", "WARD": "ward_clustering", "KMEANS": "kmeans_clustering", "MINIBATCH_KMEANS": "mini_batch_kmeans_clustering", "GAUSSIAN_MIXTURE": "gmm_clustering", "TWO_STEP": "two_step", "ISOLATION_FOREST": "isolation_forest", "MLLIB_KMEANS": "mllib_kmeans_clustering", "MLLIB_GAUSSIAN_MIXTURE": "mllib_gaussian_mixture_clustering", "H20_KMEANS": "h2o_kmeans" } def get_algorithm_settings(self, algorithm_name): """ Gets the training settings for a particular algorithm. This returns a reference to the algorithm's settings, not a copy, so changes made to the returned object will be reflected when saving. This method returns a dictionary of the settings for this algorithm. All algorithm dicts have at least an "enabled" key in the dictionary. The 'enabled' key indicates whether this algorithm will be trained Other settings are algorithm-dependent and are the various hyperparameters of the algorithm. The precise keys for each algorithm are not all documented. You can print the returned dictionary to learn more about the settings of each particular algorithm Please refer to the documentation for details on available algorithms. :param: algorithm_name: Name of the algorithm (uppercase). :type: algorithm_name: str :return: A dict of the settings for an algorithm :rtype: dict """ if algorithm_name in self.__class__._algorithm_remap: algorithm_name = self.__class__._algorithm_remap[algorithm_name] return self.mltask_settings["modeling"][algorithm_name.lower()] elif algorithm_name in self._get_custom_algorithm_names(): return self._get_custom_algorithm_settings(algorithm_name) else: raise ValueError("Unknown algorithm: {}".format(algorithm_name)) class DSSTimeseriesForecastingMLTaskSettings(AbstractTabularPredictionMLTaskSettings): __doc__ = [] _algorithm_remap = { "TRIVIAL_IDENTITY_TIMESERIES": PredictionAlgorithmMeta("trivial_identity_timeseries"), "SEASONAL_NAIVE": PredictionAlgorithmMeta("seasonal_naive_timeseries", SeasonalNaiveSettings), "AUTO_ARIMA": PredictionAlgorithmMeta("autoarima_timeseries", AutoArimaSettings), "SEASONAL_LOESS": PredictionAlgorithmMeta("seasonal_loess_timeseries", SeasonalLoessSettings), "GLUONTS_NPTS_FORECASTER": PredictionAlgorithmMeta("gluonts_npts_timeseries", GluonTSNPTSForecasterSettings), "GLUONTS_SIMPLE_FEEDFORWARD": PredictionAlgorithmMeta("gluonts_simple_feed_forward_timeseries", GluonTSSimpleFeedForwardSettings), "GLUONTS_DEEPAR": PredictionAlgorithmMeta("gluonts_deepar_timeseries", GluonTSDeepARSettings), "GLUONTS_TRANSFORMER": PredictionAlgorithmMeta("gluonts_transformer_timeseries", GluonTSTransformerSettings), "GLUONTS_MQCNN": PredictionAlgorithmMeta("gluonts_mqcnn_timeseries", GluonTSMQCNNSettings), } _TIME_UNITS = {"MILLISECOND", "SECOND", "MINUTE", "HOUR", "DAY", "BUSINESS_DAY", "WEEK", "MONTH", "QUARTER", "HALF_YEAR", "YEAR"} _INTERPOLATION_METHODS = {"NEAREST", "PREVIOUS", "NEXT", "LINEAR", "QUADRATIC", "CUBIC", "CONSTANT"} _EXTRAPOLATION_METHODS = {"PREVIOUS_NEXT", "NO_EXTRAPOLATION", "CONSTANT", "LINEAR", "QUADRATIC", "CUBIC"} _CATEGORICAL_IMPUTATION_METHODS = {"MOST_COMMON", "NULL", "CONSTANT", "PREVIOUS_NEXT", "PREVIOUS", "NEXT"} _DUPLICATE_TIMESTAMPS_HANDLING_METHODS = {"FAIL_IF_CONFLICTING", "DROP_IF_CONFLICTING", "MEAN_MODE"} class PredictionTypes: TIMESERIES_FORECAST = "TIMESERIES_FORECAST" def __init__(self, client, project_key, analysis_id, mltask_id, mltask_settings): super(DSSTimeseriesForecastingMLTaskSettings, self).__init__(client, project_key, analysis_id, mltask_id, mltask_settings) prediction_type = self.get_prediction_type() if prediction_type != self.PredictionTypes.TIMESERIES_FORECAST: raise ValueError("Unknown prediction type: {}".format(prediction_type)) def get_time_step_params(self): """ Gets the time step parameters for the time series forecasting task. This returns a reference to the time step parameters, not a copy, so changes made to the returned object will be reflected when saving :return: A dict of the time step parameters :rtype: dict """ return self.mltask_settings["timestepParams"] def set_time_step(self, time_unit=None, n_time_units=None, end_of_week_day=None, reguess=True, update_algorithm_settings=True): """ Sets the time step parameters for the time series forecasting task. :param time_unit: time unit for forecasting step. Valid values are: MILLISECOND, SECOND, MINUTE, HOUR, DAY, BUSINESS_DAY, WEEK, MONTH, QUARTER, HALF_YEAR, YEAR :type time_unit: str :param n_time_units: number of time units within a time step :type n_time_units: int :param end_of_week_day: only useful for the WEEK time unit. Valid values are: 1 (Sunday), 2 (Monday), ..., 7 (Saturday) :type end_of_week_day: int :param reguess: Defaults to true. Whether to reguess the ML task settings after changing the time step params :type reguess: bool :param update_algorithm_settings: Defaults to true. Whether the algorithm settings should be reguessed after changing time step parameters. :type update_algorithm_settings: bool :return: """ time_step_params = self.get_time_step_params() if time_unit is not None: assert time_unit in self._TIME_UNITS, "Invalid time unit, should be in {}".format(self._TIME_UNITS) time_step_params["timeunit"] = time_unit if n_time_units is not None: assert isinstance(n_time_units, int), "n_time_units should be an int" assert n_time_units > 0, "n_time_units should be positive" time_step_params["numberOfTimeunits"] = n_time_units if end_of_week_day is not None: assert end_of_week_day in {1, 2, 3, 4, 5, 6, 7}, "Invalid end_of_week_day, should be in [1, 2, 3, 4, 5, 6, 7]" if time_step_params["timeunit"] != "WEEK": logger.warning("Changing end of week day, but time unit is not WEEK") time_step_params["endOfWeekDay"] = end_of_week_day if reguess: logger.info("Reguessing ML task settings after changing time step params") self.client._perform_empty( "POST", "/projects/{project_key}/models/lab/{analysis_id}/{mltask_id}/reguess-with-forecasting-params".format( project_key=self.project_key, analysis_id=self.analysis_id, mltask_id=self.mltask_id ), body={"timestepParams": time_step_params, "updateAlgorithmSettings": update_algorithm_settings}, ) def get_resampling_params(self): """ Gets the time series resampling parameters for the time series forecasting task. This returns a reference to the time series resampling parameters, not a copy, so changes made to the returned object will be reflected when saving :return: A dict of the resampling parameters :rtype: dict """ return self.mltask_settings["preprocessing"]["timeseriesSampling"] def set_numerical_interpolation(self, method=None, constant=None): """ Sets the time series resampling numerical interpolation parameters :param method: Interpolation method. Valid values are: NEAREST, PREVIOUS, NEXT, LINEAR, QUADRATIC, CUBIC, CONSTANT :type method: str :param constant: Value for the CONSTANT interpolation method :type constant: float :return: """ resampling_params = self.get_resampling_params() if method is not None: assert method in self._INTERPOLATION_METHODS, "Invalid interpolation method, should be in {}".format(self._INTERPOLATION_METHODS) resampling_params["numericalInterpolateMethod"] = method if constant is not None: assert isinstance(constant, float) or isinstance(constant, int), "Constant value should be a number" if resampling_params["numericalInterpolateMethod"] != "CONSTANT": logger.warning("Changing numerical interpolation constant value, but interpolation method is not CONSTANT") resampling_params["numericalInterpolateConstantValue"] = constant def set_numerical_extrapolation(self, method=None, constant=None): """ Sets the time series resampling numerical extrapolation parameters :param method: Extrapolation method. Valid values are: PREVIOUS_NEXT, NO_EXTRAPOLATION, CONSTANT, LINEAR, QUADRATIC, CUBIC :type method: str :param constant: Value for the CONSTANT extrapolation method :type constant: float :return: """ resampling_params = self.get_resampling_params() if method is not None: assert method in self._EXTRAPOLATION_METHODS, "Invalid extrapolation method, should be in {}".format(self._EXTRAPOLATION_METHODS) resampling_params["numericalExtrapolateMethod"] = method if constant is not None: assert isinstance(constant, float) or isinstance(constant, int), "Constant value should be a number" if resampling_params["numericalExtrapolateMethod"] != "CONSTANT": logger.warning("Changing numerical extrapolation constant value, but extrapolation method is not CONSTANT") resampling_params["numericalExtrapolateConstantValue"] = constant def set_categorical_imputation(self, method=None, constant=None): """ Sets the time series resampling categorical imputation parameters :param method: Imputation method. Valid values are: MOST_COMMON, NULL, CONSTANT, PREVIOUS_NEXT, PREVIOUS, NEXT :type method: str :param constant: Value for the CONSTANT imputation method :type constant: str :return: """ resampling_params = self.get_resampling_params() if method is not None: assert method in self._CATEGORICAL_IMPUTATION_METHODS, "Invalid imputation method, should be in {}".format(self._CATEGORICAL_IMPUTATION_METHODS) resampling_params["categoricalImputeMethod"] = method if constant is not None: assert isinstance(constant, str), "Constant value should be a string" if resampling_params["categoricalImputeMethod"] != "CONSTANT": logger.warning("Changing categorical imputation constant value, but imputation method is not CONSTANT") resampling_params["categoricalConstantValue"] = constant def set_duplicate_timestamp_handling(self, method): """ Sets the time series duplicate timestamp handling :param method: Duplicate timestamp handling method. Valid values are: FAIL_IF_CONFLICTING, DROP_IF_CONFLICTING, MEAN_MODE :type method: str """ resampling_params = self.get_resampling_params() assert method in self._DUPLICATE_TIMESTAMPS_HANDLING_METHODS, "Invalid duplicate timestamp handling method, should be in {}".format( self._DUPLICATE_TIMESTAMPS_HANDLING_METHODS ) resampling_params["duplicateTimestampsHandlingMethod"] = method @property def forecast_horizon(self): """ :return: Number of time steps to be forecast :rtype: int """ return self.mltask_settings["predictionLength"] def set_forecast_horizon(self, forecast_horizon, reguess=True, update_algorithm_settings=True): """ :param forecast_horizon: Number of time steps to be forecast :type forecast_horizon: int :param reguess: Defaults to true. Whether to reguess the ML task settings after changing the forecast horizon :type reguess: bool :param update_algorithm_settings: Defaults to true. Whether the algorithm settings should be reguessed after the forecast horizon. :type update_algorithm_settings: bool """ assert isinstance(forecast_horizon, int) self.mltask_settings["predictionLength"] = forecast_horizon if reguess: logger.info("Reguessing ML task settings after changing forecast horizon") self.client._perform_empty( "POST", "/projects/{project_key}/models/lab/{analysis_id}/{mltask_id}/reguess-with-forecasting-params".format( project_key=self.project_key, analysis_id=self.analysis_id, mltask_id=self.mltask_id ), body={"forecastHorizon": forecast_horizon, "updateAlgorithmSettings": update_algorithm_settings}, ) @property def evaluation_gap(self): """ :return: Number of skipped time steps for evaluation :rtype: int """ return self.mltask_settings["evaluationParams"]["gapSize"] @evaluation_gap.setter def evaluation_gap(self, value): """ :param value: Number of skipped time steps for evaluation :type value: int """ assert isinstance(value, int) assert value < self.forecast_horizon, "Gap must be smaller than the forecast horizon" self.mltask_settings["evaluationParams"]["gapSize"] = value self.mltask_settings["evaluationParams"]["testSize"] = self.forecast_horizon - value @property def time_variable(self): """ :return: Feature used as time variable (read-only) :rtype: str """ return self.mltask_settings["timeVariable"] @property def timeseries_identifiers(self): """ :return: Features used as time series identifiers (read-only copy) :rtype: list """ return list(self.mltask_settings["timeseriesIdentifiers"]) @property def quantiles_to_forecast(self): """ :return: List of quantiles to forecast :rtype: list """ return self.mltask_settings["quantilesToForecast"] @quantiles_to_forecast.setter def quantiles_to_forecast(self, values): """ :param values: List of quantiles to forecast: - 0.5 should always be included - Values should be between 0.0001 and 0.9999 - Values should be defined up to 4 digits after decimal point :type values: list """ assert isinstance(values, list) self.mltask_settings["quantilesToForecast"] = sorted(set(values)) class DSSTrainedModelDetails(object): def __init__(self, details, snippet, saved_model=None, saved_model_version=None, mltask=None, mltask_model_id=None): self.details = details self.snippet = snippet self.saved_model = saved_model self.saved_model_version = saved_model_version self.mltask = mltask self.mltask_model_id = mltask_model_id def get_raw(self): """ Gets the raw dictionary of trained model details """ return self.details @property def full_id(self): return self.details["fullModelId"] def get_raw_snippet(self): """ Gets the raw dictionary of trained model snippet. The snippet is a lighter version than the details. """ return self.snippet def get_train_info(self): """ Returns various information about the train process (size of the train set, quick description, timing information) :rtype: dict """ return self.details["trainInfo"] def get_user_meta(self): """ Gets the user-accessible metadata (name, description, cluster labels, classification threshold) Returns the original object, not a copy. Changes to the returned object are persisted to DSS by calling :meth:`save_user_meta` """ return self.details["userMeta"] def save_user_meta(self): um = self.details["userMeta"] if self.mltask is not None: self.mltask.client._perform_empty( "PUT", "/projects/%s/models/lab/%s/%s/models/%s/user-meta" % (self.mltask.project_key, self.mltask.analysis_id, self.mltask.mltask_id, self.mltask_model_id), body = um) else: self.saved_model.client._perform_empty( "PUT", "/projects/%s/savedmodels/%s/versions/%s/user-meta" % (self.saved_model.project_key, self.saved_model.sm_id, self.saved_model_version), body = um) def get_origin_analysis_trained_model(self): """ Fetch details about the model in an analysis, this model has been exported from. Returns None if the deployed trained model does not have an origin analysis trained model. :rtype: DSSTrainedModelDetails | None """ if self.saved_model is None: return self else: fmi = self.get_raw().get("smOrigin", {}).get("fullModelId") if fmi is not None: origin_ml_task = DSSMLTask.from_full_model_id(self.saved_model.client, fmi, project_key=self.saved_model.project_key) return origin_ml_task.get_trained_model_details(fmi) def get_diagnostics(self): """ Retrieves diagnostics computed for this trained model :returns: list of diagnostics :rtype: list of type `dataikuapi.dss.ml.DSSMLDiagnostic` """ diagnostics = self.details.get("mlDiagnostics", {}) return [DSSMLDiagnostic(d) for d in diagnostics.get("diagnostics", [])] def generate_documentation(self, folder_id=None, path=None): """ Start the model document generation from a template docx file in a managed folder, or from the default template if no folder id and path are specified. :param folder_id: (optional) the id of the managed folder :param path: (optional) the path to the file from the root of the folder :return: A :class:`~dataikuapi.dss.future.DSSFuture` representing the model document generation process """ if bool(folder_id) != bool(path): raise ValueError("Both folder id and path arguments are required to use a template from folder. Use without argument to generate the model documentation using the default template") template_mode_url = "default-template" if folder_id is None and path is None else "template-in-folder" if self.mltask is not None: f = self.mltask.client._perform_json( "POST", "/projects/%s/models/lab/%s/%s/models/%s/generate-documentation-from-%s" % (self.mltask.project_key, self.mltask.analysis_id, self.mltask.mltask_id, self.mltask_model_id, template_mode_url), params={"folderId": folder_id, "path": path}) return DSSFuture(self.mltask.client, f["jobId"]) else: f = self.saved_model.client._perform_json( "POST", "/projects/%s/savedmodels/%s/versions/%s/generate-documentation-from-%s" % (self.saved_model.project_key, self.saved_model.sm_id, self.saved_model_version, template_mode_url), params={"folderId": folder_id, "path": path}) return DSSFuture(self.saved_model.client, job_id=f["jobId"]) def generate_documentation_from_custom_template(self, fp): """ Start the model document generation from a docx template (as a file object). :param object fp: A file-like object pointing to a template docx file :return: A :class:`~dataikuapi.dss.future.DSSFuture` representing the model document generation process """ files = {'file': fp} if self.mltask is not None: f = self.mltask.client._perform_json( "POST", "/projects/%s/models/lab/%s/%s/models/%s/generate-documentation-from-custom-template" % (self.mltask.project_key, self.mltask.analysis_id, self.mltask.mltask_id, self.mltask_model_id), files=files) return DSSFuture(self.mltask.client, f["jobId"]) else: f = self.saved_model.client._perform_json( "POST", "/projects/%s/savedmodels/%s/versions/%s/generate-documentation-from-custom-template" % (self.saved_model.project_key, self.saved_model.sm_id, self.saved_model_version), files=files) return DSSFuture(self.saved_model.client, job_id=f["jobId"]) def download_documentation_stream(self, export_id): """ Download a model documentation, as a binary stream. Warning: this stream will monopolize the DSSClient until closed. :param export_id: the id of the generated model documentation returned as the result of the future :return: A :class:`~dataikuapi.dss.future.DSSFuture` representing the model document generation process """ if self.mltask is not None: return self.mltask.client._perform_raw( "GET", "/projects/%s/models/lab/documentations/%s" % (self.mltask.project_key, export_id)) else: return self.saved_model.client._perform_raw( "GET", "/projects/%s/savedmodels/documentations/%s" % (self.saved_model.project_key, export_id)) def download_documentation_to_file(self, export_id, path): """ Download a model documentation into the given output file. :param export_id: the id of the generated model documentation returned as the result of the future :param path: the path where to download the model documentation :return: None """ stream = self.download_documentation_stream(export_id) _write_response_content_to_file(stream, path) class DSSMLDiagnostic(object): """ Object that represents a computed Diagnostic on a trained model Do not create this object directly, use :meth:`DSSTrainedModelDetails.get_diagnostics()` instead """ def __init__(self, data): self._internal_dict = data def get_raw(self): """ Gets the raw dictionary of the diagnostic :rtype: dict """ return self._internal_dict def get_type(self): """ Returns the base Diagnostic type :rtype: str """ return self._internal_dict["type"] def get_type_pretty(self): """ Returns the Diagnostic type as displayed in the UI :rtype: str """ return self._internal_dict["displayableType"] def get_message(self): """ Returns the message as displayed in the UI :rtype: str """ return self._internal_dict["message"] def __repr__(self): return "{cls}(type={type}, message={msg})".format(cls=self.__class__.__name__, type=self._internal_dict["type"], msg=self._internal_dict["message"]) class DSSMLAssertionsParams(object): """ Object that represents parameters for all assertions of a ml task Do not create this object directly, use :meth:`DSSPredictionMLTaskSettings.get_assertions_params` instead """ def __init__(self, data): self._internal_dict = data def __repr__(self): return u"{}({})".format(self.__class__.__name__, self.get_raw()) def get_raw(self): """ Gets the raw dictionary of the assertions parameters :rtype: dict """ return self._internal_dict def get_assertion(self, assertion_name): """ Gets a :class:`DSSMLAssertionParams` representing the parameters of the assertion with the provided name (or None if no assertion has that name) :param str assertion_name: Name of the assertion :rtype: :class:`DSSMLAssertionParams` or None if no assertion has that name """ for assertion_dict in self._internal_dict["assertions"]: if assertion_dict["name"] == assertion_name: return DSSMLAssertionParams(assertion_dict) return None def get_assertions_names(self): """ Gets the list of all assertions' names :return: A list of all assertions' names :rtype: list """ return [assertion_dict["name"] for assertion_dict in self._internal_dict["assertions"]] def add_assertion(self, assertion_params): """ Adds parameters of an assertion to the assertions parameters of the ml task. :param object assertion_params: A :class:`DSSMLAssertionParams` representing parameters of the assertion """ if not isinstance(assertion_params, DSSMLAssertionParams): raise ValueError('Wrong type for assertion parameters: {}'.format(type(assertion_params))) self._internal_dict["assertions"].append(assertion_params.get_raw()) def delete_assertion(self, assertion_name): """ Deletes the assertion parameters of the assertion with the provided name from the assertions parameters of the ml task. Raises a ValueError if no assertion with the provided name was found :param str assertion_name: Name of the assertion """ for idx, assertion_dict in enumerate(self._internal_dict["assertions"]): if assertion_dict["name"] == assertion_name: del self._internal_dict["assertions"][idx] return raise ValueError('No assertion found with name: {}'.format(assertion_name)) class DSSMLAssertionParams(object): """ Object that represents parameters for one assertion Do not create this object directly, use :meth:`DSSMLAssertionsParams.get_assertion` or :meth:`from_params` instead """ def __init__(self, data): self._internal_dict = data def __repr__(self): return u"{}({})".format(self.__class__.__name__, self.get_raw()) @staticmethod def from_params(name, a_filter, condition): """ Creates assertion parameters from name, filter and condition :param str name: Name of the assertion :param dict a_filter: A dict representing the filter to select assertion population. You can use a :class:`~dataikuapi.dss.utils.DSSFilterBuilder` to build the settings of the filter :param object condition: A :class:`DSSMLAssertionCondition` for the assertion to be successful :rtype: :class:`DSSMLAssertionParams` """ assertion_params = DSSMLAssertionParams({}) assertion_params.name = name assertion_params.filter = a_filter assertion_params.condition = condition return assertion_params def get_raw(self): """ Gets the raw dictionary of the assertion parameters :rtype: dict """ return self._internal_dict @property def name(self): """ Returns the assertion name :rtype: str """ return self._internal_dict["name"] @name.setter def name(self, name): self._internal_dict["name"] = name @property def filter(self): """ Returns the assertion filter :rtype: dict """ return self._internal_dict["filter"] @filter.setter def filter(self, a_filter): self._internal_dict["filter"] = a_filter @property def condition(self): """ Returns the assertion condition :rtype: :class:`DSSMLAssertionCondition` """ return DSSMLAssertionCondition(self._internal_dict["assertionCondition"]) @condition.setter def condition(self, condition): if not isinstance(condition, DSSMLAssertionCondition): raise ValueError('Wrong type for assertion condition: {}'.format(type(condition))) self._internal_dict["assertionCondition"] = condition.get_raw() class DSSMLAssertionCondition(object): """ Object that represents an assertion condition Do not create this object directly, use :meth:`DSSMLAssertionParams.condition`, :meth:`DSSMLAssertionCondition.from_expected_class` or :meth:`DSSMLAssertionCondition.from_expected_range` instead """ def __init__(self, data): self._internal_dict = data def __repr__(self): return u"{}({})".format(self.__class__.__name__, self.get_raw()) @staticmethod def from_expected_class(expected_valid_ratio, expected_class): """ Creates an assertion condition from the expected valid ratio and class :param float expected_valid_ratio: Assertion passes if this ratio of rows predicted as expected_class is attained :param str expected_class: Assertion passes if the ratio of rows predicted as expected_class is attained :rtype: :class:`DSSMLAssertionCondition` """ assertion_condition = DSSMLAssertionCondition({}) assertion_condition.expected_valid_ratio = expected_valid_ratio assertion_condition.expected_class = expected_class return assertion_condition @staticmethod def from_expected_range(expected_valid_ratio, expected_min, expected_max): """ Creates an assertion condition from expected valid ratio and range. The expected range is the interval between expected_min and expected_max (included) for the predictions in which the rows will be considered valid. :param float expected_valid_ratio: Assertion passes if this ratio of rows predicted between expected_min and expected_max (included) is attained :param float expected_min: Min value of the expected range :param float expected_max: Max value of the expected range :rtype: :class:`DSSMLAssertionCondition` """ assertion_condition = DSSMLAssertionCondition({}) assertion_condition.expected_valid_ratio = expected_valid_ratio assertion_condition.expected_min = expected_min assertion_condition.expected_max = expected_max return assertion_condition def get_raw(self): """ Gets the raw dictionary of the condition :rtype: dict """ return self._internal_dict @property def expected_class(self): """ Returns the expected class or None if it is not defined. Assertion passes if the expected_valid_ratio of rows predicted as expected_class is attained. :rtype: str """ if "expectedClass" in self._internal_dict: return self._internal_dict["expectedClass"] else: return None @expected_class.setter def expected_class(self, expected_class): self._internal_dict["expectedClass"] = expected_class @property def expected_valid_ratio(self): """ Returns the ratio of valid rows to exceed for the assertion to pass. A row is considered valid if the prediction is equal to the expected class for classification or in the expected range for regression. :rtype: str """ return self._internal_dict["expectedValidRatio"] @expected_valid_ratio.setter def expected_valid_ratio(self, expected_valid_ratio): self._internal_dict["expectedValidRatio"] = expected_valid_ratio @property def expected_min(self): """ Returns the min (included) of the expected range or None if it is not defined. Assertion passes if the expected_valid_ratio of rows predicted between expected_min and expected_max (included) is attained. :rtype: float """ if "expectedMinValue" in self._internal_dict: return self._internal_dict["expectedMinValue"] else: return None @expected_min.setter def expected_min(self, expected_min): self._internal_dict["expectedMinValue"] = expected_min @property def expected_max(self): """ Returns the max (included) of the expected range or None if it is not defined. Assertion passes if the expected_valid_ratio of rows predicted between expected_min and expected_max (included) is attained. :rtype: float """ if "expectedMaxValue" in self._internal_dict: return self._internal_dict["expectedMaxValue"] else: return None @expected_max.setter def expected_max(self, expected_max): self._internal_dict["expectedMaxValue"] = expected_max class DSSMLAssertionsMetrics(object): """ Object that represents the assertions metrics for all assertions on a trained model Do not create this object directly, use :meth:`DSSTrainedPredictionModelDetails.get_assertions_metrics` instead """ def __init__(self, data): self._internal_dict = data def __repr__(self): return u"{}({})".format(self.__class__.__name__, self.get_raw()) def get_raw(self): """ Gets the raw dictionary of the assertions metrics :rtype: dict """ return self._internal_dict def get_metrics(self, assertion_name): """ Retrieves the metrics computed for this trained model for the assertion with the provided name (or None if no assertion with that name exists) :param str assertion_name: Name of the assertion :returns: an object representing assertion metrics or None if no assertion with that name exists :rtype: :class:`DSSMLAssertionMetrics` """ for assertion_metrics_dict in self._internal_dict["perAssertion"]: if assertion_name == assertion_metrics_dict["name"]: return DSSMLAssertionMetrics(assertion_metrics_dict) return None @property def passing_assertions_ratio(self): """ Returns the ratio of passing assertions :rtype: float """ return self._internal_dict['passingAssertionsRatio'] class DSSMLAssertionMetrics(object): """ Object that represents the result of an assertion on a trained model Do not create this object directly, use :meth:`DSSMLAssertionMetrics.get_metrics` instead """ def __init__(self, data): self._internal_dict = data def __repr__(self): return u"{}(name='{}', result={}, valid_ratio={}, nb_matching_rows={}," \ u" nb_dropped_rows={})".format(self.__class__.__name__, self.name, self.result, self.valid_ratio, self.nb_matching_rows, self.nb_dropped_rows) def get_raw(self): """ Gets the raw dictionary of metrics of one assertion :rtype: dict """ return self._internal_dict @property def name(self): """ Returns the assertion name :rtype: str """ return self._internal_dict["name"] @property def result(self): """ Returns whether the assertion passes :rtype: bool """ return self._internal_dict["result"] @property def valid_ratio(self): """ Returns the ratio of rows in the assertion population with prediction equals to the expected class for classification or in the expected range for regression :rtype: float """ return self._internal_dict["validRatio"] @property def nb_matching_rows(self): """ Returns the number of rows matching filter :rtype: int """ return self._internal_dict["nbMatchingRows"] @property def nb_dropped_rows(self): """ Returns the number of rows matching filter and dropped by the model's preprocessing :rtype: int """ return self._internal_dict["nbDroppedRows"] class DSSTreeNode(object): def __init__(self, tree, i): self.tree = tree self.i = i def get_left_child(self): """Gets a :class:`dataikuapi.dss.ml.DSSTreeNode` representing the left side of the tree node (or None)""" left = self.tree.tree['leftChild'][self.i] if left < 0: return None else: return DSSTreeNode(self.tree, left) def get_right_child(self): """Gets a :class:`dataikuapi.dss.ml.DSSTreeNode` representing the right side of the tree node (or None)""" left = self.tree.tree['rightChild'][self.i] if left < 0: return None else: return DSSTreeNode(self.tree, left) def get_split_info(self): """Gets the information on the split, as a dict""" info = {} features = self.tree.tree.get("feature", None) probas = self.tree.tree.get("probas", None) leftCategories = self.tree.tree.get("leftCategories", None) impurities = self.tree.tree.get("impurity", None) predicts = self.tree.tree.get("predict", None) thresholds = self.tree.tree.get("threshold", None) nSamples = self.tree.tree.get("nSamples", None) info['feature'] = self.tree.feature_names[features[self.i]] if features is not None else None info['probas'] = probas[self.i] if probas is not None else None info['leftCategories'] = leftCategories[self.i] if leftCategories is not None else None info['impurity'] = impurities[self.i] if impurities is not None else None info['predict'] = predicts[self.i] if predicts is not None else None info['nSamples'] = nSamples[self.i] if nSamples is not None else None info['threshold'] = thresholds[self.i] if thresholds is not None else None return info class DSSTree(object): def __init__(self, tree, feature_names): self.tree = tree self.feature_names = feature_names def get_raw(self): """Gets the raw tree data structure""" return self.tree def get_root(self): """Gets a :class:`dataikuapi.dss.ml.DSSTreeNode` representing the root of the tree""" return DSSTreeNode(self, 0) class DSSTreeSet(object): def __init__(self, trees): self.trees = trees def get_raw(self): """Gets the raw trees data structure""" return self.trees def get_feature_names(self): """Gets the list of feature names (after dummification) """ return self.trees["featureNames"] def get_trees(self): """Gets the list of trees as :class:`dataikuapi.dss.ml.DSSTree` """ return [DSSTree(t, self.trees["featureNames"]) for t in self.trees["trees"]] class DSSCoefficientPaths(object): def __init__(self, paths): self.paths = paths def get_raw(self): """Gets the raw paths data structure""" return self.paths def get_feature_names(self): """Get the feature names (after dummification)""" return self.paths['features'] def get_coefficient_path(self, feature, class_index=0): """Get the path of the feature""" i = self.paths['features'].index(feature) if i >= 0 and i < len(self.paths['path'][0][class_index]): n = len(self.paths['path']) return [self.paths['path'][j][class_index][i] for j in range(0, n)] else: return None class DSSScatterPlots(object): def __init__(self, scatters): self.scatters = scatters def get_raw(self): """Gets the raw scatters data structure""" return self.scatters def get_feature_names(self): """Get the feature names (after dummification)""" feature_names = [] for k in self.scatters['features']: feature_names.append(k) return feature_names def get_scatter_plot(self, feature_x, feature_y): """Get the scatter plot between feature_x and feature_y""" ret = {'cluster':self.scatters['cluster'], 'x':self.scatters['features'].get(feature_x, None), 'y':self.scatters['features'].get(feature_x, None)} return ret class DSSTrainedPredictionModelDetails(DSSTrainedModelDetails): """ Object to read details of a trained prediction model Do not create this object directly, use :meth:`DSSMLTask.get_trained_model_details()` instead """ def __init__(self, details, snippet, saved_model=None, saved_model_version=None, mltask=None, mltask_model_id=None): super(DSSTrainedPredictionModelDetails, self).__init__(details, snippet, saved_model, saved_model_version, mltask, mltask_model_id) def get_roc_curve_data(self): roc = self.details.get("perf", {}).get("rocVizData",{}) if roc is None: raise ValueError("This model does not have ROC visualization data") return roc def get_performance_metrics(self): """ Returns all performance metrics for this model. For binary classification model, this includes both "threshold-independent" metrics like AUC and "threshold-dependent" metrics like precision. Threshold-dependent metrics are returned at the threshold value that was found to be optimal during training. To get access to the per-threshold values, use the following: .. code-block:: python # Returns a list of tested threshold values details.get_performance()["perCutData"]["cut"] # Returns a list of F1 scores at the tested threshold values details.get_performance()["perCutData"]["f1"] # Both lists have the same length If K-Fold cross-test was used, most metrics will have a "std" variant, which is the standard deviation accross the K cross-tested folds. For example, "auc" will be accompanied with "aucstd" :returns: a dict of performance metrics values :rtype: dict """ import copy clean_snippet = copy.deepcopy(self.snippet) for x in ["gridsearchData", "trainDate", "topImportance", "backendType", "userMeta", "sessionDate", "trainInfo", "fullModelId", "gridLength", "algorithm", "sessionId"]: if x in clean_snippet: del clean_snippet[x] return clean_snippet def get_assertions_metrics(self): """ Retrieves assertions metrics computed for this trained model :returns: an object representing assertion metrics :rtype: :class:`DSSMLAssertionsMetrics` """ return DSSMLAssertionsMetrics(self.snippet["assertionsMetrics"]) def get_hyperparameter_search_points(self): """ Gets the list of points in the hyperparameter search space that have been tested. Returns a list of dict. Each entry in the list represents a point. For each point, the dict contains at least: - "score": the average value of the optimization metric over all the folds at this point - "params": a dict of the parameters at this point. This dict has the same structure as the params of the best parameters """ if not "gridCells" in self.details["iperf"]: raise ValueError("No hyperparameter search result, maybe this model did not perform hyperparameter optimization") return self.details["iperf"]["gridCells"] def get_preprocessing_settings(self): """ Gets the preprocessing settings that were used to train this model :rtype: dict """ return self.details["preprocessing"] def get_modeling_settings(self): """ Gets the modeling (algorithms) settings that were used to train this model. Note: the structure of this dict is not the same as the modeling params on the ML Task (which may contain several algorithm) :rtype: dict """ return self.details["modeling"] def get_actual_modeling_params(self): """ Gets the actual / resolved parameters that were used to train this model, post hyperparameter optimization. :return: A dictionary, which contains at least a "resolved" key, which is a dict containing the post-optimization parameters :rtype: dict """ return self.details["actualParams"] def get_trees(self): """ Gets the trees in the model (for tree-based models) :return: a DSSTreeSet object to interact with the trees :rtype: :class:`dataikuapi.dss.ml.DSSTreeSet` """ data = self.mltask.client._perform_json( "GET", "/projects/%s/models/lab/%s/%s/models/%s/trees" % (self.mltask.project_key, self.mltask.analysis_id, self.mltask.mltask_id, self.mltask_model_id)) if data is None: raise ValueError("This model has no tree data") return DSSTreeSet(data) def get_coefficient_paths(self): """ Gets the coefficient paths for Lasso models :return: a DSSCoefficientPaths object to interact with the coefficient paths :rtype: :class:`dataikuapi.dss.ml.DSSCoefficientPaths` """ data = self.mltask.client._perform_json( "GET", "/projects/%s/models/lab/%s/%s/models/%s/coef-paths" % (self.mltask.project_key, self.mltask.analysis_id, self.mltask.mltask_id, self.mltask_model_id)) if data is None: raise ValueError("This model has no coefficient paths") return DSSCoefficientPaths(data) ## Model export def get_scoring_jar_stream(self, model_class="model.Model", include_libs=False): """ Get a scoring jar for this trained model, provided that you have the license to do so and that the model is compatible with optimized scoring. You need to close the stream after download. Failure to do so will result in the DSSClient becoming unusable. :param str model_class: fully-qualified class name, e.g. "com.company.project.Model" :param bool include_libs: if True, also packs the required dependencies; if False, runtime will require the scoring libs given by :func:`DSSClient.scoring_libs` :returns: a jar file, as a stream :rtype: file-like """ include_libs = "true" if include_libs else "false" if self.mltask is not None: return self.mltask.client._perform_raw( "GET", "/projects/%s/models/lab/%s/%s/models/%s/scoring-jar?fullClassName=%s&includeLibs=%s" % (self.mltask.project_key, self.mltask.analysis_id, self.mltask.mltask_id, self.mltask_model_id, model_class, include_libs)) else: return self.saved_model.client._perform_raw( "GET", "/projects/%s/savedmodels/%s/versions/%s/scoring-jar?fullClassName=%s&includeLibs=%s" % (self.saved_model.project_key, self.saved_model.sm_id, self.saved_model_version, model_class, include_libs)) def get_scoring_pmml_stream(self): """ Get a scoring PMML for this trained model, provided that you have the license to do so and that the model is compatible with PMML scoring You need to close the stream after download. Failure to do so will result in the DSSClient becoming unusable. :returns: a PMML file, as a stream :rtype: file-like """ if self.mltask is not None: return self.mltask.client._perform_raw( "GET", "/projects/%s/models/lab/%s/%s/models/%s/scoring-pmml" % (self.mltask.project_key, self.mltask.analysis_id, self.mltask.mltask_id, self.mltask_model_id)) else: return self.saved_model.client._perform_raw( "GET", "/projects/%s/savedmodels/%s/versions/%s/scoring-pmml" % (self.saved_model.project_key, self.saved_model.sm_id, self.saved_model_version)) def get_scoring_python_stream(self): """ Download the zip containing data to use for this trained model, provided that you have the license to do so and that the model is compatible with Python scoring. You need to close the stream after download. Failure to do so will result in the DSSClient becoming unusable. :returns: an archive file, as a stream :rtype: file-like """ if self.mltask is not None: return self.mltask.client._perform_raw( "GET", "/projects/%s/models/lab/%s/%s/models/%s/scoring-python?mlflowExport=false" % (self.mltask.project_key, self.mltask.analysis_id, self.mltask.mltask_id, self.mltask_model_id)) else: return self.saved_model.client._perform_raw( "GET", "/projects/%s/savedmodels/%s/versions/%s/scoring-python?mlflowExport=false" % (self.saved_model.project_key, self.saved_model.sm_id, self.saved_model_version)) def get_scoring_python(self, filename): """ Download the zip containing data to use Python scoring for this trained model in filename, provided that you have the license to do so and that the model is compatible with Python scoring. :param str filename: filename of the resulting downloaded file """ with open(filename, "wb") as f: f.write(self.get_scoring_python_stream().content) def get_scoring_mlflow_stream(self): """ Download the zip containing this trained model using MLflow Model format, provided that you have the license to do so and that the model is compatible with MLflow scoring. You need to close the stream after download. Failure to do so will result in the DSSClient becoming unusable. :returns: an archive file, as a stream :rtype: file-like """ if self.mltask is not None: return self.mltask.client._perform_raw( "GET", "/projects/%s/models/lab/%s/%s/models/%s/scoring-python?mlflowExport=true" % (self.mltask.project_key, self.mltask.analysis_id, self.mltask.mltask_id, self.mltask_model_id)) else: return self.saved_model.client._perform_raw( "GET", "/projects/%s/savedmodels/%s/versions/%s/scoring-python?mlflowExport=true" % (self.saved_model.project_key, self.saved_model.sm_id, self.saved_model_version)) def get_scoring_mlflow(self, filename): """ Download the zip containing data for this trained model, using MLflow Model format, provided that you have the license to do so and that the model is compatible with MLflow scoring :param str filename: filename to the resulting MLflow Model zip """ with open(filename, "wb") as f: f.write(self.get_scoring_mlflow_stream().content) ## Post-train computations def compute_subpopulation_analyses(self, split_by, wait=True, sample_size=1000, random_state=1337, n_jobs=1, debug_mode=False): """ Launch computation of Subpopulation analyses for this trained model. :param list|str split_by: column(s) on which subpopulation analyses are to be computed (one analysis per column) :param bool wait: if True, the call blocks until the computation is finished and returns the results directly :param int sample_size: number of records of the dataset to use for the computation :param int random_state: random state to use to build sample, for reproducibility :param int n_jobs: number of cores used for parallel training. (-1 means 'all cores') :param bool debug_mode: if True, output all logs (slower) :returns: if wait is True, an object containing the Subpopulation analyses, else a future to wait on the result :rtype: :class:`dataikuapi.dss.ml.DSSSubpopulationAnalyses` or :class:`dataikuapi.dss.future.DSSFuture` """ body = { "features": split_by if isinstance(split_by, list) else [split_by], "computationParams": { "sample_size": sample_size, "random_state": random_state, "n_jobs": n_jobs, "debug_mode": debug_mode, }} if self.mltask is not None: future_response = self.mltask.client._perform_json( "POST", "/projects/%s/models/lab/%s/%s/models/%s/subpopulation-analyses" % (self.mltask.project_key, self.mltask.analysis_id, self.mltask.mltask_id, self.mltask_model_id), body=body ) future = DSSFuture(self.mltask.client, future_response.get("jobId", None), future_response) else: future_response = self.saved_model.client._perform_json( "POST", "/projects/%s/savedmodels/%s/versions/%s/subpopulation-analyses" % (self.saved_model.project_key, self.saved_model.sm_id, self.saved_model_version), body=body ) future = DSSFuture(self.saved_model.client, future_response.get("jobId", None), future_response) if wait: prediction_type = self.details.get("coreParams", {}).get("prediction_type") return DSSSubpopulationAnalyses(future.wait_for_result(), prediction_type) else: return future def get_subpopulation_analyses(self): """ Retrieve all subpopulation analyses computed for this trained model :returns: the subpopulation analyses :rtype: :class:`dataikuapi.dss.ml.DSSSubpopulationAnalyses` """ if self.mltask is not None: data = self.mltask.client._perform_json( "GET", "/projects/%s/models/lab/%s/%s/models/%s/subpopulation-analyses" % (self.mltask.project_key, self.mltask.analysis_id, self.mltask.mltask_id, self.mltask_model_id) ) else: data = self.saved_model.client._perform_json( "GET", "/projects/%s/savedmodels/%s/versions/%s/subpopulation-analyses" % (self.saved_model.project_key, self.saved_model.sm_id, self.saved_model_version), ) prediction_type = self.details.get("coreParams", {}).get("prediction_type") return DSSSubpopulationAnalyses(data, prediction_type) def compute_partial_dependencies(self, features, wait=True, sample_size=1000, random_state=1337, n_jobs=1, debug_mode=False): """ Launch computation of Partial dependencies for this trained model. :param list|str features: feature(s) on which partial dependencies are to be computed :param bool wait: if True, the call blocks until the computation is finished and returns the results directly :param int sample_size: number of records of the dataset to use for the computation :param int random_state: random state to use to build sample, for reproducibility :param int n_jobs: number of cores used for parallel training. (-1 means 'all cores') :param bool debug_mode: if True, output all logs (slower) :returns: if wait is True, an object containing the Partial dependencies, else a future to wait on the result :rtype: :class:`dataikuapi.dss.ml.DSSPartialDependencies` or :class:`dataikuapi.dss.future.DSSFuture` """ body = { "features": features if isinstance(features, list) else [features], "computationParams": { "sample_size": sample_size, "random_state": random_state, "n_jobs": n_jobs, "debug_mode": debug_mode, }} if self.mltask is not None: future_response = self.mltask.client._perform_json( "POST", "/projects/%s/models/lab/%s/%s/models/%s/partial-dependencies" % (self.mltask.project_key, self.mltask.analysis_id, self.mltask.mltask_id, self.mltask_model_id), body=body ) future = DSSFuture(self.mltask.client, future_response.get("jobId", None), future_response) else: future_response = self.saved_model.client._perform_json( "POST", "/projects/%s/savedmodels/%s/versions/%s/partial-dependencies" % (self.saved_model.project_key, self.saved_model.sm_id, self.saved_model_version), body=body ) future = DSSFuture(self.saved_model.client, future_response.get("jobId", None), future_response) if wait: return DSSPartialDependencies(future.wait_for_result()) else: return future def get_partial_dependencies(self): """ Retrieve all partial dependencies computed for this trained model :returns: the partial dependencies :rtype: :class:`dataikuapi.dss.ml.DSSPartialDependencies` """ if self.mltask is not None: data = self.mltask.client._perform_json( "GET", "/projects/%s/models/lab/%s/%s/models/%s/partial-dependencies" % (self.mltask.project_key, self.mltask.analysis_id, self.mltask.mltask_id, self.mltask_model_id) ) else: data = self.saved_model.client._perform_json( "GET", "/projects/%s/savedmodels/%s/versions/%s/partial-dependencies" % (self.saved_model.project_key, self.saved_model.sm_id, self.saved_model_version), ) return DSSPartialDependencies(data) class DSSSubpopulationGlobal(object): """ Object to read details of performance on global population used for subpopulation analyses. Do not create this object directly, use :meth:`DSSSubpopulationAnalyses.get_global()` instead """ def __init__(self, data, prediction_type): self._internal_dict = data self.prediction_type = prediction_type def get_raw(self): """ Gets the raw dictionary of the global subpopulation performance :rtype: dict """ return self._internal_dict def __repr__(self): return "{cls}(prediction_type={type})".format(cls=self.__class__.__name__, type=self.prediction_type) def get_performance_metrics(self): """ Gets the performance results of the global population used for the subpopulation analysis """ return self._internal_dict["performanceMetrics"] def get_prediction_info(self): """ Gets the prediction info of the global population used for the subpopulation analysis """ global_metrics = self._internal_dict["perf"]["globalMetrics"] if self.prediction_type == "BINARY_CLASSIFICATION": return { "predictedPositiveRatio": global_metrics["predictionAvg"][0], "actualPositiveRatio": global_metrics["targetAvg"][0], "testWeight": global_metrics["testWeight"] } elif self.prediction_type == "REGRESSION": return { "predictedAvg":global_metrics["predictionAvg"][0], "predictedStd":global_metrics["predictionStd"][0], "actualAvg":global_metrics["targetAvg"][0], "actualStd":global_metrics["targetStd"][0], "testWeight":global_metrics["testWeight"] } class DSSSubpopulationModality(object): """ Object to read details of a subpopulation analysis modality Do not create this object directly, use :meth:`DSSSubpopulationAnalysis.get_modality_data(definition)` instead """ def __init__(self, feature_name, computed_as_type, data, prediction_type): self._internal_dict = data self.prediction_type = prediction_type if computed_as_type == "CATEGORY": self.definition = DSSSubpopulationCategoryModalityDefinition(feature_name, data) elif computed_as_type == "NUMERIC": self.definition = DSSSubpopulationNumericModalityDefinition(feature_name, data) def get_raw(self): """ Gets the raw dictionary of the subpopulation analysis modality :rtype: dict """ return self._internal_dict def __repr__(self): computed_as_type = "CATEGORY" if isinstance(self.definition, DSSSubpopulationCategoryModalityDefinition) else 'NUMERIC' return "{cls}(prediction_type={type}, feature={feature}, computed_as={computed_as_type})".format( cls=self.__class__.__name__, type=self.prediction_type, feature=self.definition.feature_name, computed_as_type=computed_as_type) def get_definition(self): """ Gets the definition of the subpopulation analysis modality :returns: definition :rtype: :class:`dataikuapi.dss.ml.DSSSubpopulationModalityDefinition` """ return self.definition def is_excluded(self): """ Whether modality has been excluded from analysis (e.g. too few rows in the subpopulation) """ return self._internal_dict.get("excluded", False) def get_performance_metrics(self): """ Gets the performance results of the modality """ if self.is_excluded(): raise ValueError("Excluded modalities do not have performance metrics") return self._internal_dict["performanceMetrics"] def get_prediction_info(self): """ Gets the prediction info of the modality """ if self.is_excluded(): raise ValueError("Excluded modalities do not have prediction info") global_metrics = self._internal_dict["perf"]["globalMetrics"] if self.prediction_type == "BINARY_CLASSIFICATION": return { "predictedPositiveRatio": global_metrics["predictionAvg"][0], "actualPositiveRatio": global_metrics["targetAvg"][0], "testWeight": global_metrics["testWeight"] } elif self.prediction_type == "REGRESSION": return { "predictedAvg":global_metrics["predictionAvg"][0], "predictedStd":global_metrics["predictionStd"][0], "actualAvg":global_metrics["targetAvg"][0], "actualStd":global_metrics["targetStd"][0], "testWeight":global_metrics["testWeight"] } class DSSSubpopulationModalityDefinition(object): MISSING_VALUES = "__DSSSubpopulationModalidityDefinition__MISSINGVALUES" def __init__(self, feature_name, data): self.missing_values = data.get("missing_values", False) self.index = data.get("index") self.feature_name = feature_name def is_missing_values(self): return self.missing_values class DSSSubpopulationNumericModalityDefinition(DSSSubpopulationModalityDefinition): def __init__(self, feature_name, data): super(DSSSubpopulationNumericModalityDefinition, self).__init__(feature_name, data) self.lte = data.get("lte", None) self.gt = data.get("gt", None) self.gte = data.get("gte", None) def contains(self, value): lte = self.lte if self.lte is not None else float("inf") gt = self.gt if self.gt is not None else float("-inf") gte = self.gte if self.gte is not None else float("-inf") return not self.missing_values and gt < value and gte <= value and lte >= value def __repr__(self): if self.missing_values: return "DSSSubpopulationNumericModalityDefinition(missing_values)" else: if self.gt is not None: repr_gt = "%s<" % self.gt elif self.gte is not None: repr_gt = "%s<=" % self.gte else: repr_gt = "" if self.lte is not None: repr_lt = "<=%s" % self.lte else: repr_lt = "" return "DSSSubpopulationNumericModalityDefinition(%s%s%s)" % (repr_gt, self.feature_name, repr_lt) class DSSSubpopulationCategoryModalityDefinition(DSSSubpopulationModalityDefinition): def __init__(self, feature_name, data): super(DSSSubpopulationCategoryModalityDefinition, self).__init__(feature_name, data) self.value = data.get("value", None) def contains(self, value): return value == self.value def __repr__(self): if self.missing_values: return "DSSSubpopulationCategoryModalityDefinition(missing_values)" else: return "DSSSubpopulationCategoryModalityDefinition(%s='%s')" % (self.feature_name, self.value) class DSSSubpopulationAnalysis(object): """ Object to read details of a subpopulation analysis of a trained model Do not create this object directly, use :meth:`DSSSubpopulationAnalyses.get_analysis(feature)` instead """ def __init__(self, analysis, prediction_type): self._internal_dict = analysis self.computed_as_type = analysis["computed_as_type"] self.modalities = [DSSSubpopulationModality(analysis["feature"], self.computed_as_type, m, prediction_type) for m in analysis.get("modalities", [])] def get_raw(self): """ Gets the raw dictionary of the subpopulation analysis :rtype: dict """ return self._internal_dict def __repr__(self): return "{cls}(computed_as_type={type}, feature={feature}, modalities_count={modalities_count})".format( cls=self.__class__.__name__, type=self.computed_as_type, feature=self._internal_dict["feature"], modalities_count=len(self.modalities)) def get_computation_params(self): """ Gets computation params """ return { "nbRecords": self._internal_dict["nbRecords"], "randomState": self._internal_dict["randomState"], "onSample": self._internal_dict["onSample"] } def list_modalities(self): """ List definitions of modalities """ return [m.definition for m in self.modalities] def get_modality_data(self, definition): """ Retrieves modality from definition :param definition: definition of modality to retrieve. Can be: * :class:`dataikuapi.dss.ml.DSSSubpopulationModalityDefinition` * `dataikuapi.dss.ml.DSSSubpopulationModalityDefinition.MISSING_VALUES` to retrieve modality corresponding to missing values * for category modality, can be a str corresponding to the value of the modality * for numeric modality, can be a number inside the modality :returns: the modality :rtype: :class:`dataikuapi.dss.ml.DSSSubpopulationModality` """ if definition == DSSSubpopulationModalityDefinition.MISSING_VALUES: for m in self.modalities: if m.definition.missing_values: return m raise ValueError("No 'missing values' modality found") if isinstance(definition, DSSSubpopulationModalityDefinition): modality_candidates = [m for m in self.modalities if m.definition.index == definition.index] if len(modality_candidates) == 0: raise ValueError("Modality with index '%s' not found" % definition.index) return modality_candidates[0] for m in self.modalities: if m.definition.contains(definition): return m raise ValueError("Modality not found: %s" % definition) class DSSSubpopulationAnalyses(object): """ Object to read details of subpopulation analyses of a trained model Do not create this object directly, use :meth:`DSSTrainedPredictionModelDetails.get_subpopulation_analyses()` instead """ def __init__(self, data, prediction_type): self._internal_dict = data self.prediction_type = prediction_type self.analyses = [] for analysis in data.get("subpopulationAnalyses", []): self.analyses.append(DSSSubpopulationAnalysis(analysis, prediction_type)) def get_raw(self): """ Gets the raw dictionary of subpopulation analyses :rtype: dict """ return self._internal_dict def __repr__(self): return "{cls}(prediction_type={type}, analyses={analyses})".format(cls=self.__class__.__name__, type=self.prediction_type, analyses=self.list_analyses()) def get_global(self): """ Retrieves information and performance on the full dataset used to compute the subpopulation analyses """ return DSSSubpopulationGlobal(self._internal_dict["global"], self.prediction_type) def list_analyses(self): """ Lists all features on which subpopulation analyses have been computed """ return [analysis.get_raw()["feature"] for analysis in self.analyses] def get_analysis(self, feature): """ Retrieves the subpopulation analysis for a particular feature """ try: return next(analysis for analysis in self.analyses if analysis.get_raw()["feature"] == feature) except StopIteration: raise ValueError("Subpopulation analysis for feature '%s' cannot be found" % feature) class DSSPartialDependence(object): """ Object to read details of partial dependence of a trained model Do not create this object directly, use :meth:`DSSPartialDependencies.get_partial_dependence(feature)` instead """ def __init__(self, data): self._internal_dict = data def get_raw(self): """ Gets the raw dictionary of the partial dependence :rtype: dict """ return self._internal_dict def __repr__(self): return "{cls}(feature={feature})".format(cls=self.__class__.__name__, feature=self._internal_dict["feature"]) def get_computation_params(self): """ Gets computation params """ return { "nbRecords": self._internal_dict["nbRecords"], "randomState": self._internal_dict["randomState"], "onSample": self._internal_dict["onSample"] } class DSSPartialDependencies(object): """ Object to read details of partial dependencies of a trained model Do not create this object directly, use :meth:`DSSTrainedPredictionModelDetails.get_partial_dependencies()` instead """ def __init__(self, data): self._internal_dict = data self.partial_dependencies = [] for pd in data.get("partialDependencies", []): self.partial_dependencies.append(DSSPartialDependence(pd)) def get_raw(self): """ Gets the raw dictionary of partial dependencies :rtype: dict """ return self._internal_dict def __repr__(self): return "{cls}(features={features})".format(cls=self.__class__.__name__, features=self.list_features()) def list_features(self): """ Lists all features on which partial dependencies have been computed """ return [partial_dep.get_raw()["feature"] for partial_dep in self.partial_dependencies] def get_partial_dependence(self, feature): """ Retrieves the partial dependencies for a particular feature """ try: return next(pd for pd in self.partial_dependencies if pd.get_raw()["feature"] == feature) except StopIteration: raise ValueError("Partial dependence for feature '%s' cannot be found" % feature) class DSSClustersFacts(object): def __init__(self, clusters_facts): self.clusters_facts = clusters_facts def get_raw(self): """Gets the raws facts data structure""" return self.clusters_facts def get_cluster_size(self, cluster_index): """Gets the size of a cluster identified by its index""" return self.clusters_facts["clusters"][cluster_index]["size"] def get_facts_for_cluster(self, cluster_index): """ Gets all facts for a cluster identified by its index. Returns a list of dicts :rtype: list """ return self.clusters_facts["clusters"][cluster_index]["facts"] def get_facts_for_cluster_and_feature(self, cluster_index, feature_name): """ Gets all facts for a cluster identified by its index, limited to a feature name. Returns a list of dicts :rtype: list """ return [x for x in self.get_facts_for_cluster(cluster_index) if x["feature_label"] == feature_name] class DSSTrainedClusteringModelDetails(DSSTrainedModelDetails): """ Object to read details of a trained clustering model Do not create this object directly, use :meth:`DSSMLTask.get_trained_model_details()` instead """ def __init__(self, details, snippet, saved_model=None, saved_model_version=None, mltask=None, mltask_model_id=None): super(DSSTrainedClusteringModelDetails, self).__init__(details, snippet, saved_model, saved_model_version, mltask, mltask_model_id) def get_raw(self): """ Gets the raw dictionary of trained model details """ return self.details def get_train_info(self): """ Returns various information about the train process (size of the train set, quick description, timing information) :rtype: dict """ return self.details["trainInfo"] def get_facts(self): """ Gets the 'cluster facts' data, i.e. the structure behind the screen "for cluster X, average of Y is Z times higher than average :rtype: :class:`DSSClustersFacts` """ return DSSClustersFacts(self.details["facts"]) def get_performance_metrics(self): """ Returns all performance metrics for this clustering model. :returns: a dict of performance metrics values :rtype: dict """ import copy clean_snippet = copy.deepcopy(self.snippet) for x in ["fullModelId", "algorithm", "trainInfo", "userMeta", "backendType", "sessionId", "sessionDate", "facts"]: if x in clean_snippet: del clean_snippet[x] return clean_snippet def get_preprocessing_settings(self): """ Gets the preprocessing settings that were used to train this model :rtype: dict """ return self.details["preprocessing"] def get_modeling_settings(self): """ Gets the modeling (algorithms) settings that were used to train this model. Note: the structure of this dict is not the same as the modeling params on the ML Task (which may contain several algorithm) :rtype: dict """ return self.details["modeling"] def get_actual_modeling_params(self): """ Gets the actual / resolved parameters that were used to train this model. :return: A dictionary, which contains at least a "resolved" key :rtype: dict """ return self.details["actualParams"] def get_scatter_plots(self): """ Gets the cluster scatter plot data :return: a DSSScatterPlots object to interact with the scatter plots :rtype: :class:`dataikuapi.dss.ml.DSSScatterPlots` """ scatters = self.mltask.client._perform_json( "GET", "/projects/%s/models/lab/%s/%s/models/%s/scatter-plots" % (self.mltask.project_key, self.mltask.analysis_id, self.mltask.mltask_id, self.mltask_model_id)) return DSSScatterPlots(scatters) class DSSMLTask(object): @staticmethod def from_full_model_id(client, fmi, project_key=None): match = re.match(r"^A-(\w+)-(\w+)-(\w+)-(s[0-9]+)-(pp[0-9]+(-part-(\w+)|-base)?)-(m[0-9]+)$", fmi) if match is None: return DataikuException("Invalid model id: {}".format(fmi)) else: if project_key is None: project_key = match.group(1) return DSSMLTask(client, project_key, match.group(2), match.group(3)) """A handle to interact with a MLTask for prediction or clustering in a DSS visual analysis""" def __init__(self, client, project_key, analysis_id, mltask_id): self.client = client self.project_key = project_key self.analysis_id = analysis_id self.mltask_id = mltask_id def delete(self): """ Delete the present ML task """ return self.client._perform_json( "DELETE", "/projects/%s/models/lab/%s/%s/" % (self.project_key, self.analysis_id, self.mltask_id)) def wait_guess_complete(self): """ Waits for guess to be complete. This should be called immediately after the creation of a new ML Task (if the ML Task was created with wait_guess_complete=False), before calling ``get_settings`` or ``train`` """ while True: status = self.get_status() if status.get("guessing", "???") == False: break time.sleep(0.2) def get_status(self): """ Gets the status of this ML Task :return: a dict """ return self.client._perform_json( "GET", "/projects/%s/models/lab/%s/%s/status" % (self.project_key, self.analysis_id, self.mltask_id)) def get_settings(self): """ Gets the settings of this ML Tasks :return: a DSSMLTaskSettings object to interact with the settings :rtype: :class:`dataikuapi.dss.ml.DSSMLTaskSettings` """ settings = self.client._perform_json( "GET", "/projects/%s/models/lab/%s/%s/settings" % (self.project_key, self.analysis_id, self.mltask_id)) if settings["taskType"] == "PREDICTION": if settings["predictionType"] == DSSTimeseriesForecastingMLTaskSettings.PredictionTypes.TIMESERIES_FORECAST: return DSSTimeseriesForecastingMLTaskSettings(self.client, self.project_key, self.analysis_id, self.mltask_id, settings) else: return DSSPredictionMLTaskSettings(self.client, self.project_key, self.analysis_id, self.mltask_id, settings) else: return DSSClusteringMLTaskSettings(self.client, self.project_key, self.analysis_id, self.mltask_id, settings) def train(self, session_name=None, session_description=None, run_queue=False): """ Trains models for this ML Task :param str session_name: name for the session :param str session_description: description for the session This method waits for train to complete. If you want to train asynchronously, use :meth:`start_train` and :meth:`wait_train_complete` This method returns the list of trained model identifiers. It returns models that have been trained for this train session, not all trained models for this ML task. To get all identifiers for all models trained across all training sessions, use :meth:`get_trained_models_ids` These identifiers can be used for :meth:`get_trained_model_snippet`, :meth:`get_trained_model_details` and :meth:`deploy_to_flow` :return: A list of model identifiers :rtype: list of strings """ train_ret = self.start_train(session_name, session_description, run_queue) self.wait_train_complete() return self.get_trained_models_ids(session_id = train_ret["sessionId"]) def ensemble(self, model_ids=None, method=None): """ Create an ensemble model of a set of models :param list model_ids: A list of model identifiers (defaults to `[]`) :param str method: the ensembling method. One of: AVERAGE, PROBA_AVERAGE, MEDIAN, VOTE, LINEAR_MODEL, LOGISTIC_MODEL This method waits for the ensemble train to complete. If you want to train asynchronously, use :meth:`start_ensembling` and :meth:`wait_train_complete` This method returns the identifier of the trained ensemble. To get all identifiers for all models trained across all training sessions, use :meth:`get_trained_models_ids` This identifier can be used for :meth:`get_trained_model_snippet`, :meth:`get_trained_model_details` and :meth:`deploy_to_flow` :return: A model identifier :rtype: string """ if model_ids is None: model_ids = [] train_ret = self.start_ensembling(model_ids, method) self.wait_train_complete() return train_ret def start_train(self, session_name=None, session_description=None, run_queue=False): """ Starts asynchronously a new train session for this ML Task. :param str session_name: name for the session :param str session_description: description for the session This returns immediately, before train is complete. To wait for train to complete, use ``wait_train_complete()`` """ session_info = { "sessionName" : session_name, "sessionDescription" : session_description, "runQueue": run_queue } return self.client._perform_json( "POST", "/projects/%s/models/lab/%s/%s/train" % (self.project_key, self.analysis_id, self.mltask_id), body=session_info) def start_ensembling(self, model_ids=None, method=None): """ Creates asynchronously a new ensemble models of a set of models. :param list model_ids: A list of model identifiers (defaults to `[]`) :param str method: the ensembling method (AVERAGE, PROBA_AVERAGE, MEDIAN, VOTE, LINEAR_MODEL, LOGISTIC_MODEL) This returns immediately, before train is complete. To wait for train to complete, use :meth:`wait_train_complete` :return: the model identifier of the ensemble :rtype: string """ if model_ids is None: model_ids = [] ensembling_request = { "method" : method, "modelsIds" : model_ids } return self.client._perform_json( "POST", "/projects/%s/models/lab/%s/%s/ensemble" % (self.project_key, self.analysis_id, self.mltask_id), body=ensembling_request)['id'] def wait_train_complete(self): """ Waits for train to be complete (if started with :meth:`start_train`) """ while True: status = self.get_status() if status.get("training", "???") == False: break time.sleep(2) def get_trained_models_ids(self, session_id=None, algorithm=None): """ Gets the list of trained model identifiers for this ML task. These identifiers can be used for :meth:`get_trained_model_snippet` and :meth:`deploy_to_flow` :return: A list of model identifiers :rtype: list of strings """ full_model_ids = self.get_status()["fullModelIds"] if session_id is not None: full_model_ids = [fmi for fmi in full_model_ids if fmi.get('fullModelId', {}).get('sessionId', '') == session_id] model_ids = [x["id"] for x in full_model_ids] if algorithm is not None: # algorithm is in the snippets model_ids = [fmi for fmi, s in self.get_trained_model_snippet(ids=model_ids).iteritems() if s.get("algorithm", "") == algorithm] return model_ids def get_trained_model_snippet(self, id=None, ids=None): """ Gets a quick summary of a trained model, as a dict. For complete information and a structured object, use :meth:`get_trained_model_detail` :param str id: a model id :param list ids: a list of model ids :rtype: dict """ if id is not None: obj = { "modelsIds" : [id] } elif ids is not None: obj = { "modelsIds" : ids } else: obj = {} ret = self.client._perform_json( "GET", "/projects/%s/models/lab/%s/%s/models-snippets" % (self.project_key, self.analysis_id, self.mltask_id), body = obj) if id is not None: return ret[id] else: return ret def get_trained_model_details(self, id): """ Gets details for a trained model :param str id: Identifier of the trained model, as returned by :meth:`get_trained_models_ids` :return: A :class:`DSSTrainedPredictionModelDetails` or :class:`DSSTrainedClusteringModelDetails` representing the details of this trained model id :rtype: :class:`DSSTrainedPredictionModelDetails` or :class:`DSSTrainedClusteringModelDetails` """ ret = self.client._perform_json( "GET", "/projects/%s/models/lab/%s/%s/models/%s/details" % (self.project_key, self.analysis_id, self.mltask_id,id)) snippet = self.get_trained_model_snippet(id) if "facts" in ret: return DSSTrainedClusteringModelDetails(ret, snippet, mltask=self, mltask_model_id=id) else: return DSSTrainedPredictionModelDetails(ret, snippet, mltask=self, mltask_model_id=id) def delete_trained_model(self, model_id): """ Deletes a trained model :param str model_id: Model identifier, as returend by :meth:`get_trained_models_ids` """ self.client._perform_empty( "DELETE", "/projects/%s/models/lab/%s/%s/models/%s" % (self.project_key, self.analysis_id, self.mltask_id, model_id)) def train_queue(self): """ Trains this MLTask's queue :return: A dict including the next sessionID to be trained in the queue :rtype dict """ return self.client._perform_json( "POST", "/projects/%s/models/lab/%s/%s/actions/train-queue" % (self.project_key, self.analysis_id, self.mltask_id)) def deploy_to_flow(self, model_id, model_name, train_dataset, test_dataset=None, redo_optimization=True): """ Deploys a trained model from this ML Task to a saved model + train recipe in the Flow. :param str model_id: Model identifier, as returned by :meth:`get_trained_models_ids` :param str model_name: Name of the saved model to deploy in the Flow :param str train_dataset: Name of the dataset to use as train set. May either be a short name or a PROJECT.name long name (when using a shared dataset) :param str test_dataset: Name of the dataset to use as test set. If null, split will be applied to the train set. May either be a short name or a PROJECT.name long name (when using a shared dataset). Only for PREDICTION tasks :param bool redo_optimization: Should the hyperparameters optimization phase be done ? Defaults to True. Only for PREDICTION tasks :return: A dict containing: "savedModelId" and "trainRecipeName" - Both can be used to obtain further handles :rtype: dict """ obj = { "trainDatasetRef" : train_dataset, "testDatasetRef" : test_dataset, "modelName" : model_name, "redoOptimization": redo_optimization } return self.client._perform_json( "POST", "/projects/%s/models/lab/%s/%s/models/%s/actions/deployToFlow" % (self.project_key, self.analysis_id, self.mltask_id, model_id), body = obj) def redeploy_to_flow(self, model_id, recipe_name=None, saved_model_id=None, activate=True): """ Redeploys a trained model from this ML Task to a saved model + train recipe in the Flow. Either recipe_name of saved_model_id need to be specified :param str model_id: Model identifier, as returned by :meth:`get_trained_models_ids` :param str recipe_name: Name of the training recipe to update :param str saved_model_id: Name of the saved model to update :param bool activate: Should the deployed model version become the active version :return: A dict containing: "impactsDownstream" - whether the active version changed and downstream recipes are impacted :rtype: dict """ obj = { "recipeName" : recipe_name, "savedModelId" : saved_model_id, "activate" : activate } return self.client._perform_json( "POST", "/projects/%s/models/lab/%s/%s/models/%s/actions/redeployToFlow" % (self.project_key, self.analysis_id, self.mltask_id, model_id), body = obj) def remove_unused_splits(self): """ Deletes all stored splits data that are not anymore in use for this ML Task. It is generally not needed to call this method """ self.client._perform_empty( "POST", "/projects/%s/models/lab/%s/%s/actions/removeUnusedSplits" % (self.project_key, self.analysis_id, self.mltask_id)) def remove_all_splits(self): """ Deletes all stored splits data for this ML Task. This operation saves disk space. After performing this operation, it will not be possible anymore to: * Ensemble already trained models * View the "predicted data" or "charts" for already trained models * Resume training of models for which optimization had been previously interrupted Training new models remains possible """ self.client._perform_empty( "POST", "/projects/%s/models/lab/%s/%s/actions/removeAllSplits" % (self.project_key, self.analysis_id, self.mltask_id)) def guess(self, prediction_type=None, reguess_level=None, target_variable=None, timeseries_identifiers=None, time_variable=None, full_reguess=None): """ Reguess all the settings of the ML task when no optional parameter are given. For prediction ML tasks only, set a new value for a core parameter of the task (target variable or prediction type) and subsequently reguess the impacted settings. :param string prediction_type: Only valid for prediction tasks of either `BINARY_CLASSIFICATION`, `MULTICLASS` or `REGRESSION` type, ignored otherwise. The prediction type to set. Cannot be set if target_variable, time_variable, or timeseries_identifiers is also specified. :param string target_variable: Only valid for prediction tasks, ignored for clustering. The target variable to set. Cannot be set if prediction_type, time_variable, or timeseries_identifiers is also specified. :param list timeseries_identifiers: Only valid for time series forecasting tasks. List of columns to be used as time series identifiers. Cannot be set if prediction_type, target_variable, or time_variable is also specified. :param string time_variable: Only valid for time series forecasting tasks. Column to be used as time variable. Cannot be set if prediction_type, target_variable, or timeseries_identifiers is also specified. :param bool full_reguess: Only valid for prediction tasks, ignored for clustering. Scope of the reguess process: whether it should reguess all the settings after changing a core parameter, or only reguess impacted settings (e.g. target remapping when changing the target, metrics when changing the prediction type...). Ignored if no core parameter is given. Defaults to true. :param string reguess_level: Deprecated, use `full_reguess` instead. Only valid for prediction tasks. Can be one of the following values: - TARGET_CHANGE: Change the target if target_variable is specified, reguess the target remapping, and clear the model's assertions if any. Equivalent to `full_reguess`=False (recommended usage) - FULL_REGUESS: All the settings of the ML task are reguessed. Equivalent to `full_reguess`=True (recommended usage) """ obj = {} if prediction_type is not None: obj["predictionType"] = prediction_type if timeseries_identifiers is not None: obj["timeseriesIdentifiers"] = timeseries_identifiers if time_variable is not None: obj["timeVariable"] = time_variable if target_variable is not None: obj["targetVariable"] = target_variable if full_reguess is not None: obj["fullReguess"] = full_reguess if reguess_level is not None: logger.info("'reguess_level' parameter ignored, superseded by 'full_reguess'") if reguess_level is not None and full_reguess is None: logger.info("'reguess_level' is deprecated, use 'full_reguess' instead") if reguess_level == "TARGET_CHANGE": obj["fullReguess"] = False elif reguess_level == "FULL_REGUESS": obj["fullReguess"] = True else: raise ValueError("Unsupported value for 'reguess_level': " + reguess_level) if set(obj.keys()) == {"fullReguess"}: logger.info("No core params given, reguessing all settings") self.client._perform_empty( "PUT", "/projects/{project_key}/models/lab/{analysis_id}/{mltask_id}/guess".format( project_key=self.project_key, analysis_id=self.analysis_id, mltask_id=self.mltask_id ), params=obj ) class DSSMLTaskQueues(object): """ Iterable listing of MLTask queues """ def __init__(self, data): self.data = data def __iter__(self): return self.data["queues"].__iter__() def get_raw(self): return self.data dataiku-api-client-python-11.4.0/dataikuapi/dss/mlflow.py000066400000000000000000000337001440464736400233570ustar00rootroot00000000000000import json import time from datetime import datetime from dataikuapi.dss.utils import DSSDatasetSelectionBuilder from dataikuapi.dss.savedmodel import ExternalModelVersionHandler class DSSMLflowExtension(object): """ A handle to interact with specific endpoints of the DSS MLflow integration. Do not create this directly, use :meth:`dataikuapi.dss.project.DSSProject.get_mlflow_extension` """ def __init__(self, client, project_key): self.client = client self.project = client.get_project(project_key) self.project_key = project_key def list_models(self, run_id): """ Returns the list of models of given run :param run_id: run_id for which to return a list of models :type run_id: str """ response = self.client._perform_http( "GET", "/api/2.0/mlflow/extension/models/{}".format(run_id), headers={"x-dku-mlflow-project-key": self.project_key} ) return response.json() def list_experiments(self, view_type="ACTIVE_ONLY", max_results=1000): """ Returns the list of experiments in the DSS project for which MLflow integration is setup :param view_type: ACTIVE_ONLY, DELETED_ONLY or ALL :type view_type: str :param max_results: max results count :type max_results: int :rtype: dict """ response = self.client._perform_http( "GET", "/api/2.0/mlflow/experiments/list?view_type={view_type}&max_results={max_results}".format(view_type=view_type, max_results=max_results), headers={"x-dku-mlflow-project-key": self.project_key} ) return response.json() def rename_experiment(self, experiment_id, new_name): """ Renames an experiment :param experiment_id: experiment id :type experiment_id: str :param new_name: new name :type new_name: str """ response = self.client._perform_http( "POST", "/api/2.0/mlflow/experiments/update", headers={"x-dku-mlflow-project-key": self.project_key}, body={"experiment_id": experiment_id, "new_name": new_name} ) return response.json() def restore_experiment(self, experiment_id): """ Restores a deleted experiment :param experiment_id: experiment id :type experiment_id: str """ response = self.client._perform_http( "POST", "/api/2.0/mlflow/experiments/restore", headers={"x-dku-mlflow-project-key": self.project_key}, body={"experiment_id": experiment_id} ) return response.json() def restore_run(self, run_id): """ Restores a deleted run :param run_id: run id :type run_id: str """ response = self.client._perform_http( "POST", "/api/2.0/mlflow/runs/restore", headers={"x-dku-mlflow-project-key": self.project_key}, body={"run_id": run_id} ) return response.json() def garbage_collect(self): """ Permanently deletes the experiments and runs marked as "Deleted" """ self.client._perform_http( "POST", "/api/2.0/mlflow/extension/garbage-collect", headers={"x-dku-mlflow-project-key": self.project_key} ) def create_experiment_tracking_dataset(self, dataset_name, experiment_ids=[], view_type="ACTIVE_ONLY", filter_expr="", order_by=[], format="LONG"): """ Creates a virtual dataset exposing experiment tracking data. :param dataset_name: name of the dataset :type dataset_name: str :param experiment_ids: list of ids of experiments to filter on. No filtering if empty :type experiment_ids: list(str) :param view_type: one of ACTIVE_ONLY, DELETED_ONLY and ALL. Default is ACTIVE_ONLY :type view_type: str :param filter_expr: MLflow search expression :type filter_expr: str :param order_by: list of order by clauses. Default is ordered by start_time, then runId :type order_by: list(str) :param format: LONG or JSON. Default is LONG :type format: str """ self.client._perform_http( "POST", "/api/2.0/mlflow/extension/create-project-experiments-dataset", headers={"x-dku-mlflow-project-key": self.project_key}, body={ "datasetName": dataset_name, "experimentIds": experiment_ids, "viewType": view_type, "filter": filter_expr, "orderBy": order_by, "format": format } ) def clean_experiment_tracking_db(self): """ Cleans the experiments, runs, params, metrics, tags, etc. for this project This call requires an API key with admin rights """ self.client._perform_raw("DELETE", "/api/2.0/mlflow/extension/clean-db/%s" % self.project_key) def set_run_inference_info(self, run_id, prediction_type, classes=None, code_env_name=None, target=None): """ Sets the type of the model, and optionally other information useful to deploy or evaluate it. prediction_type must be one of: - REGRESSION - BINARY_CLASSIFICATION - MULTICLASS - OTHER Classes must be specified if and only if the model is a BINARY_CLASSIFICATION or MULTICLASS model. This information is leveraged to filter saved models on their prediction type and prefill the classes when deploying (using the GUI or :meth:`deploy_run_model`) an MLflow model as a version of a DSS Saved Model. :param prediction_type: prediction type (see doc) :type prediction_type: str :param run_id: run_id for which to set the classes :type run_id: str :param classes: ordered list of classes (not for all prediction types, see doc). Every class will be converted by calling str(). The classes must be specified in the same order as learned by the model. Some flavors such as scikit-learn may allow you to build this list from the model itself. :type classes: list :param code_env_name: name of an adequate DSS python code environment :type code_env_name: str :param target: name of the target :type target: str """ if prediction_type not in {"REGRESSION", "BINARY_CLASSIFICATION", "MULTICLASS", "OTHER"}: raise ValueError('Invalid prediction type: {}'.format(prediction_type)) if classes and prediction_type not in {"BINARY_CLASSIFICATION", "MULTICLASS"}: raise ValueError('Classes can be specified only for BINARY_CLASSIFICATION or MULTICLASS prediction types') if prediction_type in {"BINARY_CLASSIFICATION", "MULTICLASS"}: if not classes: raise ValueError('Classes must be specified for {} prediction type'.format(prediction_type)) if not isinstance(classes, list): raise ValueError('Wrong type for classes: {}'.format(type(classes))) for cur_class in classes: if cur_class is None: raise ValueError('class cannot be None') classes = [str(cur_class) for cur_class in classes] if code_env_name and not isinstance(code_env_name, str): raise ValueError('code_env_name must be a string') if target and not isinstance(target, str): raise ValueError('target must be a string') params = { "run_id": run_id, "prediction_type": prediction_type } if classes: params["classes"] = json.dumps(classes) if code_env_name: params["code_env_name"] = code_env_name if target: params["target"] = target self.client._perform_http( "POST", "/api/2.0/mlflow/extension/set-run-inference-info", headers={"x-dku-mlflow-project-key": self.project_key}, body=params ) def deploy_run_model(self, run_id, sm_id, version_id=None, use_inference_info=True, code_env_name=None, evaluation_dataset=None, target_column_name=None, class_labels=None, model_sub_folder=None, selection=None, activate=True, binary_classification_threshold=0.5, use_optimal_threshold=True): """ Deploys a model from an experiment run, with lineage. Simple usage: .. code-block:: python mlflow_ext.set_run_inference_info(run_id, "BINARY_CLASSIFICATION", list_of_classes, code_env_name, target_column_name) sm_id = project.create_mlflow_pyfunc_model("model_name", "BINARY_CLASSIFICATION").id mlflow_extension.deploy_run_model(run_id, sm_id, evaluation_dataset) If the optional `evaluation_dataset` is not set, the model will be deployed but not evaluated: this makes `target_column_name` optional as well in :meth:`set_run_inference_info` :param run_id: The id of the run to deploy :type run_id: str :param sm_id: The id of the saved model to deploy the run to :type sm_id: str :param version_id: [optional] Unique identifier of a Saved Model Version. If id already exists, existing version is overwritten. Whitespaces or dashes are not allowed. If not set, a timestamp will be used as version_id. :type version_id: str :param use_inference_info: [optional] default to True. if set, uses the :meth:`set_inference_info` previously done on the run to retrieve the prediction type of the model, its code environment, classes and target. :type use_inference_info: bool :param evaluation_dataset: [optional] The evaluation dataset, if the deployment of the models can imply an evaluation. :type evaluation_dataset: str :param target_column_name: [optional] The target column of the evaluation dataset. Can be set by :meth:`set_inference_info`. :type target_column_name: str :param class_labels: [optional] The class labels of the target. Can be set by :meth:`set_inference_info`. :type class_labels: list(str) :param code_env_name: [optional] The code environment to be used. Must contain a supported version of the mlflow package and the ML libs used to train the model. Can be set by :meth:`set_inference_info`. :type code_env_name: str :param model_sub_folder: [optional] The name of the subfolder containing the model. Optional if it is unique. Existing values can be retrieved with `project.get_mlflow_extension().list_models(run_id)` :type model_sub_folder: str :param str selection: [optional] will default to HEAD_SEQUENTIAL with a maxRecords of 10_000. :type selection: :class:`DSSDatasetSelectionBuilder` optional sampling parameter for the evaluation or dict e.g. DSSDatasetSelectionBuilder().with_head_sampling(100) {"samplingMethod": "HEAD_SEQUENTIAL", "maxRecords": 100} :param activate: [optional] True by default. Activate or not the version after deployment :type activate: bool :param binary_classification_threshold: [optional] Threshold (or cut-off) value to override if the model is a binary classification :type binary_classification_threshold: float :param use_optimal_threshold: [optional] Use or not the optimal threshold for the saved model metric computed at evaluation :type use_optimal_threshold: bool :return: a handler in order to interact with the new MLFlow model version :rtype: :class:`dataikuapi.dss.savedmodel.ExternalModelVersionHandler` """ sampling_param = selection.build() if isinstance( selection, DSSDatasetSelectionBuilder) else selection if version_id is None: dt = datetime.fromtimestamp(time.time()) version_id = dt.strftime("%Y_%m_%dT%H_%M_%S") prediction_type = self.project.get_saved_model(sm_id).get_settings().settings["miniTask"].get("predictionType") output_labels = [] if class_labels: for label in class_labels: output_labels.append({'label': label}) model_version_info_obj = {} if evaluation_dataset is not None: model_version_info_obj["gatherFeaturesFromDataset"] = evaluation_dataset if target_column_name is not None: model_version_info_obj["targetColumnName"] = target_column_name if output_labels is not None: model_version_info_obj["classLabels"] = output_labels if code_env_name is not None: model_version_info_obj["pythonCodeEnvName"] = code_env_name if prediction_type is not None: model_version_info_obj["predictionType"] = prediction_type self.client._perform_http("POST", "/api/2.0/mlflow/extension/deploy-run", headers={"x-dku-mlflow-project-key": self.project_key}, params={"projectKey": self.project_key, "runId": run_id, "smId": sm_id, "modelSubfolder": model_sub_folder, "versionId": version_id, "modelVersionInfo": json.dumps(model_version_info_obj), "samplingParam": sampling_param, "activate": activate, "binaryClassificationThreshold": binary_classification_threshold, "useOptimalThreshold": use_optimal_threshold, "useInferenceInfo": use_inference_info } ) return ExternalModelVersionHandler(sm_id, version_id) dataiku-api-client-python-11.4.0/dataikuapi/dss/modelcomparison.py000066400000000000000000000147511440464736400252570ustar00rootroot00000000000000from .discussion import DSSObjectDiscussions import re class DSSModelComparison(object): """ A handle to interact with a model comparison on the DSS instance Do not create this directly, use :meth:`dataikuapi.dss.DSSProject.get_model_comparison` """ def __init__(self, client, project_key, mec_id): self.client = client self.project = client.get_project(project_key) self.project_key = project_key self.mec_id = mec_id @property def id(self): return self.mec_id def get_settings(self): """ Returns the settings of this model comparison :rtype: :class:`dataikuapi.dss.modelcomparison.DSSModelComparisonSettings` """ data = self.client._perform_json( "GET", "/projects/%s/modelcomparisons/%s" % (self.project_key, self.mec_id)) return DSSModelComparisonSettings(self, data) def get_object_discussions(self): """ Get a handle to manage discussions on the model comparison :returns: the handle to manage discussions :rtype: :class:`dataikuapi.discussion.DSSObjectDiscussions` """ return DSSObjectDiscussions(self.client, self.project_key, "MODEL_COMPARISON", self.mec_id) def get_evaluation_like_from_full_id(self, full_id): """ Retrieves a Saved Model from the flow, a Lab Model from an Analysis or a Model Evaluation from a Model Evaluation Store) using its full id. :param string full_id: the full id of the item to retrieve :returns: A handle on the Saved Model, the Model Evaluation or the Lab Model :rtype: :class:`dataikuapi.dss.savedmodel.DSSSavedModel` :rtype: :class:`dataikuapi.dss.modelevaluationstore.DSSModelEvaluation` :rtype: :class:`dataikuapi.dss.ml.DSSTrainedPredictionModelDetails` """ saved_model_pattern = re.compile("^S-(\\w+)-(\\w+)-(\\w+)(?:-part-(\\w+)-(v?\\d+))?$\\Z") analysis_model_pattern = re.compile("^A-(\\w+)-(\\w+)-(\\w+)-(s[0-9]+)-(pp[0-9]+(?:-part-(\\w+)|-base)?)-(m[0-9]+)$\\Z") model_evaluation_pattern = re.compile("^ME-(\\w+)-(\\w+)-(\\w+)$\\Z") if saved_model_pattern.match(full_id): return self.project.get_saved_model(full_id) elif model_evaluation_pattern.match(full_id): mes_id = full_id.split('-')[2] evaluation_id = full_id.split('-')[3] mes = self.project.get_model_evaluation_store(mes_id) return mes.get_model_evaluation(evaluation_id) elif analysis_model_pattern.match(full_id): analysis_id = full_id.split('-')[2] task_id = full_id.split('-')[3] return self.project.get_ml_task(analysis_id, task_id).get_trained_model_details(full_id) raise ValueError("{} is not a valid full model id or full model evaluation id.".format(full_id)) ######################################################## # Deletion ######################################################## def delete(self): """ Delete the model comparison """ return self.client._perform_empty("DELETE", "/projects/%s/modelcomparisons/%s" % (self.project_key, self.mec_id)) class DSSModelComparisonSettings(object): """ A handle on the settings of a model comparison A model comparison has: - a display name ; - a prediction type ; - a list of full ids of items to compare The prediction type can be: - BINARY_CLASSIFICATION, - REGRESSION, - MULTICLASS The full ids are: - the model id of a Lab Model, - the model id of a saved model version, - the model evaluation id of a model evaluation. Do not create this class directly, instead use :meth:`dataikuapi.dss.DSSModelComparison.get_settings` """ def __init__(self, model_comparison, settings): self.model_comparison = model_comparison self.settings = settings def get_raw(self): """ Get raw settings of a model comparison :return: the raw settings of comparison, as a dict. Modifications made to the returned object are reflected when saving :rtype: dict """ return self.settings def add_compared_item(self, full_id): """ Add an item to the list of compared items :param full_id: full id of the item (lab model, saved model version, model evaluation) to add """ if "comparedModels" not in self.settings: self.settings["comparedModels"] = [] self.settings["comparedModels"].append({ "refId": full_id }) def remove_compared_item(self, full_id): """ Remove an item from the list of compared items :param full_id: full id of the item (lab model, saved model version, model evaluation) to remove """ if not self.settings["comparedModels"]: return self.settings["comparedModels"] = list(filter(lambda x: x["refId"] != full_id, self.settings["comparedModels"])) def get_compared_items(self): """ Get the full ids of items compared in this comparison :return: the list of the full ids of compared items :rtype: list[str] """ if not self.settings["comparedModels"]: return [] return [x["refId"] for x in self.settings["comparedModels"]] @property def prediction_type(self): """ Get the prediction type of this comparison :return: str """ return self.settings["predictionType"] @prediction_type.setter def prediction_type(self, prediction_type): """ Set the prediction type of this comparison. Must be consistent with the prediction types of compared items. :param prediction_type: """ self.settings["predictionType"] = prediction_type @property def display_name(self): """ Human readable name of this comparison :return: str """ return self.settings["displayName"] @display_name.setter def display_name(self, display_name): """ Set the human readable name of this comparison :param display_name: """ self.settings["displayName"] = display_name def save(self): """ Save settings modifications """ self.model_comparison.client._perform_empty( "PUT", "/projects/%s/modelcomparisons/%s" % (self.model_comparison.project_key, self.model_comparison.mec_id), body=self.settings) dataiku-api-client-python-11.4.0/dataikuapi/dss/modelevaluationstore.py000066400000000000000000000532521440464736400263300ustar00rootroot00000000000000import json from io import BytesIO from .metrics import ComputedMetrics from .discussion import DSSObjectDiscussions from .future import DSSFuture try: basestring except NameError: basestring = str class DSSModelEvaluationStore(object): """ A handle to interact with a model evaluation store on the DSS instance. Do not create this directly, use :meth:`dataikuapi.dss.DSSProject.get_model_evaluation_store` """ def __init__(self, client, project_key, mes_id): self.client = client self.project = client.get_project(project_key) self.project_key = project_key self.mes_id = mes_id @property def id(self): return self.mes_id def get_settings(self): """ Returns the settings of this model evaluation store. :rtype: DSSModelEvaluationStoreSettings """ data = self.client._perform_json( "GET", "/projects/%s/modelevaluationstores/%s" % (self.project_key, self.mes_id)) return DSSModelEvaluationStoreSettings(self, data) ######################################################## # Misc ######################################################## def get_zone(self): """ Gets the flow zone of this model evaluation store :rtype: :class:`dataikuapi.dss.flow.DSSFlowZone` """ return self.project.get_flow().get_zone_of_object(self) def move_to_zone(self, zone): """ Moves this object to a flow zone :param object zone: a :class:`dataikuapi.dss.flow.DSSFlowZone` where to move the object """ if isinstance(zone, basestring): zone = self.project.get_flow().get_zone(zone) zone.add_item(self) def share_to_zone(self, zone): """ Share this object to a flow zone :param object zone: a :class:`dataikuapi.dss.flow.DSSFlowZone` where to share the object """ if isinstance(zone, basestring): zone = self.project.get_flow().get_zone(zone) zone.add_shared(self) def unshare_from_zone(self, zone): """ Unshare this object from a flow zone :param object zone: a :class:`dataikuapi.dss.flow.DSSFlowZone` from where to unshare the object """ if isinstance(zone, basestring): zone = self.project.get_flow().get_zone(zone) zone.remove_shared(self) def get_usages(self): """ Get the recipes referencing this model evaluation store Returns: a list of usages """ return self.client._perform_json("GET", "/projects/%s/modelevaluationstores/%s/usages" % (self.project_key, self.mes_id)) def get_object_discussions(self): """ Get a handle to manage discussions on the model evaluation store :returns: the handle to manage discussions :rtype: :class:`dataikuapi.discussion.DSSObjectDiscussions` """ return DSSObjectDiscussions(self.client, self.project_key, "MODEL_EVALUATION_STORE", self.mes_id) ######################################################## # Deletion ######################################################## def delete(self): """ Delete the model evaluation store """ return self.client._perform_empty("DELETE", "/projects/%s/modelevaluationstores/%s" % (self.project_key, self.mes_id)) ######################################################## # Model evaluations ######################################################## def list_model_evaluations(self): """ List the model evaluations in this model evaluation store. The list is sorted by ME creation date. :returns: The list of the model evaluations :rtype: list of :class:`dataikuapi.dss.modelevaluationstore.DSSModelEvaluation` """ items = self.client._perform_json("GET", "/projects/%s/modelevaluationstores/%s/evaluations/" % (self.project_key, self.mes_id)) return [DSSModelEvaluation(self, item["ref"]["evaluationId"]) for item in items] def get_model_evaluation(self, evaluation_id): """ Get a handle to interact with a specific model evaluation :param string evaluation_id: the id of the desired model evaluation :returns: A :class:`dataikuapi.dss.modelevaluationstore.DSSModelEvaluation` model evaluation handle """ return DSSModelEvaluation(self, evaluation_id) def get_latest_model_evaluation(self): """ Get a handle to interact with the latest model evaluation computed :returns: A :class:`dataikuapi.dss.modelevaluationstore.DSSModelEvaluation` model evaluation handle if the store is not empty, else None """ latest_evaluation_id = self.client._perform_text( "GET", "/projects/%s/modelevaluationstores/%s/latestEvaluationId" % (self.project_key, self.mes_id)) if not latest_evaluation_id: return None return DSSModelEvaluation(self, latest_evaluation_id) def delete_model_evaluations(self, evaluations): """ Remove model evaluations from this store """ obj = [] for evaluation in evaluations: if isinstance(evaluation, DSSModelEvaluation): obj.append(evaluation.evaluation_id) elif isinstance(evaluation, dict): obj.append(evaluation['evaluation_id']) else: obj.append(evaluation) self.client._perform_json( "DELETE", "/projects/%s/modelevaluationstores/%s/evaluations/" % (self.project_key, self.mes_id), body=obj) def build(self, job_type="NON_RECURSIVE_FORCED_BUILD", wait=True, no_fail=False): """ Starts a new job to build this model evaluation store and wait for it to complete. Raises if the job failed. .. code-block:: python job = mes.build() print("Job %s done" % job.id) :param job_type: The job type. One of RECURSIVE_BUILD, NON_RECURSIVE_FORCED_BUILD or RECURSIVE_FORCED_BUILD :param wait: wait for the build to finish before returning :param no_fail: if True, does not raise if the job failed. Valid only when wait is True :return: the :class:`dataikuapi.dss.job.DSSJob` job handle corresponding to the built job :rtype: :class:`dataikuapi.dss.job.DSSJob` """ jd = self.project.new_job(job_type) jd.with_output(self.mes_id, object_type="MODEL_EVALUATION_STORE") if wait: return jd.start_and_wait(no_fail) else: return jd.start(allowFail=not no_fail) ######################################################## # Metrics ######################################################## def get_last_metric_values(self): """ Get the metrics of the latest model evaluation built Returns: a list of metric objects and their value """ return ComputedMetrics(self.client._perform_json( "GET", "/projects/%s/modelevaluationstores/%s/metrics/last" % (self.project_key, self.mes_id))) def get_metric_history(self, metric): """ Get the history of the values of the metric on this model evaluation store Returns: an object containing the values of the metric, cast to the appropriate type (double, boolean,...) """ return self.client._perform_json( "GET", "/projects/%s/modelevaluationstores/%s/metrics/history" % (self.project_key, self.mes_id), params={'metricLookup': metric if isinstance(metric, str)or isinstance(metric, unicode) else json.dumps(metric)}) def compute_metrics(self, metric_ids=None, probes=None): """ Compute metrics on this model evaluation store. If the metrics are not specified, the metrics setup on the model evaluation store are used. """ url = "/projects/%s/modelevaluationstores/%s/actions" % (self.project_key, self.mes_id) if metric_ids is not None: return self.client._perform_json( "POST" , "%s/computeMetricsFromIds" % url, body={"metricIds" : metric_ids}) elif probes is not None: return self.client._perform_json( "POST" , "%s/computeMetrics" % url, body=probes) else: return self.client._perform_json( "POST" , "%s/computeMetrics" % url) class DSSModelEvaluationStoreSettings: """ A handle on the settings of a model evaluation store Do not create this class directly, instead use :meth:`dataikuapi.dss.DSSModelEvaluationStore.get_settings` """ def __init__(self, model_evaluation_store, settings): self.model_evaluation_store = model_evaluation_store self.settings = settings def get_raw(self): return self.settings def save(self): self.model_evaluation_store.client._perform_empty( "PUT", "/projects/%s/modelevaluationstores/%s" % (self.model_evaluation_store.project_key, self.model_evaluation_store.mes_id), body=self.settings) class DSSModelEvaluation: """ A handle on a model evaluation Do not create this class directly, instead use :meth:`dataikuapi.dss.DSSModelEvaluationStore.get_model_evaluation` """ def __init__(self, model_evaluation_store, evaluation_id): self.model_evaluation_store = model_evaluation_store self.client = model_evaluation_store.client # unpack some fields self.evaluation_id = evaluation_id self.project_key = model_evaluation_store.project_key self.mes_id = model_evaluation_store.mes_id def get_full_info(self): """ Retrieve the model evaluation with its performance data :return: the model evaluation full info, as a :class:`dataikuapi.dss.DSSModelEvaluationInfo` """ data = self.client._perform_json( "GET", "/projects/%s/modelevaluationstores/%s/evaluations/%s" % (self.project_key, self.mes_id, self.evaluation_id)) return DSSModelEvaluationFullInfo(self, data) def get_full_id(self): return "ME-{}-{}-{}".format(self.project_key, self.mes_id, self.evaluation_id) def delete(self): """ Remove this model evaluation """ obj = [self.evaluation_id] self.client._perform_json( "DELETE", "/projects/%s/modelevaluationstores/%s/evaluations/" % (self.project_key, self.mes_id), body=obj) @property def full_id(self): return "ME-%s-%s-%s"%(self.project_key, self.mes_id, self.evaluation_id) def compute_data_drift(self, reference=None, data_drift_params=None, wait=True): """ Compute data drift against a reference model or model evaluation. The reference is determined automatically unless specified. :param reference: saved model version (full ID or DSSTrainedPredictionModelDetails) or model evaluation (full ID or DSSModelEvaluation) to use as reference (optional) :type reference: Union[str, DSSModelEvaluation, DSSTrainedPredictionModelDetails] :param data_drift_params: data drift computation settings as a :class:`dataikuapi.dss.modelevaluationstore.DataDriftParams` (optional) :type data_drift_params: DataDriftParams :param wait: data drift computation settings (optional) :returns: a :class:`dataikuapi.dss.modelevaluationstore.DataDriftResult` containing data drift analysis results if `wait` is `True`, or a :class:`~dataikuapi.dss.future.DSSFuture` handle otherwise """ if hasattr(reference, 'full_id'): reference = reference.full_id if data_drift_params: data_drift_params = data_drift_params.data future_response = self.client._perform_json( "POST", "/projects/%s/modelevaluationstores/%s/evaluations/%s/computeDataDrift" % (self.project_key, self.mes_id, self.evaluation_id), body={ "referenceId": reference, "dataDriftParams": data_drift_params }) future = DSSFuture(self.client, future_response.get('jobId', None), future_response, result_wrapper=DataDriftResult) return future.wait_for_result() if wait else future def get_metrics(self): """ Get the metrics for this model evaluation. Metrics must be understood here as Metrics in DSS Metrics & Checks :return: the metrics, as a JSON object """ return self.client._perform_json( "GET", "/projects/%s/modelevaluationstores/%s/evaluations/%s/metrics" % (self.project_key, self.mes_id, self.evaluation_id)) def get_sample_df(self): """ Get the sample of the evaluation dataset on which the evaluation was performed :return: the sample content, as a :class:`pandas.DataFrame` """ buf = BytesIO() with self.client._perform_raw( "GET", "/projects/%s/modelevaluationstores/%s/evaluations/%s/sample" % (self.project_key, self.mes_id, self.evaluation_id) ).raw as f: buf.write(f.read()) schema_txt = self.client._perform_raw( "GET", "/projects/%s/modelevaluationstores/%s/evaluations/%s/schema" % (self.project_key, self.mes_id, self.evaluation_id) ).text schema = json.loads(schema_txt) import pandas as pd return pd.read_csv(BytesIO(buf.getvalue()), compression='gzip', sep='\t', header=None, names=[c["name"] for c in schema["columns"]]) class DSSModelEvaluationFullInfo: """ A handle on the full information on a model evaluation. Includes information such as the full id of the evaluated model, the evaluation params, the performance and drift metrics, if any, etc. Do not create this class directly, instead use :meth:`dataikuapi.dss.DSSModelEvaluation.get_full_info` """ def __init__(self, model_evaluation, full_info): self.model_evaluation = model_evaluation self.full_info = full_info self.metrics = self.full_info["metrics"] # type: dict """The performance and data drift metric, if any.""" self.creation_date = self.full_info["evaluation"]["created"] # type: int """The date and time of the creation of the model evaluation, as an epoch.""" self.full_id = self.full_info["evaluation"]["ref"]["fullId"] # type: str if "modelRef" in self.full_info["evaluation"]: self.model_full_id = self.full_info["evaluation"]["modelRef"]["fullId"] # type: str else: self.model_full_id = None self.prediction_type = self.full_info["evaluation"]["predictionType"] # type: str self.prediction_variable = self.full_info["evaluation"]["predictionVariable"] # type: str self.target_variable = self.full_info["evaluation"].get("targetVariable") # type: str self.user_meta = self.full_info["evaluation"]["userMeta"] # type: dict """The user-accessible metadata (name, labels) Returns the original object, not a copy. Changes to the returned object are persisted to DSS by calling :meth:`save_user_meta`.""" def get_raw(self): return self.full_info def save_user_meta(self): return self.model_evaluation.client._perform_text( "PUT", "/projects/%s/modelevaluationstores/%s/evaluations/%s/user-meta" % (self.model_evaluation.project_key, self.model_evaluation.mes_id, self.model_evaluation.evaluation_id), body=self.user_meta) class DataDriftParams(object): """ Object that represents parameters for data drift computation. Do not create this object directly, use :meth:`dataikuapi.dss.modelevaluationstore.DataDriftParams.from_params` instead. """ def __init__(self, data): self.data = data def __repr__(self): return u"{}({})".format(self.__class__.__name__, self.data) @staticmethod def from_params(per_column_settings, nb_bins=10, compute_histograms=True, confidence_level=0.95): """ Creates parameters for data drift computation from columns, number of bins, compute histograms and confidence level :param dict per_column_settings: A dict representing the per column settings. You should use a :class:`~dataikuapi.dss.modelevaluationstore.PerColumnDriftParamBuilder` to build it. :param int nb_bins: (optional) Nb. bins in histograms (apply to all columns) - default: 10 :param bool compute_histograms: (optional) Enable/disable histograms - default: True :param float confidence_level: (optional) Used to compute confidence interval on drift's model accuracy - default: 0.95 :rtype: :class:`dataikuapi.dss.modelevaluationstore.DataDriftParams` """ return DataDriftParams({ "columns": per_column_settings, "nbBins": nb_bins, "computeHistograms": compute_histograms, "confidenceLevel": confidence_level }) class PerColumnDriftParamBuilder(object): """ Builder for a map of per column drift params settings. Used as a helper before computing data drift to build columns param expected in :meth:`dataikuapi.dss.modelevaluationstore.DataDriftParams.from_params`. """ def __init__(self): self.columns = {} def build(self): """Returns the built dict for per column drift params settings""" return self.columns def with_column_drift_param(self, name, handling="AUTO", enabled=True): """ Sets the drift params settings for given column name. :param: string name: The name of the column :param: string handling: (optional) The column type, should be either NUMERICAL, CATEGORICAL or AUTO (default: AUTO) :param: bool enabled: (optional) False means the column is ignored in drift computation (default: True) """ self.columns[name] = { "handling": handling, "enabled": enabled } return self class DataDriftResult(object): """ A handle on the data drift result of a model evaluation. Do not create this class directly, instead use :meth:`dataikuapi.dss.DSSModelEvaluation.compute_data_drift` """ def __init__(self, data): self.data = data self.drift_model_result = DriftModelResult(self.data["driftModelResult"]) """Drift analysis based on drift modeling.""" self.univariate_drift_result = UnivariateDriftResult(self.data["univariateDriftResult"]) """Per-column drift analysis based on pairwise comparison of distributions.""" self.per_column_settings = [ColumnSettings(cs) for cs in self.data["perColumnSettings"]] """Information about column handling that has been used (errors, types, etc).""" def get_raw(self): """ :return: the raw data drift result :rtype: dict """ return self.data class DriftModelResult(object): """ A handle on the drift model result. Do not create this class directly, instead use :attr:`dataikuapi.dss.modelevaluationstore.DataDriftResult.drift_model_result` """ def __init__(self, data): self.data = data self.drift_model_accuracy = DriftModelAccuracy(self.data["driftModelAccuracy"]) self.feature_drift_importance = self.data["driftVersusImportance"] # type: dict def get_raw(self): """ :return: the raw drift model result :rtype: dict """ return self.data class UnivariateDriftResult(object): """ A handle on the univariate data drift. Do not create this class directly, instead use :attr:`dataikuapi.dss.modelevaluationstore.DataDriftResult.univariate_drift_result` """ def __init__(self, data): self.data = data self.per_column_drift_data = self.data["columns"] # type: dict """Drift data per column, as a dict of column name -> drift data.""" def get_raw(self): """ :return: the raw univariate data drift :rtype: dict """ return self.data class ColumnSettings(object): """ A handle on column handling information. Do not create this class directly, instead use :meth:`dataikuapi.dss.modelevaluationstore.DataDriftResult.get_per_column_settings` """ def __init__(self, data): self.data = data self.name = self.data["name"] # type: str self.actual_column_handling = self.data["actualHandling"] # type: str """The actual column handling (either forced via drift params or inferred from model evaluation preprocessings). It can be any of NUMERICAL, CATEGORICAL, or IGNORED.""" self.default_column_handling = self.data["defaultHandling"] # type: str """The default column handling (based on model evaluation preprocessing only). It can be any of NUMERICAL, CATEGORICAL, or IGNORED.""" self.error_message = self.data.get("errorMessage", None) def get_raw(self): """ :return: the raw column handling information :rtype: dict """ return self.data class DriftModelAccuracy(object): """ A handle on the drift model accuracy. Do not create this class directly, instead use :attr:`dataikuapi.dss.modelevaluationstore.DriftModelResult.drift_model_accuracy` """ def __init__(self, data): self.data = data self.value = self.data["value"] # type: float self.lower_confidence_interval = self.data["lower"] # type: float self.upper_confidence_interval = self.data["upper"] # type: float self.pvalue = self.data["pvalue"] # type: float def get_raw(self): """ :return: the drift model accuracy data :rtype: dict """ return self.data dataiku-api-client-python-11.4.0/dataikuapi/dss/notebook.py000066400000000000000000000057061440464736400237040ustar00rootroot00000000000000from .discussion import DSSObjectDiscussions class DSSNotebook(object): """ A Python/R/Scala notebook on the DSS instance. .. attention:: Deprecated. Use :class:`dataikuapi.dss.jupyternotebook.DSSJupyterNotebook` """ def __init__(self, client, project_key, notebook_name, state=None): self.client = client self.project_key = project_key self.notebook_name = notebook_name self.state = state self.state_is_peek = True def unload(self, session_id=None): """ Stop this Jupyter notebook and release its resources. .. attention:: Deprecated. Use :class:`dataikuapi.dss.jupyternotebook.DSSJupyterNotebook` """ state = self.get_state() if state is None: raise Exception("Notebook isn't running") if state.get('activeSessions', None) is None: raise Exception("Notebook isn't running") if len(state['activeSessions']) == 0: raise Exception("Notebook isn't running") if session_id is None: if len(state['activeSessions']) > 1: raise Exception("Several sessions of the notebook are running, choose one") else: session_id = state['activeSessions'][0].get('sessionId', None) return self.client._perform_json("DELETE", "/projects/%s/notebooks/" % self.project_key, params={'notebookName' : self.notebook_name, 'sessionId' : session_id}) def get_state(self): """ Get the metadata associated to this Jupyter notebook. .. attention:: Deprecated. Use :class:`dataikuapi.dss.jupyternotebook.DSSJupyterNotebook` """ if self.state is None: self.state = self.client._perform_json("GET", "/projects/%s/notebooks/" % self.project_key, params={'notebookName' : self.notebook_name}) return self.state def get_sessions(self): """ Get the list of running sessions of this Jupyter notebook. .. attention:: Deprecated. Use :class:`dataikuapi.dss.jupyternotebook.DSSJupyterNotebook` """ state = self.get_state() if state is None: raise Exception("Notebook isn't running") if state.get('activeSessions', None) is None: raise Exception("Notebook isn't running") return state['activeSessions'] ######################################################## # Discussions ######################################################## def get_object_discussions(self): """ Get a handle to manage discussions on the notebook. .. attention:: Deprecated. Use :class:`dataikuapi.dss.jupyternotebook.DSSJupyterNotebook` :return: the handle to manage discussions :rtype: :class:`dataikuapi.dss.discussion.DSSObjectDiscussions` """ return DSSObjectDiscussions(self.client, self.project_key, "JUPYTER_NOTEBOOK", self.notebook_name) dataiku-api-client-python-11.4.0/dataikuapi/dss/plugin.py000066400000000000000000000633471440464736400233670ustar00rootroot00000000000000from ..utils import DataikuException from .future import DSSFuture class DSSPluginSettings(object): """ The settings of a plugin. .. important:: Do not instantiate directly, use :meth:`DSSPlugin.get_settings`. """ def __init__(self, client, plugin_id, settings): self.client = client self.plugin_id = plugin_id self.settings = settings def get_raw(self): """ Get the raw settings object. .. note:: This method returns a reference to the settings, not a copy. Changing values in the reference then calling :meth:`save()` results in these changes being saved. :return: the settings as a dict. Top-level fields are: * **permissions** : list of permissions, per group, as a list of dict, each with fields * **group** : name of a DSS group * **admin** : whether the group can administer the plugin (edit settings and presets) * **defaultPermission** : default permissions for the plugin, as a dict. The dict has the same fields as in **permissions**, minus the **group** * **detailsNotVisible** : if True, only **presets** and **accessibleParameterSetDescs** are filled * **config** : plugin-level settings (deprecated) * **codeEnvName** : name of the code env currently associated to the plugin * **accessibleParameterSetDescs** : list of parameter set definitions that the user can make presets for. Each definition is a dict (see :meth:`DSSPluginParameterSet.desc()`) * **presets** : list of presets (see :meth:`DSSPluginParameterSet.presets()`) * **parameterSets** : list of settings for the parameter sets, as a list of dict (see :meth:`DSSPluginParameterSet.settings()`) * **appTemplates** : list of settings for the Dataiku App templates in this plugin, as a list of dict. Each dict has keys: * **name** : name of the template * **remapping** : settings to remap connections and code envs upon instantiation of the app template, as a dict. The dict has fields: * **connections** : list of remappings for connections, as a list of dict with fields **source** and **target**, applied to connection names * **codeEnvs** : list of remappings for code envs, as a list of dict with fields **source** and **target**, applied to code env names :rtype: dict """ return self.settings def list_parameter_set_names(self): """ List the names of the parameter sets defined in this plugin. :rtype: list[string] """ return [d["id"] for d in self.settings.get("accessibleParameterSetDescs", [])] def get_parameter_set(self, parameter_set_name): """ Get a parameter set in this plugin. :param string parameter_set_name: name of the parameter set :return: a handle on the parameter set :rtype: :class:`DSSPluginParameterSet` """ desc = None data = None for ps in self.settings.get("accessibleParameterSetDescs", []): if ps["id"] == parameter_set_name: desc = ps for ps in self.settings.get("parameterSets", []): if ps["name"] == parameter_set_name: data = ps if desc is None or data is None: return None return DSSPluginParameterSet(desc, data, self.settings["presets"]) def set_code_env(self, code_env_name): """ Set the name of the code env to use for this plugin. :param string code_env_name: name of a code env """ self.settings["codeEnvName"] = code_env_name def save(self): """ Save the settings to DSS. """ self.client._perform_empty("POST", "/plugins/%s/settings" % (self.plugin_id), body=self.settings) class DSSPluginParameterSet(object): """ A parameter set in a plugin. The values in this class can be modified directly, and changes will be taken into account when calling :meth:`DSSPluginSettings.save()`. """ def __init__(self, desc, settings, presets): self._desc = desc self._settings = settings self._presets = presets @property def desc(self): """ Get the raw definition of the parameter set. :return: a parameter set definition, as a dict of: * **id** : name of the parameter set * **ownerPluginId** : name of the plugin * **elementType** : unique identifier of the parameter set on the DSS instance, built from plugin and parameter set names * **folderName** : name of the folder holding the parameter set files, inside the plugin directory * **desc** : the parameter set's contents as a dict. See `the doc `_ :rtype: dict """ return self.desc @property def settings(self): """ Get the settings of the parameter set. These settings control the behavior of the parameter set, and comprise notably the permissions, but not the presets of this parameter set. :return: the settings of the parameter set, as a dict of: * **name** : name of the parameter set in the plugin * **type** : identifier of the parameter set (unique on the DSS instance) * **permissions** : list of permissions, per group, as a list of dict. Unless given access via **permissions**, only admins of the plugin can add presets for the parameter set. Each dict has fields: * **group** : name of a DSS group * **definableInline** : whether presets of this parameter set can be defined directly in the forms of the plugin components that use them * **definableAtProjectLevel** : whether project-level presets can be made of this parameter set * **defaultPermission** : default permissions for the parameter set, as a dict. The dict has the same fields as in **permissions**, minus the **group** :rtype: dict """ return self._settings def list_preset_names(self): """ List the names of the presets of this parameter set. :rtype: list[string] """ return [p["name"] for p in self._presets if p["type"] == self._settings["type"]] def get_preset(self, preset_name): """ Get a preset of this parameter set. :param string preset_name: name of a preset :return: a preset definition, as a dict of: * **type** : type of the parameter set this preset belongs to (type being the instance-wide unique identifier) * **name** : preset name * **description** : preset description * **owner** : login of the owner of the preset * **permissions** : list of permissions, per group, as a list of dict, each with fields * **group** : name of a DSS group * **use** : whether the group can use the preset * **defaultPermission** : default permissions for the plugin, as a dict. The dict has the same fields as in **permissions**, minus the **group** * **config** : values of the preset, as a dict. Each key of the dict is the name of some parameter in the definition of the parameter set * **pluginConfig** : plugin-level values of the preset, as a dict. These values can only be set in the plugin's settings, not in the components using the preset. :rtype: dict """ for p in self._presets: if p["name"] == preset_name and p["type"] == self._settings["type"]: return p return None def delete_preset(self, preset_name): """ Remove a preset from this plugin's settings :param string preset_name: name for the preset to remove """ preset = self.get_preset(preset_name) if preset is not None: self._presets.remove(preset) def create_preset(self, preset_name): """ Create a new preset of this parameter set in the plugin settings. :param string preset_name: name for the preset to create :return: a preset definition, as a dict (see :meth:`get_preset()`) :rtype: dict """ for p in self._presets: if p["name"] == preset_name: raise Exception("A preset of the same name already exists") # since self._presets is directly the list from the plugin settings, this adds the preset to the plugin self._presets.append({"name":preset_name, "type":self._settings["type"], "config":{}, "pluginConfig":{}, "defaultPermissions":{}, "permissions":[]}) return self.get_preset(preset_name) class DSSPlugin(object): """ A plugin on the DSS instance. .. important:: Do not instantiate directly, use :meth:`dataikuapi.DSSClient.get_plugin()` """ def __init__(self, client, plugin_id): self.client = client self.plugin_id = plugin_id ######################################################## # Settings ######################################################## def get_settings(self): """ Get the plugin-level settings. .. note:: This call requires an API key with either: * DSS admin permissions * permission to develop plugins * tied to a user with admin privileges on the plugin :return: a handle on the settings :rtype: :class:`DSSPluginSettings` """ settings = self.client._perform_json("GET", "/plugins/%s/settings" % (self.plugin_id)) return DSSPluginSettings(self.client, self.plugin_id, settings) ######################################################## # Code env ######################################################## def create_code_env(self, python_interpreter=None, conda=False): """ Start the creation of the code env of the plugin. .. note:: This call requires an API key with either: * DSS admin permissions * permission to develop plugins * tied to a user with admin privileges on the plugin If not passing any value to **python_interpreter**, the default defined by the plugin will be used. Usage example: .. code-block:: python # create a default code env for the plugin plugin = client.get_plugin('the-plugin-id') future = plugin.create_code_env() creation = future.wait_for_result() # take the name of the new code env env_name = creation["envName"] # set it as the current plugin code env settings = plugin.get_settings() settings.set_code_env(env_name) settings.save() :param string python_interpreter: which version of python to use. Possible values: PYTHON27, PYTHON34, PYTHON35, PYTHON36, PYTHON37, PYTHON38, PYTHON39, PYTHON310, PYTHON311 :param boolean conda: if True use conda to create the code env, if False use virtualenv and pip. :return: a handle on the operation :rtype: :class:`dataikuapi.dss.future.DSSFuture` """ ret = self.client._perform_json("POST", "/plugins/%s/code-env/actions/create" % (self.plugin_id), body={ "deploymentMode" : "PLUGIN_MANAGED", "conda": conda, "pythonInterpreter": python_interpreter }) return DSSFuture.from_resp(self.client, ret) def update_code_env(self): """ Start an update of the code env of the plugin. .. note:: This call requires an API key with either: * DSS admin permissions * permission to develop plugins * tied to a user with admin privileges on the plugin Usage example: .. code-block:: python # update the plugin code env after updating the plugin plugin = client.get_plugin('the-plugin-id') future = plugin.update_code_env() future.wait_for_result() :return: a handle on the operation :rtype: :class:`dataikuapi.dss.future.DSSFuture` """ ret = self.client._perform_json("POST", "/plugins/%s/code-env/actions/update" % (self.plugin_id)) return DSSFuture.from_resp(self.client, ret) ######################################################## # Plugin update ######################################################## def update_from_zip(self, fp): """ Update the plugin from a plugin archive (as a file object). .. note:: This call requires an API key with either: * DSS admin permissions * permission to develop plugins * tied to a user with admin privileges on the plugin :param file-like fp: A file-like object pointing to a plugin archive zip """ files = {'file': fp } self.client._perform_json("POST", "/plugins/%s/actions/updateFromZip" % (self.plugin_id), files=files) def update_from_store(self): """ Update the plugin from the Dataiku plugin store. .. note:: This call requires an API key with either: * DSS admin permissions * permission to develop plugins * tied to a user with admin privileges on the plugin Usage example: .. code-block:: python # update a plugin that was installed from the store plugin = client.get_plugin("my-plugin-id") future = plugin.update_from_store() future.wait_for_result() :return: a handle on the operation :rtype: :class:`dataikuapi.dss.future.DSSFuture` """ ret = self.client._perform_json("POST", "/plugins/%s/actions/updateFromStore" % (self.plugin_id)) return DSSFuture.from_resp(self.client, ret) def update_from_git(self, repository_url, checkout = "master", subpath=None): """ Updates the plugin from a Git repository. .. note:: DSS must be setup to allow access to the repository. .. note:: This call requires an API key with either: * DSS admin permissions * permission to develop plugins * tied to a user with admin privileges on the plugin Usage example: .. code-block:: python # update a plugin that was installed from git plugin = client.get_plugin("my-plugin-id") future = plugin.update_from_git("git@github.com:myorg/myrepo") future.wait_for_result() :param string repository_url: URL of a Git remote :param string checkout: branch/tag/SHA1 to commit. For example "master" :param string subpath: Optional, path within the repository to use as plugin. Should contain a 'plugin.json' file :return: a handle on the operation :rtype: :class:`dataikuapi.dss.future.DSSFuture` """ ret = self.client._perform_json("POST", "/plugins/%s/actions/updateFromGit" % (self.plugin_id), body={ "gitRepositoryUrl": repository_url, "gitCheckout" : checkout, "gitSubpath": subpath }) return DSSFuture.from_resp(self.client, ret) ######################################################## # Plugin uninstall/delete ######################################################## def list_usages(self, project_key=None): """ Get the list of usages of the plugin. .. note:: This call requires an API key with either: * DSS admin permissions * permission to develop plugins * tied to a user with admin privileges on the plugin :param string project_key: optional key of project where to look for usages. Default is None and looking in all projects. :rtype: :class:`DSSPluginUsages` """ return DSSPluginUsages( self.client._perform_json("GET", "/plugins/{pluginId}/actions/listUsages".format(pluginId=self.plugin_id), params={"projectKey": project_key}) ) def delete(self, force=False): """ Delete a plugin. .. note:: This call requires an API key with either: * DSS admin permissions * permission to develop plugins * tied to a user with admin privileges on the plugin :param boolean force: if True, plugin will be deleted even if usages are found or errors occurred during usages analysis. Default: False. :rtype: :class:`dataikuapi.dss.future.DSSFuture` """ params = { "force": force } ret = self.client._perform_json("POST", "/plugins/{pluginId}/actions/delete".format(pluginId=self.plugin_id), body=params) return DSSFuture.from_resp(self.client, ret) ######################################################## # Managing the dev plugin's contents ######################################################## def list_files(self): """ Get the hierarchy of files in the plugin. .. note:: Dev plugins only .. note:: This call requires an API key with either: * DSS admin permissions * permission to develop plugins * tied to a user with admin privileges on the plugin :return: list of files or directories, each one a dict. Directories have a **children** field for recursion. Each dict has fields **name** and **path** (the path from the root of the plugin files) :rtype: dict """ return self.client._perform_json("GET", "/plugins/%s/contents" % (self.plugin_id)) def get_file(self, path): """ Get a file from the plugin folder. .. note:: Dev plugins only .. note:: This call requires an API key with either: * DSS admin permissions * permission to develop plugins * tied to a user with admin privileges on the plugin Usage example: .. code-block:: python # read the code env desc of a plugin plugin = client.get_plugin("my-plugin-name") with plugin.get_file('code-env/python/desc.json') as fp: desc = json.load(fp) :param string path: the path of the file, relative to the root of the plugin :return: the file's content :rtype: file-like """ return self.client._perform_raw("GET", "/plugins/%s/contents/%s" % (self.plugin_id, path)).raw def put_file(self, path, f): """ Update a file in the plugin folder. .. note:: Dev plugins only .. note:: This call requires an API key with either: * DSS admin permissions * permission to develop plugins * tied to a user with admin privileges on the plugin :param file-like f: the file contents, as a file-like object :param string path: the path of the file, relative ot the root of the plugin """ file_name = path.split('/')[-1] data = f.read() # eat it all, because making it work with a path variable and a MultifilePart in swing looks complicated self.client._perform_empty("POST", "/plugins/%s/contents/%s" % (self.plugin_id, path), raw_body=data) def rename_file(self, path, new_name): """ Rename a file/folder in the plugin. .. note:: Dev plugins only :param string path: the path of the file/folder, relative ot the root of the plugin :param string new_name: the new name of the file/folder """ self.client._perform_empty("POST", "/plugins/%s/contents-actions/rename" % self.plugin_id, body={"oldPath": path, "newName": new_name}) def move_file(self, path, new_path): """ Move a file/folder in the plugin. .. note:: Dev plugins only .. note:: This call requires an API key with either: * DSS admin permissions * permission to develop plugins * tied to a user with admin privileges on the plugin :param string path: the path of the file/folder, relative ot the root of the plugin :param string new_path: the new path relative at the root of the plugin """ self.client._perform_empty("POST", "/plugins/%s/contents-actions/move" % self.plugin_id, body={"oldPath": path, "newPath": new_path}) class DSSPluginUsage(object): """ Information on a usage of an element of a plugin. .. important:: Do no instantiate directly, use :meth:`dataikuapi.dss.plugin.DSSPlugin.list_usages()` """ def __init__(self, data): self._data = data @property def element_kind(self): """ Get the type of the plugin component. :return: a kind of plugin component, like 'python-clusters', 'python-connectors', 'python-fs-providers', 'webapps', ... :rtype: string """ return self._data["elementKind"] @property def element_type(self): """ Get the identifier of the plugin component. :rtype: string """ return self._data["elementType"] @property def object_id(self): """ Get the identifier of the object using the plugin component. :rtype: string """ return self._data["objectId"] @property def object_type(self): """ Get the type of the object using the plugin component. :return: a type of DSS object, like 'CLUSTER', 'DATASET', 'RECIPE', ... :rtype: string """ return self._data["objectType"] @property def project_key(self): """ Get the project key of the object using the plugin component. :rtype: string """ return self._data["projectKey"] class DSSMissingType(object): """ Information on a type not found while analyzing usages of a plugin. Missing types can occur when plugins stop defining a given component, for example during development, and DSS object still use the now-removed component. .. important:: Do no instantiate directly, use :meth:`dataikuapi.dss.plugin.DSSPlugin.list_usages()` """ def __init__(self, data): self._data = data @property def missing_type(self): """ Get the type of the plugin component. :rtype: string """ return self._data["missingType"] @property def object_id(self): """ Get the identifier of the object using the plugin component. :rtype: string """ return self._data["objectId"] @property def object_type(self): """ Get the type of the object using the plugin component. :return: a type of DSS object, like 'CLUSTER', 'DATASET', 'RECIPE', ... :rtype: string """ return self._data["objectType"] @property def project_key(self): """ Get the project key of the object using the plugin component :rtype: string """ return self._data["projectKey"] class DSSPluginUsages(object): """ Information on the usages of a plugin. .. important:: Do no instantiate directly, use :meth:`dataikuapi.dss.plugin.DSSPlugin.list_usages()` Some custom types may not be found during usage analysis, typically when a plugin was removed but is still used. This prevents some detailed analysis and may hide some uses. This information is provided in :meth:`missing_types()`. """ def __init__(self, data): """ Initialize a DSSPluginUsages from a dict of its properties :param dict data: the usages as json dict """ self._data = data self._usages = [] self._missing_types = [] for json_usage in data.get("usages", []): self._usages.append(DSSPluginUsage(json_usage)) for json_missing_type in data.get("missingTypes"): self._missing_types.append(DSSMissingType(json_missing_type)) def get_raw(self): """ Get the raw plugin usages. :return: a summary of the usages, as a dict with fields **usages** and **missingTypes**. :rtype: dict """ return self._data @property def usages(self): """ Get the list of usages of components of the plugin. :return: list of usages, each a :class:`DSSPluginUsage` :rtype: list """ return self._usages @property def missing_types(self): """ Get the list of usages of missing components of the plugin. :return: list of missing component types, each a :class:`DSSMissingType` :rtype: list """ return self._missing_types def maybe_used(self): """ Whether the plugin maybe in use. :return: True if usages were found, False if errors were encountered during analysis :rtype: boolean """ return not (not self.usages and not self.missing_types) dataiku-api-client-python-11.4.0/dataikuapi/dss/project.py000066400000000000000000003072171440464736400235340ustar00rootroot00000000000000import os.path as osp import warnings from . import recipe from .analysis import DSSAnalysis from .apiservice import DSSAPIService, DSSAPIServiceListItem from .app import DSSAppManifest from .codestudio import DSSCodeStudioObject, DSSCodeStudioObjectListItem from .continuousactivity import DSSContinuousActivity from .dataset import DSSDataset, DSSDatasetListItem, DSSManagedDatasetCreationHelper from .discussion import DSSObjectDiscussions from .flow import DSSProjectFlow from .future import DSSFuture from .job import DSSJob, DSSJobWaiter from .jupyternotebook import DSSJupyterNotebook, DSSJupyterNotebookListItem from .macro import DSSMacro from .managedfolder import DSSManagedFolder from .ml import DSSMLTask, DSSMLTaskQueues from .mlflow import DSSMLflowExtension from .modelcomparison import DSSModelComparison from .modelevaluationstore import DSSModelEvaluationStore from .notebook import DSSNotebook from .projectlibrary import DSSLibrary from .recipe import DSSRecipeListItem, DSSRecipe from .savedmodel import DSSSavedModel from .scenario import DSSScenario, DSSScenarioListItem from .streaming_endpoint import DSSStreamingEndpoint, DSSStreamingEndpointListItem, \ DSSManagedStreamingEndpointCreationHelper from .webapp import DSSWebApp, DSSWebAppListItem from .wiki import DSSWiki from ..dss_plugin_mlflow import MLflowHandle class DSSProject(object): """ A handle to interact with a project on the DSS instance. .. important:: Do not create this class directly, instead use :meth:`dataikuapi.DSSClient.get_project` """ def __init__(self, client, project_key): self.client = client self.project_key = project_key def get_summary(self): """ Returns a summary of the project. The summary is a read-only view of some of the state of the project. You cannot edit the resulting dict and use it to update the project state on DSS, you must use the other more specific methods of this :class:`dataikuapi.dss.project.DSSProject` object :returns: a dict containing a summary of the project. Each dict contains at least a **projectKey** field :rtype: dict """ return self.client._perform_json("GET", "/projects/%s" % self.project_key) def get_project_folder(self): """ Get the folder containing this project :rtype: :class:`dataikuapi.dss.projectfolder.DSSProjectFolder` """ root = self.client.get_root_project_folder() def rec(pf): if self.project_key in pf.list_project_keys(): return pf else: for spf in pf.list_child_folders(): found_in_child = rec(spf) if found_in_child: return found_in_child return None found_in = rec(root) if found_in: return found_in else: return root def move_to_folder(self, folder): """ Moves this project to a project folder :param folder: destination folder :type folder: :class:`dataikuapi.dss.projectfolder.DSSProjectFolder` """ current_folder = self.get_project_folder() current_folder.move_project_to(self.project_key, folder) ######################################################## # Project deletion ######################################################## def delete(self, clear_managed_datasets=False, clear_output_managed_folders=False, clear_job_and_scenario_logs=True, **kwargs): """ Delete the project .. attention:: This call requires an API key with admin rights :param bool clear_managed_datasets: Should the data of managed datasets be cleared (defaults to **False**) :param bool clear_output_managed_folders: Should the data of managed folders used as outputs of recipes be cleared (defaults to **False**) :param bool clear_job_and_scenario_logs: Should the job and scenario logs be cleared (defaults to **True**) """ # For backwards compatibility if 'drop_data' in kwargs and kwargs['drop_data']: clear_managed_datasets = True return self.client._perform_empty( "DELETE", "/projects/%s" % self.project_key, params={ "clearManagedDatasets": clear_managed_datasets, "clearOutputManagedFolders": clear_output_managed_folders, "clearJobAndScenarioLogs": clear_job_and_scenario_logs }) ######################################################## # Project export ######################################################## def get_export_stream(self, options=None): """ Return a stream of the exported project .. warning:: You need to close the stream after download. Failure to do so will result in the DSSClient becoming unusable. :param dict options: Dictionary of export options (defaults to **{}**). The following options are available: - **exportUploads** (boolean): Exports the data of Uploaded datasets (default to **False**) - **exportManagedFS** (boolean): Exports the data of managed Filesystem datasets (default to **False**) - **exportAnalysisModels** (boolean): Exports the models trained in analysis (default to **False**) - **exportSavedModels** (boolean): Exports the models trained in saved models (default to **False**) - **exportManagedFolders** (boolean): Exports the data of managed folders (default to **False**) - **exportAllInputDatasets** (boolean): Exports the data of all input datasets (default to **False**) - **exportAllDatasets** (boolean): Exports the data of all datasets (default to **False**) - **exportAllInputManagedFolders** (boolean): \ Exports the data of all input managed folders (default to **False**) - **exportGitRepository** (boolean): Exports the Git repository history (default to **False**) - **exportInsightsData** (boolean): Exports the data of static insights (default to **False**) :returns: a stream of the export archive :rtype: file-like object """ if options is None: options = {} return self.client._perform_raw( "POST", "/projects/%s/export" % self.project_key, body=options).raw def export_to_file(self, path, options=None): """ Export the project to a file :param str path: the path of the file in which the exported project should be saved :param dict options: Dictionary of export options (defaults to **{}**). The following options are available: * **exportUploads** (boolean): Exports the data of Uploaded datasets (default to **False**) * **exportManagedFS** (boolean): Exports the data of managed Filesystem datasets (default to **False**) * **exportAnalysisModels** (boolean): Exports the models trained in analysis (default to **False**) * **exportSavedModels** (boolean): Exports the models trained in saved models (default to **False**) * **exportModelEvaluationStores** (boolean): Exports the evaluation stores (default to **False**) * **exportManagedFolders** (boolean): Exports the data of managed folders (default to **False**) * **exportAllInputDatasets** (boolean): Exports the data of all input datasets (default to **False**) * **exportAllDatasets** (boolean): Exports the data of all datasets (default to **False**) * **exportAllInputManagedFolders** (boolean): \ Exports the data of all input managed folders (default to **False**) * **exportGitRepository** (boolean): Exports the Git repository history (default to **False**) * **exportInsightsData** (boolean): Exports the data of static insights (default to **False**) """ if options is None: options = {} with open(path, 'wb') as f: export_stream = self.client._perform_raw( "POST", "/projects/%s/export" % self.project_key, body=options) for chunk in export_stream.iter_content(chunk_size=32768): if chunk: f.write(chunk) f.flush() ######################################################## # Project duplicate ######################################################## def duplicate(self, target_project_key, target_project_name, duplication_mode="MINIMAL", export_analysis_models=True, export_saved_models=True, export_git_repository=True, export_insights_data=True, remapping=None, target_project_folder=None): """ Duplicate the project :param str target_project_key: The key of the new project :param str target_project_name: The name of the new project :param str duplication_mode: can be one of the following values: MINIMAL, SHARING, FULL, NONE (defaults to **MINIMAL**) :param bool export_analysis_models: (defaults to **True**) :param bool export_saved_models: (defaults to **True**) :param bool export_git_repository: (defaults to **True**) :param bool export_insights_data: (defaults to **True**) :param dict remapping: dict of connections to be remapped for the new project (defaults to **{}**) :param target_project_folder: the project folder where to put the duplicated project (defaults to **None**) :type target_project_folder: A :class:`dataikuapi.dss.projectfolder.DSSProjectFolder` :returns: A dict containing the original and duplicated project's keys :rtype: dict """ if remapping is None: remapping = {} obj = { "targetProjectName": target_project_name, "targetProjectKey": target_project_key, "duplicationMode": duplication_mode, "exportAnalysisModels": export_analysis_models, "exportSavedModels": export_saved_models, "exportGitRepository": export_git_repository, "exportInsightsData": export_insights_data, "remapping": remapping } if target_project_folder is not None: obj["targetProjectFolderId"] = target_project_folder.project_folder_id ref = self.client._perform_json("POST", "/projects/%s/duplicate/" % self.project_key, body=obj) return ref ######################################################## # Project infos ######################################################## def get_metadata(self): """ Get the metadata attached to this project. The metadata contains label, description checklists, tags and custom metadata of the project. .. note:: For more information on available metadata, please see https://doc.dataiku.com/dss/api/6.0/rest/ :returns: the project metadata. :rtype: dict """ return self.client._perform_json("GET", "/projects/%s/metadata" % self.project_key) def set_metadata(self, metadata): """ Set the metadata on this project. Usage example: .. code-block:: python project_metadata = project.get_metadata() project_metadata['tags'] = ['tag1','tag2'] project.set_metadata(project_metadata) :param dict metadata: the new state of the metadata for the project. You should only set a metadata object\ that has been retrieved using the :meth:`get_metadata` call. """ return self.client._perform_empty( "PUT", "/projects/%s/metadata" % self.project_key, body=metadata) def get_settings(self): """ Gets the settings of this project. This does not contain permissions. See :meth:`get_permissions` :returns: a handle to read, modify and save the settings :rtype: :class:`dataikuapi.dss.project.DSSProjectSettings` """ ret = self.client._perform_json("GET", "/projects/%s/settings" % self.project_key) return DSSProjectSettings(self.client, self.project_key, ret) def get_permissions(self): """ Get the permissions attached to this project :returns: A dict containing the owner and the permissions, as a list of pairs of group name and permission type :rtype: dict """ return self.client._perform_json( "GET", "/projects/%s/permissions" % self.project_key) def set_permissions(self, permissions): """ Sets the permissions on this project Usage example: .. code-block:: python project_permissions = project.get_permissions() project_permissions['permissions'].append({'group':'data_scientists', 'readProjectContent': True, 'readDashboards': True}) project.set_permissions(project_permissions) :param dict permissions: a permissions object with the same structure as the one returned by\ :meth:`get_permissions` call """ return self.client._perform_empty( "PUT", "/projects/%s/permissions" % self.project_key, body=permissions) def get_interest(self): """ Get the interest of this project. The interest means the number of watchers and the number of stars. :returns: a dict object containing the interest of the project with two fields: * **starCount**: number of stars for this project * **watchCount**: number of users watching this project :rtype: dict """ return self.client._perform_json("GET", "/projects/%s/interest" % self.project_key) def get_timeline(self, item_count=100): """ Get the timeline of this project. The timeline consists of information about the creation of this project (by whom, and when), the last modification of this project (by whom and when), a list of contributors, and a list of modifications. This list of modifications contains a maximum of **item_count** elements (default to 100). If **item_count** is greater than the real number of modification, **item_count** is adjusted. :param int item_count: maximum number of modifications to retrieve in the items list :returns: a timeline where the top-level fields are : * **allContributors**: all contributors who have been involved in this project * **items**: a history of the modifications of the project * **createdBy**: who created this project * **createdOn**: when the project was created * **lastModifiedBy**: who modified this project for the last time * **lastModifiedOn**: when this modification took place :rtype: dict """ return self.client._perform_json("GET", "/projects/%s/timeline" % self.project_key, params={ "itemCount": item_count }) ######################################################## # Datasets ######################################################## def list_datasets(self, as_type="listitems"): """ List the datasets in this project. :param str as_type: How to return the list. Supported values are "listitems" and "objects" (defaults to **listitems**). :returns: The list of the datasets. If "as_type" is "listitems", each one as a :class:`dataikuapi.dss.dataset.DSSDatasetListItem`. If "as_type" is "objects", each one as a :class:`dataikuapi.dss.dataset.DSSDataset` :rtype: list """ items = self.client._perform_json("GET", "/projects/%s/datasets/" % self.project_key) if as_type == "listitems" or as_type == "listitem": return [DSSDatasetListItem(self.client, item) for item in items] elif as_type == "objects" or as_type == "object": return [DSSDataset(self.client, self.project_key, item["name"]) for item in items] else: raise ValueError("Unknown as_type") def get_dataset(self, dataset_name): """ Get a handle to interact with a specific dataset :param str dataset_name: the name of the desired dataset :returns: A dataset handle :rtype: :class:`dataikuapi.dss.dataset.DSSDataset` """ return DSSDataset(self.client, self.project_key, dataset_name) def create_dataset(self, dataset_name, type, params=None, formatType=None, formatParams=None): """ Create a new dataset in the project, and return a handle to interact with it. The precise structure of **params** and **formatParams** depends on the specific dataset type and dataset format type. To know which fields exist for a given dataset type and format type, create a dataset from the UI, and use :meth:`get_dataset` to retrieve the configuration of the dataset and inspect it. Then reproduce a similar structure in the :meth:`create_dataset` call. Not all settings of a dataset can be set at creation time (for example partitioning). After creation, you'll have the ability to modify the dataset :param str dataset_name: the name of the dataset to create. Must not already exist :param str type: the type of the dataset :param dict params: the parameters for the type, as a python dict (defaults to **{}**) :param str formatType: an optional format to create the dataset with (only for file-oriented datasets) :param dict formatParams: the parameters to the format, as a python dict (only for file-oriented datasets, default to **{}**) :returns: A dataset handle :rtype: :class:`dataikuapi.dss.dataset.DSSDataset` """ if params is None: params = {} if formatParams is None: formatParams = {} obj = { "name": dataset_name, "projectKey": self.project_key, "type": type, "params": params, "formatType": formatType, "formatParams": formatParams } self.client._perform_json("POST", "/projects/%s/datasets/" % self.project_key, body=obj) return DSSDataset(self.client, self.project_key, dataset_name) def create_upload_dataset(self, dataset_name, connection=None): """ Create a new dataset of type 'UploadedFiles' in the project, and return a handle to interact with it. :param str dataset_name: the name of the dataset to create. Must not already exist :param str connection: the name of the upload connection (defaults to **None**) :returns: A dataset handle :rtype: :class:`dataikuapi.dss.dataset.DSSDataset` """ obj = { "name": dataset_name, "projectKey": self.project_key, "type": "UploadedFiles", "params": {} } if connection is not None: obj["params"]["uploadConnection"] = connection self.client._perform_json("POST", "/projects/%s/datasets/" % self.project_key, body=obj) return DSSDataset(self.client, self.project_key, dataset_name) def create_filesystem_dataset(self, dataset_name, connection, path_in_connection): """ Create a new filesystem dataset in the project, and return a handle to interact with it. :param str dataset_name: the name of the dataset to create. Must not already exist :param str connection: the name of the connection :param str path_in_connection: the path of the dataset in the connection :returns: A dataset handle :rtype: :class:`dataikuapi.dss.dataset.DSSDataset` """ return self.create_fslike_dataset(dataset_name, "Filesystem", connection, path_in_connection) def create_s3_dataset(self, dataset_name, connection, path_in_connection, bucket=None): """ Creates a new external S3 dataset in the project and returns a :class:`dataikuapi.dss.dataset.DSSDataset` to interact with it. The created dataset does not have its format and schema initialized, it is recommended to use :meth:`~dataikuapi.dss.dataset.DSSDataset.autodetect_settings` on the returned object :param str dataset_name: the name of the dataset to create. Must not already exist :param str connection: the name of the connection :param str path_in_connection: the path of the dataset in the connection :param str bucket: the name of the s3 bucket (defaults to **None**) :returns: A dataset handle :rtype: :class:`dataikuapi.dss.dataset.DSSDataset` """ extra_params = {} if bucket is not None: extra_params["bucket"] = bucket return self.create_fslike_dataset(dataset_name, "S3", connection, path_in_connection, extra_params) def create_fslike_dataset(self, dataset_name, dataset_type, connection, path_in_connection, extra_params=None): """ Create a new file-based dataset in the project, and return a handle to interact with it. :param str dataset_name: the name of the dataset to create. Must not already exist :param str dataset_type: the type of the dataset :param str connection: the name of the connection :param str path_in_connection: the path of the dataset in the connection :param dict extra_params: a python dict of extra parameters (defaults to **None**) :returns: A dataset handle :rtype: :class:`dataikuapi.dss.dataset.DSSDataset` """ body = { "name": dataset_name, "projectKey": self.project_key, "type": dataset_type, "params": { "connection": connection, "path": path_in_connection } } if extra_params is not None: body["params"].update(extra_params) self.client._perform_json("POST", "/projects/%s/datasets/" % self.project_key, body=body) return DSSDataset(self.client, self.project_key, dataset_name) def create_sql_table_dataset(self, dataset_name, type, connection, table, schema): """ Create a new SQL table dataset in the project, and return a handle to interact with it. :param str dataset_name: the name of the dataset to create. Must not already exist :param str type: the type of the dataset :param str connection: the name of the connection :param str table: the name of the table in the connection :param str schema: the schema of the table :returns: A dataset handle :rtype: :class:`dataikuapi.dss.dataset.DSSDataset` """ obj = { "name": dataset_name, "projectKey": self.project_key, "type": type, "params": { "connection": connection, "mode": "table", "table": table, "schema": schema } } self.client._perform_json("POST", "/projects/%s/datasets/" % self.project_key, body=obj) return DSSDataset(self.client, self.project_key, dataset_name) def new_managed_dataset_creation_helper(self, dataset_name): """ .. caution:: Deprecated. Please use :meth:`new_managed_dataset` """ warnings.warn("new_managed_dataset_creation_helper is deprecated, please use new_managed_dataset", DeprecationWarning) return DSSManagedDatasetCreationHelper(self, dataset_name) def new_managed_dataset(self, dataset_name): """ Initializes the creation of a new managed dataset. Returns a :class:`dataikuapi.dss.dataset.DSSManagedDatasetCreationHelper` or one of its subclasses to complete the creation of the managed dataset. Usage example: .. code-block:: python builder = project.new_managed_dataset("my_dataset") builder.with_store_into("target_connection") dataset = builder.create() :param str dataset_name: Name of the dataset to create :returns: An object to create the managed dataset :rtype: :class:`dataikuapi.dss.dataset.DSSManagedDatasetCreationHelper` """ return DSSManagedDatasetCreationHelper(self, dataset_name) ######################################################## # Streaming endpoints ######################################################## def list_streaming_endpoints(self, as_type="listitems"): """ List the streaming endpoints in this project. :param str as_type: How to return the list. Supported values are "listitems" and "objects" (defaults to **listitems**). :returns: The list of the streaming endpoints. If "as_type" is "listitems", each one as a :class:`dataikuapi.dss.streaming_endpoint.DSSStreamingEndpointListItem`. If "as_type" is "objects", each one as a :class:`dataikuapi.dss.streaming_endpoint.DSSStreamingEndpoint` :rtype: list """ items = self.client._perform_json("GET", "/projects/%s/streamingendpoints/" % self.project_key) if as_type == "listitems" or as_type == "listitem": return [DSSStreamingEndpointListItem(self.client, item) for item in items] elif as_type == "objects" or as_type == "object": return [DSSStreamingEndpoint(self.client, self.project_key, item["id"]) for item in items] else: raise ValueError("Unknown as_type") def get_streaming_endpoint(self, streaming_endpoint_name): """ Get a handle to interact with a specific streaming endpoint :param str streaming_endpoint_name: the name of the desired streaming endpoint :returns: A streaming endpoint handle :rtype: :class:`dataikuapi.dss.streaming_endpoint.DSSStreamingEndpoint` """ return DSSStreamingEndpoint(self.client, self.project_key, streaming_endpoint_name) def create_streaming_endpoint(self, streaming_endpoint_name, type, params=None): """ Create a new streaming endpoint in the project, and return a handle to interact with it. The precise structure of **params** depends on the specific streaming endpoint type. To know which fields exist for a given streaming endpoint type, create a streaming endpoint from the UI, and use :meth:`get_streaming_endpoint` to retrieve the configuration of the streaming endpoint and inspect it. Then reproduce a similar structure in the :meth:`create_streaming_endpoint` call. Not all settings of a streaming endpoint can be set at creation time (for example partitioning). After creation, you'll have the ability to modify the streaming endpoint. :param str streaming_endpoint_name: the name for the new streaming endpoint :param str type: the type of the streaming endpoint :param dict params: the parameters for the type, as a python dict (defaults to **{}**) :returns: A streaming endpoint handle :rtype: :class:`dataikuapi.dss.streaming_endpoint.DSSStreamingEndpoint` """ if params is None: params = {} obj = { "id": streaming_endpoint_name, "projectKey": self.project_key, "type": type, "params": params } self.client._perform_json("POST", "/projects/%s/streamingendpoints/" % self.project_key, body=obj) return DSSStreamingEndpoint(self.client, self.project_key, streaming_endpoint_name) def create_kafka_streaming_endpoint(self, streaming_endpoint_name, connection=None, topic=None): """ Create a new kafka streaming endpoint in the project, and return a handle to interact with it. :param str streaming_endpoint_name: the name for the new streaming endpoint :param str connection: the name of the kafka connection (defaults to **None**) :param str topic: the name of the kafka topic (defaults to **None**) :returns: A streaming endpoint handle :rtype: :class:`dataikuapi.dss.streaming_endpoint.DSSStreamingEndpoint` """ obj = { "id": streaming_endpoint_name, "projectKey": self.project_key, "type": "kafka", "params": {} } if connection is not None: obj["params"]["connection"] = connection if topic is not None: obj["params"]["topic"] = topic self.client._perform_json("POST", "/projects/%s/streamingendpoints/" % self.project_key, body=obj) return DSSStreamingEndpoint(self.client, self.project_key, streaming_endpoint_name) def create_httpsse_streaming_endpoint(self, streaming_endpoint_name, url=None): """ Create a new https streaming endpoint in the project, and return a handle to interact with it. :param str streaming_endpoint_name: the name for the new streaming endpoint :param str url: the url of the endpoint (defaults to **None**) :returns: A streaming endpoint handle :rtype: :class:`dataikuapi.dss.streaming_endpoint.DSSStreamingEndpoint` """ obj = { "id": streaming_endpoint_name, "projectKey": self.project_key, "type": "httpsse", "params": {} } if url is not None: obj["params"]["url"] = url self.client._perform_json("POST", "/projects/%s/streamingendpoints/" % self.project_key, body=obj) return DSSStreamingEndpoint(self.client, self.project_key, streaming_endpoint_name) def new_managed_streaming_endpoint(self, streaming_endpoint_name, streaming_endpoint_type=None): """ Initializes the creation of a new streaming endpoint. Returns a :class:`dataikuapi.dss.streaming_endpoint.DSSManagedStreamingEndpointCreationHelper` to complete the creation of the streaming endpoint :param str streaming_endpoint_name: Name of the new streaming endpoint - must be unique in the project :param str streaming_endpoint_type: Type of the new streaming endpoint (optional if it can be inferred from a connection type) :returns: An object to create the streaming endpoint :rtype: :class:`~dataikuapi.dss.streaming_endpoint.DSSManagedStreamingEndpointCreationHelper` """ return DSSManagedStreamingEndpointCreationHelper(self, streaming_endpoint_name, streaming_endpoint_type) ######################################################## # Lab and ML # Don't forget to synchronize with DSSDataset.* ######################################################## def create_prediction_ml_task(self, input_dataset, target_variable, ml_backend_type="PY_MEMORY", guess_policy="DEFAULT", prediction_type=None, wait_guess_complete=True): """Creates a new prediction task in a new visual analysis lab for a dataset. :param str input_dataset: the dataset to use for training/testing the model :param str target_variable: the variable to predict :param str ml_backend_type: ML backend to use, one of PY_MEMORY, MLLIB or H2O (defaults to **PY_MEMORY**) :param str guess_policy: Policy to use for setting the default parameters. Valid values are: DEFAULT, SIMPLE_FORMULA, DECISION_TREE, EXPLANATORY and PERFORMANCE (defaults to **DEFAULT**) :param str prediction_type: The type of prediction problem this is. If not provided the prediction type will be guessed. Valid values are: BINARY_CLASSIFICATION, REGRESSION, MULTICLASS (defaults to **None**) :param boolean wait_guess_complete: if False, the returned ML task will be in 'guessing' state, i.e. analyzing the input dataset to determine feature handling and algorithms (defaults to **True**). You should wait for the guessing to be completed by calling **wait_guess_complete** on the returned object before doing anything else (in particular calling **train** or **get_settings**) :returns: A ML task handle of type 'PREDICTION' :rtype: :class:`dataikuapi.dss.ml.DSSMLTask` """ obj = { "inputDataset": input_dataset, "taskType": "PREDICTION", "targetVariable": target_variable, "backendType": ml_backend_type, "guessPolicy": guess_policy } if prediction_type is not None: obj["predictionType"] = prediction_type ref = self.client._perform_json("POST", "/projects/%s/models/lab/" % self.project_key, body=obj) ret = DSSMLTask(self.client, self.project_key, ref["analysisId"], ref["mlTaskId"]) if wait_guess_complete: ret.wait_guess_complete() return ret def create_clustering_ml_task(self, input_dataset, ml_backend_type="PY_MEMORY", guess_policy="KMEANS", wait_guess_complete=True): """Creates a new clustering task in a new visual analysis lab for a dataset. The returned ML task will be in 'guessing' state, i.e. analyzing the input dataset to determine feature handling and algorithms. You should wait for the guessing to be completed by calling **wait_guess_complete** on the returned object before doing anything else (in particular calling **train** or **get_settings**) :param str ml_backend_type: ML backend to use, one of PY_MEMORY, MLLIB or H2O (defaults to **PY_MEMORY**) :param str guess_policy: Policy to use for setting the default parameters. Valid values are: KMEANS and ANOMALY_DETECTION (defaults to **KMEANS**) :param boolean wait_guess_complete: if False, the returned ML task will be in 'guessing' state, i.e. analyzing the input dataset to determine feature handling and algorithms (defaults to **True**). You should wait for the guessing to be completed by calling **wait_guess_complete** on the returned object before doing anything else (in particular calling **train** or **get_settings**) :returns: A ML task handle of type 'CLUSTERING' :rtype: :class:`dataikuapi.dss.ml.DSSMLTask` """ obj = { "inputDataset": input_dataset, "taskType": "CLUSTERING", "backendType": ml_backend_type, "guessPolicy": guess_policy } ref = self.client._perform_json("POST", "/projects/%s/models/lab/" % self.project_key, body=obj) mltask = DSSMLTask(self.client, self.project_key, ref["analysisId"], ref["mlTaskId"]) if wait_guess_complete: mltask.wait_guess_complete() return mltask def create_timeseries_forecasting_ml_task(self, input_dataset, target_variable, time_variable, timeseries_identifiers=None, guess_policy="TIMESERIES_DEFAULT", wait_guess_complete=True): """Creates a new time series forecasting task in a new visual analysis lab for a dataset. :param string input_dataset: The dataset to use for training/testing the model :param string target_variable: The variable to forecast :param string time_variable: Column to be used as time variable. Should be a Date (parsed) column. :param list timeseries_identifiers: List of columns to be used as time series identifiers (when the dataset has multiple series) :param string guess_policy: Policy to use for setting the default parameters. Valid values are: TIMESERIES_DEFAULT, TIMESERIES_STATISTICAL, and TIMESERIES_DEEP_LEARNING :param boolean wait_guess_complete: If False, the returned ML task will be in 'guessing' state, i.e. analyzing the input dataset to determine feature handling and algorithms. You should wait for the guessing to be completed by calling ``wait_guess_complete`` on the returned object before doing anything else (in particular calling ``train`` or ``get_settings``) :return :class dataiku.dss.ml.DSSMLTask """ obj = { "inputDataset": input_dataset, "taskType": "PREDICTION", "targetVariable": target_variable, "timeVariable": time_variable, "timeseriesIdentifiers": timeseries_identifiers, "backendType": "PY_MEMORY", "guessPolicy": guess_policy, "predictionType": "TIMESERIES_FORECAST" } ref = self.client._perform_json( "POST", "/projects/{project_key}/models/lab/".format(project_key=self.project_key), body=obj ) ret = DSSMLTask(self.client, self.project_key, ref["analysisId"], ref["mlTaskId"]) if wait_guess_complete: ret.wait_guess_complete() return ret def list_ml_tasks(self): """ List the ML tasks in this project :returns: the list of the ML tasks summaries, each one as a python dict :rtype: list """ return self.client._perform_json("GET", "/projects/%s/models/lab/" % self.project_key) def get_ml_task(self, analysis_id, mltask_id): """ Get a handle to interact with a specific ML task :param str analysis_id: the identifier of the visual analysis containing the desired ML task :param str mltask_id: the identifier of the desired ML task :returns: A ML task handle :rtype: :class:`dataikuapi.dss.ml.DSSMLTask` """ return DSSMLTask(self.client, self.project_key, analysis_id, mltask_id) def list_mltask_queues(self): """ List non-empty ML task queues in this project :returns: an iterable listing of MLTask queues (each a dict) :rtype: :class:`dataikuapi.dss.ml.DSSMLTaskQueues` """ data = self.client._perform_json("GET", "/projects/%s/models/labs/mltask-queues" % self.project_key) return DSSMLTaskQueues(data) def create_analysis(self, input_dataset): """ Creates a new visual analysis lab for a dataset. :param str input_dataset: the dataset to use for the analysis :returns: A visual analysis handle :rtype: :class:`dataikuapi.dss.analysis.DSSAnalysis` """ obj = { "inputDataset": input_dataset } ref = self.client._perform_json("POST", "/projects/%s/lab/" % self.project_key, body=obj) return DSSAnalysis(self.client, self.project_key, ref["id"]) def list_analyses(self): """ List the visual analyses in this project :returns: the list of the visual analyses summaries, each one as a python dict :rtype: list """ return self.client._perform_json("GET", "/projects/%s/lab/" % self.project_key) def get_analysis(self, analysis_id): """ Get a handle to interact with a specific visual analysis :param str analysis_id: the identifier of the desired visual analysis :returns: A visual analysis handle :rtype: :class:`dataikuapi.dss.analysis.DSSAnalysis` """ return DSSAnalysis(self.client, self.project_key, analysis_id) ######################################################## # Saved models ######################################################## def list_saved_models(self): """ List the saved models in this project :returns: the list of the saved models, each one as a python dict :rtype: list """ return self.client._perform_json( "GET", "/projects/%s/savedmodels/" % self.project_key) def get_saved_model(self, sm_id): """ Get a handle to interact with a specific saved model :param str sm_id: the identifier of the desired saved model :returns: A saved model handle :rtype: :class:`dataikuapi.dss.savedmodel.DSSSavedModel` """ return DSSSavedModel(self.client, self.project_key, sm_id) def create_mlflow_pyfunc_model(self, name, prediction_type=None): """ Creates a new external saved model for storing and managing MLFlow models :param str name: Human readable name for the new saved model in the flow :param str prediction_type: Optional (but needed for most operations). One of BINARY_CLASSIFICATION, MULTICLASS, REGRESSION or None. Defaults to None, standing for other prediction types. If the Saved Model has a None prediction type, scoring, inclusion in a bundle or in an API service will be possible, but features related to performance analysis and explainability will not be available. :returns: The created saved model handle :rtype: :class:`dataikuapi.dss.savedmodel.DSSSavedModel` """ model = { "savedModelType": "MLFLOW_PYFUNC", "predictionType": prediction_type, "name": name } id = self.client._perform_json("POST", "/projects/%s/savedmodels/" % self.project_key, body=model)["id"] return self.get_saved_model(id) def create_proxy_model(self, name, prediction_type): """ EXPERIMENTAL. Creates a new external saved model that can contain proxy model as versions. This is an experimental API, subject to change. :param string name: Human readable name for the new saved model in the flow :param string prediction_type: One of BINARY_CLASSIFICATION, MULTICLASS or REGRESSION """ model = { "savedModelType": "PROXY_MODEL", "predictionType": prediction_type, "name": name } saved_model_id = self.client._perform_json("POST", "/projects/%s/savedmodels/" % self.project_key, body=model)["id"] return self.get_saved_model(saved_model_id) ######################################################## # Managed folders ######################################################## def list_managed_folders(self): """ List the managed folders in this project :returns: the list of the managed folders, each one as a python dict :rtype: list """ return self.client._perform_json( "GET", "/projects/%s/managedfolders/" % self.project_key) def get_managed_folder(self, odb_id): """ Get a handle to interact with a specific managed folder :param str odb_id: the identifier of the desired managed folder :returns: A managed folder handle :rtype: :class:`dataikuapi.dss.managedfolder.DSSManagedFolder` """ return DSSManagedFolder(self.client, self.project_key, odb_id) def create_managed_folder(self, name, folder_type=None, connection_name="filesystem_folders"): """ Create a new managed folder in the project, and return a handle to interact with it :param str name: the name of the managed folder :param str folder_type: type of storage (defaults to **None**) :param str connection_name: the connection name (defaults to **filesystem_folders**) :returns: A managed folder handle :rtype: :class:`dataikuapi.dss.managedfolder.DSSManagedFolder` """ obj = { "name": name, "projectKey": self.project_key, "type": folder_type, "params": { "connection": connection_name, "path": "/${projectKey}/${odbId}" } } res = self.client._perform_json("POST", "/projects/%s/managedfolders/" % self.project_key, body=obj) odb_id = res['id'] return DSSManagedFolder(self.client, self.project_key, odb_id) ######################################################## # Model evaluation stores ######################################################## def list_model_evaluation_stores(self): """ List the model evaluation stores in this project. :returns: The list of the model evaluation stores :rtype: list of :class:`dataikuapi.dss.modelevaluationstore.DSSModelEvaluationStore` """ items = self.client._perform_json("GET", "/projects/%s/modelevaluationstores/" % self.project_key) return [DSSModelEvaluationStore(self.client, self.project_key, item["id"]) for item in items] def get_model_evaluation_store(self, mes_id): """ Get a handle to interact with a specific model evaluation store :param str mes_id: the id of the desired model evaluation store :returns: A model evaluation store handle :rtype: :class:`dataikuapi.dss.modelevaluationstore.DSSModelEvaluationStore` """ return DSSModelEvaluationStore(self.client, self.project_key, mes_id) def create_model_evaluation_store(self, name): """ Create a new model evaluation store in the project, and return a handle to interact with it. :param str name: the name for the new model evaluation store :returns: A model evaluation store handle :rtype: :class:`dataikuapi.dss.modelevaluationstore.DSSModelEvaluationStore` """ obj = { "projectKey": self.project_key, "name": name } res = self.client._perform_json("POST", "/projects/%s/modelevaluationstores/" % self.project_key, body=obj) mes_id = res['id'] return DSSModelEvaluationStore(self.client, self.project_key, mes_id) ######################################################## # Model comparisons ######################################################## def list_model_comparisons(self): """ List the model comparisons in this project. :returns: The list of the model comparisons :rtype: list """ items = self.client._perform_json("GET", "/projects/%s/modelcomparisons/" % self.project_key) return [DSSModelComparison(self.client, self.project_key, item["id"]) for item in items] def get_model_comparison(self, mec_id): """ Get a handle to interact with a specific model comparison :param str mec_id: the id of the desired model comparison :returns: A model comparison handle :rtype: :class:`dataikuapi.dss.modelcomparison.DSSModelComparison` """ return DSSModelComparison(self.client, self.project_key, mec_id) def create_model_comparison(self, name, prediction_type): """ Create a new model comparison in the project, and return a handle to interact with it. :param str name: the name for the new model comparison :param str prediction_type: one of BINARY_CLASSIFICATION, REGRESSION, MULTICLASS, and TIMESERIES_FORECAST :returns: A new model comparison handle :rtype: :class:`dataikuapi.dss.modelcomparison.DSSModelComparison` """ obj = { "projectKey": self.project_key, "displayName": name, "predictionType": prediction_type } res = self.client._perform_json("POST", "/projects/%s/modelcomparisons/" % self.project_key, body=obj) mec_id = res['id'] return DSSModelComparison(self.client, self.project_key, mec_id) ######################################################## # Jobs ######################################################## def list_jobs(self): """ List the jobs in this project :returns: a list of the jobs, each one as a python dict, containing both the definition and the state :rtype: list """ return self.client._perform_json( "GET", "/projects/%s/jobs/" % self.project_key) def get_job(self, id): """ Get a handler to interact with a specific job :param str id: the id of the desired job :returns: A job handle :rtype: :class:`dataikuapi.dss.job.DSSJob` """ return DSSJob(self.client, self.project_key, id) def start_job(self, definition): """ Create a new job, and return a handle to interact with it :param dict definition: The definition should contain: * the type of job (RECURSIVE_BUILD, NON_RECURSIVE_FORCED_BUILD, RECURSIVE_FORCED_BUILD,\ RECURSIVE_MISSING_ONLY_BUILD) * a list of outputs to build from the available types: (DATASET, MANAGED_FOLDER, SAVED_MODEL,\ STREAMING_ENDPOINT) * (Optional) a refreshHiveMetastore field (True or False) to specify whether to re-synchronize the Hive\ metastore for recomputed HDFS datasets. :returns: A job handle :rtype: :class:`dataikuapi.dss.job.DSSJob` """ job_def = self.client._perform_json("POST", "/projects/%s/jobs/" % self.project_key, body=definition) return DSSJob(self.client, self.project_key, job_def['id']) def start_job_and_wait(self, definition, no_fail=False): """ Starts a new job and waits for it to complete. :param dict definition: The definition should contain: * the type of job (RECURSIVE_BUILD, NON_RECURSIVE_FORCED_BUILD, RECURSIVE_FORCED_BUILD,\ RECURSIVE_MISSING_ONLY_BUILD) * a list of outputs to build from the available types: (DATASET, MANAGED_FOLDER, SAVED_MODEL,\ STREAMING_ENDPOINT) * (Optional) a refreshHiveMetastore field (True or False) to specify whether to re-synchronize the Hive\ metastore for recomputed HDFS datasets. :param bool no_fail: if true, the function won't fail even if the job fails or aborts (defaults to **False**) :returns: the final status of the job :rtype: str """ job_def = self.client._perform_json("POST", "/projects/%s/jobs/" % self.project_key, body=definition) job = DSSJob(self.client, self.project_key, job_def['id']) waiter = DSSJobWaiter(job) return waiter.wait(no_fail) def new_job(self, job_type='NON_RECURSIVE_FORCED_BUILD'): """ Create a job to be run. You need to add outputs to the job (i.e. what you want to build) before running it. .. code-block:: python job_builder = project.new_job() job_builder.with_output("mydataset") complete_job = job_builder.start_and_wait() print("Job %s done" % complete_job.id) :param str job_type: the type of job (RECURSIVE_BUILD, NON_RECURSIVE_FORCED_BUILD, RECURSIVE_FORCED_BUILD, RECURSIVE_MISSING_ONLY_BUILD) (defaults to `NON_RECURSIVE_FORCED_BUILD`) :returns: A job handle :rtype: :class:`dataikuapi.dss.project.JobDefinitionBuilder` """ return JobDefinitionBuilder(self, job_type) def new_job_definition_builder(self, job_type='NON_RECURSIVE_FORCED_BUILD'): """ .. caution:: Deprecated. Please use :meth:`new_job` """ warnings.warn("new_job_definition_builder is deprecated, please use new_job", DeprecationWarning) return JobDefinitionBuilder(self, job_type) ######################################################## # Jupyter Notebooks ######################################################## def list_jupyter_notebooks(self, active=False, as_type="object"): """ List the jupyter notebooks of a project. :param bool active: if True, only return currently running jupyter notebooks (defaults to **active**). :param bool as_type: How to return the list. Supported values are "listitems" and "object" (defaults to **object**). :returns: The list of the notebooks. If "as_type" is "listitems", each one as a :class:`dataikuapi.dss.jupyternotebook.DSSJupyterNotebookListItem`, if "as_type" is "objects", each one as a :class:`dataikuapi.dss.jupyternotebook.DSSJupyterNotebook` :rtype: list of :class:`dataikuapi.dss.jupyternotebook.DSSJupyterNotebook` or list of :class:`dataikuapi.dss.jupyternotebook.DSSJupyterNotebookListItem` """ notebook_items = self.client._perform_json("GET", "/projects/%s/jupyter-notebooks/" % self.project_key, params={"active": active}) if as_type == "listitems" or as_type == "listitem": return [DSSJupyterNotebookListItem(self.client, notebook_item) for notebook_item in notebook_items] elif as_type == "objects" or as_type == "object": return [DSSJupyterNotebook(self.client, self.project_key, notebook_item["name"]) for notebook_item in notebook_items] else: raise ValueError("Unknown as_type") def get_jupyter_notebook(self, notebook_name): """ Get a handle to interact with a specific jupyter notebook :param str notebook_name: The name of the jupyter notebook to retrieve :returns: A handle to interact with this jupyter notebook :rtype: :class:`dataikuapi.dss.jupyternotebook.DSSJupyterNotebook` jupyter notebook handle """ return DSSJupyterNotebook(self.client, self.project_key, notebook_name) def create_jupyter_notebook(self, notebook_name, notebook_content): """ Create a new jupyter notebook and get a handle to interact with it :param str notebook_name: the name of the notebook to create :param dict notebook_content: the data of the notebook to create, as a dict. The data will be converted to a JSON string internally. Use :meth:`DSSJupyterNotebook.get_content()` on a similar existing **DSSJupyterNotebook** object in order to get a sample definition object. :returns: A handle to interact with the newly created jupyter notebook :rtype: :class:`dataikuapi.dss.jupyternotebook.DSSJupyterNotebook` jupyter notebook handle """ self.client._perform_json("POST", "/projects/%s/jupyter-notebooks/%s" % (self.project_key, notebook_name), body=notebook_content) return self.get_jupyter_notebook(notebook_name) ######################################################## # Continuous activities ######################################################## def list_continuous_activities(self, as_objects=True): """ List the continuous activities in this project :param bool as_objects: if True, returns a list of :class:`dataikuapi.dss.continuousactivity.DSSContinuousActivity` objects, else returns a list of python dicts (defaults to **True**) :returns: a list of the continuous activities, each one as a python dict, containing both the definition and the state :rtype: list """ list = self.client._perform_json("GET", "/projects/%s/continuous-activities/" % self.project_key) if as_objects: return [DSSContinuousActivity(self.client, a['projectKey'], a['recipeId']) for a in list] else: return list def get_continuous_activity(self, recipe_id): """ Get a handler to interact with a specific continuous activities :param str recipe_id: the identifier of the recipe controlled by the continuous activity :returns: A job handle :rtype: :class:`dataikuapi.dss.continuousactivity.DSSContinuousActivity` """ return DSSContinuousActivity(self.client, self.project_key, recipe_id) ######################################################## # Variables ######################################################## def get_variables(self): """ Gets the variables of this project. :returns: a dictionary containing two dictionaries : "standard" and "local". "standard" are regular variables, exported with bundles. "local" variables are not part of the bundles for this project :rtype: dict """ return self.client._perform_json( "GET", "/projects/%s/variables/" % self.project_key) def set_variables(self, obj): """ Sets the variables of this project. .. warning:: If executed from a python recipe, the changes made by `set_variables` will not be "seen" in that recipe. Use the internal API dataiku.get_custom_variables() instead if this behavior is needed :param dict obj: must be a modified version of the object returned by get_variables """ if not "standard" in obj: raise ValueError("Missing 'standard' key in argument") if not "local" in obj: raise ValueError("Missing 'local' key in argument") self.client._perform_empty( "PUT", "/projects/%s/variables/" % self.project_key, body=obj) def update_variables(self, variables, type="standard"): """ Updates a set of variables for this project :param variables dict: a dict of variable name -> value to set. Keys of the dict must be strings. Values in the dict can be strings, numbers, booleans, lists or dicts :param type str: Can be "standard" to update regular variables or "local" to update local-only variables that are not part of bundles for this project (defaults to **standard**) """ current_variables = self.get_variables() current_variables[type].update(variables) self.set_variables(current_variables) ######################################################## # API Services ######################################################## def list_api_services(self, as_type="listitems"): """ List the API services in this project :param str as_type: How to return the list. Supported values are "listitems" and "objects" (defaults to **listitems**). :returns: The list of the datasets. If "as_type" is "listitems", each one as a :class:`dataikuapi.dss.apiservice.DSSAPIServiceListItem`. If "as_type" is "objects", each one as a :class:`dataikuapi.dss.apiservice.DSSAPIService` :rtype: list """ items = self.client._perform_json("GET", "/projects/%s/apiservices/" % self.project_key) if as_type == "listitems" or as_type == "listitem": return [DSSAPIServiceListItem(self.client, item) for item in items] elif as_type == "objects" or as_type == "object": return [DSSAPIService(self.client, self.project_key, item["id"]) for item in items] else: raise ValueError("Unknown as_type") def create_api_service(self, service_id): """ Create a new API service, and returns a handle to interact with it. The newly-created service does not have any endpoint. :param str service_id: the ID of the API service to create :returns: A API Service handle :rtype: :class:`dataikuapi.dss.apiservice.DSSAPIService` """ self.client._perform_empty("POST", "/projects/%s/apiservices/%s" % (self.project_key, service_id)) return DSSAPIService(self.client, self.project_key, service_id) def get_api_service(self, service_id): """ Get a handle to interact with a specific API Service from the API Designer :param str service_id: The identifier of the API Designer API Service to retrieve :returns: A handle to interact with this API Service :rtype: :class:`dataikuapi.dss.apiservice.DSSAPIService` """ return DSSAPIService(self.client, self.project_key, service_id) ######################################################## # Bundles / Export and Publish (Design node) ######################################################## def list_exported_bundles(self): """ List all the bundles created in this project on the Design Node. :returns: A dictionary of all bundles for a project on the Design node. :rtype: dict """ return self.client._perform_json("GET", "/projects/%s/bundles/exported" % self.project_key) def export_bundle(self, bundle_id): """ Creates a new project bundle on the Design node :param str bundle_id: bundle id tag """ return self.client._perform_json("PUT", "/projects/%s/bundles/exported/%s" % (self.project_key, bundle_id)) def get_exported_bundle_archive_stream(self, bundle_id): """ Download a bundle archive that can be deployed in a DSS automation Node, as a binary stream. .. warning:: The stream must be closed after use. Use a **with** statement to handle closing the stream at the end of the block by default. For example: .. code-block:: python with project.get_exported_bundle_archive_stream('v1') as fp: # use fp # or explicitly close the stream after use fp = project.get_exported_bundle_archive_stream('v1') # use fp, then close fp.close() :param str bundle_id: the identifier of the bundle """ return self.client._perform_raw("GET", "/projects/%s/bundles/exported/%s/archive" % (self.project_key, bundle_id)) def download_exported_bundle_archive_to_file(self, bundle_id, path): """ Download a bundle archive that can be deployed in a DSS automation Node into the given output file. :param str bundle_id: the identifier of the bundle :param str path: if "-", will write to /dev/stdout """ if path == "-": path = "/dev/stdout" stream = self.get_exported_bundle_archive_stream(bundle_id) with open(path, 'wb') as f: for chunk in stream.iter_content(chunk_size=10000): if chunk: f.write(chunk) f.flush() stream.close() def publish_bundle(self, bundle_id, published_project_key=None): """ Publish a bundle on the Project Deployer. :param str bundle_id: The identifier of the bundle :param str published_project_key: The key of the project on the Project Deployer where the bundle will be published.A new published project will be created if none matches the key. If the parameter is not set, the key from the current :class:`DSSProject` is used. :returns: a dict with info on the bundle state once published. It contains the keys "publishedOn" for the publish date, "publishedBy" for the user who published the bundle, and "publishedProjectKey" for the key of the Project Deployer project used. :rtype: dict """ params = None if published_project_key is not None: params = {"publishedProjectKey": published_project_key} return self.client._perform_json("POST", "/projects/%s/bundles/%s/publish" % (self.project_key, bundle_id), params=params) ######################################################## # Bundles / Import (Automation node) ######################################################## def list_imported_bundles(self): """ List all the bundles imported for this project, on the Automation node. :returns: a dict containing bundle imports for a project, on the Automation node. :rtype: dict """ return self.client._perform_json("GET", "/projects/%s/bundles/imported" % self.project_key) def import_bundle_from_archive(self, archive_path): """ Imports a bundle from a zip archive path on the Automation node. :param str archive_path: A full path to a zip archive, for example `/home/dataiku/my-bundle-v1.zip` """ return self.client._perform_json("POST", "/projects/%s/bundles/imported/actions/importFromArchive" % (self.project_key), params={"archivePath": osp.abspath(archive_path)}) def import_bundle_from_stream(self, fp): """ Imports a bundle from a file stream, on the Automation node. Usage example: .. code-block:: python project = client.get_project('MY_PROJECT') with open('/home/dataiku/my-bundle-v1.zip', 'rb') as f: project.import_bundle_from_stream(f) :param file-like fp: file handler. """ files = {'file': fp} return self.client._perform_empty("POST", "/projects/%s/bundles/imported/actions/importFromStream" % (self.project_key), files=files) def activate_bundle(self, bundle_id, scenarios_to_enable=None): """ Activates a bundle in this project. :param str bundle_id: The ID of the bundle to activate :param dict scenarios_to_enable: An optional dict of scenarios to enable or disable upon bundle activation. The format of the dict should be scenario IDs as keys with values of True or False (defaults to **{}**). :returns: A report containing any error or warning messages that occurred during bundle activation :rtype: dict """ options = {"scenariosActiveOnActivation": scenarios_to_enable} if scenarios_to_enable else {} return self.client._perform_json("POST", "/projects/%s/bundles/imported/%s/actions/activate" % ( self.project_key, bundle_id), body=options) def preload_bundle(self, bundle_id): """ Preloads a bundle that has been imported on the Automation node :param str bundle_id: the bundle_id for an existing imported bundle """ return self.client._perform_json("POST", "/projects/%s/bundles/imported/%s/actions/preload" % ( self.project_key, bundle_id)) ######################################################## # Scenarios ######################################################## def list_scenarios(self, as_type="listitems"): """ List the scenarios in this project. :param str as_type: How to return the list. Supported values are "listitems" and "objects" (defaults to **listitems**). :returns: The list of the datasets. If "rtype" is "listitems", each one as a :class:`dataikuapi.dss.scenario.DSSScenarioListItem`. If "rtype" is "objects", each one as a :class:`dataikuapi.dss.scenario.DSSScenario` :rtype: list """ items = self.client._perform_json("GET", "/projects/%s/scenarios/" % self.project_key) if as_type == "listitems": return [DSSScenarioListItem(self.client, item) for item in items] elif as_type == "objects": return [DSSScenario(self.client, self.project_key, item["id"]) for item in items] else: raise ValueError("Unknown as_type") def get_scenario(self, scenario_id): """ Get a handle to interact with a specific scenario :param str: scenario_id: the ID of the desired scenario :returns: A scenario handle :rtype: :class:`dataikuapi.dss.scenario.DSSScenario` """ return DSSScenario(self.client, self.project_key, scenario_id) def create_scenario(self, scenario_name, type, definition=None): """ Create a new scenario in the project, and return a handle to interact with it :param str scenario_name: The name for the new scenario. This does not need to be unique (although this is strongly recommended) :param str type: The type of the scenario. MUst be one of 'step_based' or 'custom_python' :param dict definition: the JSON definition of the scenario. Use **get_definition(with_status=False)** on an existing **DSSScenario** object in order to get a sample definition object (defaults to **{'params': {}}**) :returns: a :class:`dataikuapi.dss.scenario.DSSScenario` handle to interact with the newly-created scenario """ if definition is None: definition = {'params': {}} definition['type'] = type definition['name'] = scenario_name scenario_id = self.client._perform_json("POST", "/projects/%s/scenarios/" % self.project_key, body=definition)['id'] return DSSScenario(self.client, self.project_key, scenario_id) ######################################################## # Recipes ######################################################## def list_recipes(self, as_type="listitems"): """ List the recipes in this project :param str as_type: How to return the list. Supported values are "listitems" and "objects" (defaults to **listitems**). :returns: The list of the recipes. If "as_type" is "listitems", each one as a :class:`dataikuapi.dss.recipe.DSSRecipeListItem`. If "as_type" is "objects", each one as a :class:`dataikuapi.dss.recipe.DSSRecipe` :rtype: list """ items = self.client._perform_json("GET", "/projects/%s/recipes/" % self.project_key) if as_type == "listitems" or as_type == "listitem": return [DSSRecipeListItem(self.client, item) for item in items] elif as_type == "objects" or as_type == "object": return [DSSRecipe(self.client, self.project_key, item["name"]) for item in items] else: raise ValueError("Unknown as_type") def get_recipe(self, recipe_name): """ Gets a :class:`dataikuapi.dss.recipe.DSSRecipe` handle to interact with a recipe :param str recipe_name: The name of the recipe :returns: A recipe handle :rtype: :class:`dataikuapi.dss.recipe.DSSRecipe` """ return DSSRecipe(self.client, self.project_key, recipe_name) def create_recipe(self, recipe_proto, creation_settings): """ Create a new recipe in the project, and return a handle to interact with it. We strongly recommend that you use the creator helpers instead of calling this directly. Some recipe types require additional parameters in creation_settings: * 'grouping' : a 'groupKey' column name * 'python', 'sql_query', 'hive', 'impala' : the code of the recipe as a 'payload' string :param dict recipe_proto: a prototype for the recipe object. Must contain at least 'type' and 'name' :param dict creation_settings: recipe-specific creation settings :returns: A recipe handle :rtype: :class:`dataikuapi.dss.recipe.DSSRecipe` """ recipe_proto["projectKey"] = self.project_key definition = {'recipePrototype': recipe_proto, 'creationSettings': creation_settings} recipe_name = self.client._perform_json("POST", "/projects/%s/recipes/" % self.project_key, body=definition)['name'] return DSSRecipe(self.client, self.project_key, recipe_name) def new_recipe(self, type, name=None): """ Initializes the creation of a new recipe. Returns a :class:`dataikuapi.dss.recipe.DSSRecipeCreator` or one of its subclasses to complete the creation of the recipe. Usage example: .. code-block:: python grouping_recipe_builder = project.new_recipe("grouping") grouping_recipe_builder.with_input("dataset_to_group_on") # Create a new managed dataset for the output in the "filesystem_managed" connection grouping_recipe_builder.with_new_output("grouped_dataset", "filesystem_managed") grouping_recipe_builder.with_group_key("column") recipe = grouping_recipe_builder.build() # After the recipe is created, you can edit its settings recipe_settings = recipe.get_settings() recipe_settings.set_column_aggregations("value", sum=True) recipe_settings.save() # And you may need to apply new schemas to the outputs recipe.compute_schema_updates().apply() :param str type: Type of the recipe :param str name: Optional, base name for the new recipe. :returns: A new DSS Recipe Creator handle :rtype: :class:`dataikuapi.dss.recipe.DSSRecipeCreator` """ if type == "grouping": return recipe.GroupingRecipeCreator(name, self) elif type == "window": return recipe.WindowRecipeCreator(name, self) elif type == "sync": return recipe.SyncRecipeCreator(name, self) elif type == "csync": return recipe.ContinuousSyncRecipeCreator(name, self) elif type == "pivot": return recipe.PivotRecipeCreator(name, self) elif type == "sort": return recipe.SortRecipeCreator(name, self) elif type == "topn": return recipe.TopNRecipeCreator(name, self) elif type == "distinct": return recipe.DistinctRecipeCreator(name, self) elif type == "join": return recipe.JoinRecipeCreator(name, self) elif type == "vstack": return recipe.StackRecipeCreator(name, self) elif type == "sampling": return recipe.SamplingRecipeCreator(name, self) elif type == "split": return recipe.SplitRecipeCreator(name, self) elif type == "prepare" or type == "shaker": return recipe.PrepareRecipeCreator(name, self) elif type == "prediction_scoring": return recipe.PredictionScoringRecipeCreator(name, self) elif type == "evaluation": return recipe.EvaluationRecipeCreator(name, self) elif type == "standalone_evaluation": return recipe.StandaloneEvaluationRecipeCreator(name, self) elif type == "clustering_scoring": return recipe.ClusteringScoringRecipeCreator(name, self) elif type == "download": return recipe.DownloadRecipeCreator(name, self) elif type == "sql_query": return recipe.SQLQueryRecipeCreator(name, self) elif type in ["python", "r", "sql_script", "pyspark", "sparkr", "spark_scala", "shell"]: return recipe.CodeRecipeCreator(name, type, self) elif type in ["cpython", "ksql", "streaming_spark_scala"]: return recipe.CodeRecipeCreator(name, type, self) ######################################################## # Flow ######################################################## def get_flow(self): """ :returns: A Flow handle :rtype: A :class:`dataikuapi.dss.flow.DSSProjectFlow` """ return DSSProjectFlow(self.client, self) ######################################################## # Security ######################################################## def sync_datasets_acls(self): """ Resync permissions on HDFS datasets in this project .. attention:: This call requires an API key with admin rights :returns: a handle to the task of resynchronizing the permissions :rtype: :class:`dataikuapi.dss.future.DSSFuture` """ future_response = self.client._perform_json( "POST", "/projects/%s/actions/sync" % (self.project_key)) return DSSFuture(self.client, future_response.get('jobId', None), future_response) ######################################################## # Notebooks ######################################################## def list_running_notebooks(self, as_objects=True): """ .. caution:: Deprecated. Use :meth:`DSSProject.list_jupyter_notebooks` List the currently-running notebooks :returns: list of notebooks. Each object contains at least a 'name' field :rtype: list """ notebook_list = self.client._perform_json("GET", "/projects/%s/notebooks/active" % self.project_key) if as_objects: return [DSSNotebook(self.client, notebook['projectKey'], notebook['name'], notebook) for notebook in notebook_list] else: return notebook_list ######################################################## # Tags ######################################################## def get_tags(self): """ List the tags of this project. :returns: a dictionary containing the tags with a color :rtype: dict """ return self.client._perform_json("GET", "/projects/%s/tags" % self.project_key) def set_tags(self, tags=None): """ Set the tags of this project. :param dict tags: must be a modified version of the object returned by list_tags (defaults to **{}**) """ if tags is None: tags = {} return self.client._perform_empty("PUT", "/projects/%s/tags" % self.project_key, body=tags) ######################################################## # Macros ######################################################## def list_macros(self, as_objects=False): """ List the macros accessible in this project :param as_objects: if True, return the macros as :class:`dataikuapi.dss.macro.DSSMacro` macro handles instead of a list of python dicts (defaults to **False**) :returns: the list of the macros :rtype: list """ macros = self.client._perform_json( "GET", "/projects/%s/runnables/" % self.project_key) if as_objects: return [DSSMacro(self.client, self.project_key, m["runnableType"], m) for m in macros] else: return macros def get_macro(self, runnable_type): """ Get a handle to interact with a specific macro :param str runnable_type: the identifier of a macro :returns: A macro handle :rtype: :class:`dataikuapi.dss.macro.DSSMacro` """ return DSSMacro(self.client, self.project_key, runnable_type) ######################################################## # Wiki ######################################################## def get_wiki(self): """ Get the wiki :returns: the wiki associated to the project :rtype: :class:`dataikuapi.dss.wiki.DSSWiki` """ return DSSWiki(self.client, self.project_key) ######################################################## # Discussions ######################################################## def get_object_discussions(self): """ Get a handle to manage discussions on the project :returns: the handle to manage discussions :rtype: :class:`dataikuapi.dss.discussion.DSSObjectDiscussions` """ return DSSObjectDiscussions(self.client, self.project_key, "PROJECT", self.project_key) ######################################################## # Tables import ######################################################## def init_tables_import(self): """ Start an operation to import Hive or SQL tables as datasets into this project :returns: a :class:`dataikuapi.dss.project.TablesImportDefinition` to add tables to import :rtype: :class:`dataikuapi.dss.project.TablesImportDefinition` """ return TablesImportDefinition(self.client, self.project_key) def list_sql_schemas(self, connection_name): """ Lists schemas from which tables can be imported in a SQL connection :param str connection_name: name of the SQL connection :returns: an array of schemas names :rtype: list """ return self._list_schemas(connection_name) def list_hive_databases(self): """ Lists Hive databases from which tables can be imported :returns: an array of databases names :rtype: list """ return self._list_schemas("@virtual(hive-jdbc):default") def _list_schemas(self, connection_name): return self.client._perform_json("GET", "/projects/%s/datasets/tables-import/actions/list-schemas" % ( self.project_key), params={"connectionName": connection_name}) def list_sql_tables(self, connection_name, schema_name=None): """ Lists tables to import in a SQL connection :param str connection_name: name of the SQL connection :param str schema_name: Optional, name of the schema in the SQL connection in which to list tables. :returns: an array of tables :rtype: list """ ret = self.client._perform_json("GET", "/projects/%s/datasets/tables-import/actions/list-tables" % (self.project_key), params={"connectionName": connection_name, "schemaName": schema_name}) def to_schema_table_pair(x): return {"schema": x.get("schema", None), "table": x["table"]} return [to_schema_table_pair(x) for x in DSSFuture.get_result_wait_if_needed(self.client, ret)['tables']] def list_hive_tables(self, hive_database): """ Lists tables to import in a Hive database :param str hive_database: name of the Hive database :returns: an array of tables :rtype: list """ connection_name = "@virtual(hive-jdbc):" + hive_database ret = self.client._perform_json("GET", "/projects/%s/datasets/tables-import/actions/list-tables" % (self.project_key), params={"connectionName": connection_name}) def to_schema_table_pair(x): return {"schema": x.get("databaseName", None), "table": x["table"]} return [to_schema_table_pair(x) for x in DSSFuture.get_result_wait_if_needed(self.client, ret)['tables']] def list_elasticsearch_indices_or_aliases(self, connection_name): ret = self.client._perform_json("GET", "/projects/%s/datasets/tables-import/actions/list-indices" % self.project_key, params={"connectionName": connection_name}) return DSSFuture.get_result_wait_if_needed(self.client, ret) ######################################################## # App designer ######################################################## def get_app_manifest(self): """ Gets the manifest of the application if the project is an app template or an app instance, fails otherwise. :returns: the manifest of the application associated to the project :rtype: :class:`dataikuapi.dss.app.DSSAppManifest` """ raw_data = self.client._perform_json("GET", "/projects/%s/app-manifest" % self.project_key) return DSSAppManifest(self.client, raw_data, self.project_key) ######################################################## # MLflow experiment tracking ######################################################## def setup_mlflow(self, managed_folder, host=None): """ Set up the dss-plugin for MLflow :param object managed_folder: the managed folder where MLflow artifacts should be stored. Can be either a managed folder id as a string, a :class:`dataikuapi.dss.managedfolder.DSSManagedFolder`, or a :class:`dataiku.Folder` :param str host: setup a custom host if the backend used is not DSS (defaults to **None**). """ return MLflowHandle(client=self.client, project=self, managed_folder=managed_folder, host=host) def get_mlflow_extension(self): """ Get a handle to interact with the extension of MLflow provided by DSS :returns: A Mlflow Extension handle :rtype: :class:`dataikuapi.dss.mlflow.DSSMLflowExtension` """ return DSSMLflowExtension(client=self.client, project_key=self.project_key) ######################################################## # Code studios ######################################################## def list_code_studios(self, as_type="listitems"): """ List the code studio objects in this project :param str as_type: How to return the list. Supported values are "listitems" and "objects" (defaults to **listitems**). :returns: the list of the code studio objects, each one as a python dict :rtype: list """ items = self.client._perform_json( "GET", "/projects/%s/code-studios/" % self.project_key) if as_type == "listitems" or as_type == "listitem": return [DSSCodeStudioObjectListItem(self.client, self.project_key, item) for item in items] elif as_type == "objects" or as_type == "object": return [DSSCodeStudioObject(self.client, self.project_key, item["id"]) for item in items] else: raise ValueError("Unknown as_type") def get_code_studio(self, code_studio_id): """ Get a handle to interact with a specific code studio object :param str code_studio_id: the identifier of the desired code studio object :returns: A code studio object handle :rtype: :class:`dataikuapi.dss.codestudio.DSSCodeStudioObject` """ return DSSCodeStudioObject(self.client, self.project_key, code_studio_id) def create_code_studio(self, name, template_id): """ Create a new code studio object in the project, and return a handle to interact with it :param str name: the name of the code studio object :param str template_id: the identifier of a code studio template :returns: A code studio object handle :rtype: :class:`dataikuapi.dss.codestudio.DSSCodeStudioObject` """ obj = { "name": name, "templateId": template_id } res = self.client._perform_json("POST", "/projects/%s/code-studios/" % self.project_key, body=obj) code_studio_id = res['codeStudio']['id'] return DSSCodeStudioObject(self.client, self.project_key, code_studio_id) ######################################################## # Project libraries ######################################################## def get_library(self): """ Get a handle to manage the project library :returns: A :class:`dataikuapi.dss.projectlibrary.DSSLibrary` handle :rtype: :class:`dataikuapi.dss.projectlibrary.DSSLibrary` """ return DSSLibrary(self.client, self.project_key) ######################################################## # Webapps ######################################################## def list_webapps(self, as_type="listitems"): """ List the webapp heads of this project :param str as_type: How to return the list. Supported values are "listitems" and "objects". :returns: The list of the webapps. If "as_type" is "listitems", each one as a :class:`scenario.DSSWebAppListItem`. If "as_type" is "objects", each one as a :class:`scenario.DSSWebApp` :rtype: list """ webapps = self.client._perform_json("GET", "/projects/%s/webapps/" % self.project_key) if as_type == "listitems": return [DSSWebAppListItem(self.client, item) for item in webapps] elif as_type == "objects": return [DSSWebApp(self.client, self.project_key, item["id"]) for item in webapps] else: raise ValueError("Unknown as_type") def get_webapp(self, webapp_id): """ Get a handle to interact with a specific webapp :param webapp_id: the identifier of a webapp :returns: A :class:`dataikuapi.dss.webapp.DSSWebApp` webapp handle """ return DSSWebApp(self.client, self.project_key, webapp_id) class TablesImportDefinition(object): """ Temporary structure holding the list of tables to import """ def __init__(self, client, project_key): """Do not call this directly, use :meth:`DSSProject.init_tables_import`""" self.client = client self.project_key = project_key self.keys = [] def add_hive_table(self, hive_database, hive_table): """Add a Hive table to the list of tables to import :param str hive_database: the name of the Hive database :param str hive_table: the name of the Hive table """ self.keys.append({ "connectionName": "@virtual(hive-jdbc):" + hive_database, "name": hive_table }) def add_sql_table(self, connection, schema, table, catalog=None): """Add a SQL table to the list of tables to import :param str connection: the name of the SQL connection :param str schema: the schema of the table :param str table: the name of the SQL table :param str catalog: the database of the SQL table. Leave to None to use the default database associated with the connection """ self.keys.append({ "connectionName": connection, "schema": schema, "name": table, "catalog": catalog }) def add_elasticsearch_index_or_alias(self, connection, index_or_alias): """Add an Elastic Search index or alias to the list of tables to import""" self.keys.append({"connectionName": connection, "name": index_or_alias}) def prepare(self): """ Run the first step of the import process. In this step, DSS will check the tables whose import you have requested and prepare dataset names and target connections :returns: an object that allows you to finalize the import process :rtype: :class:`TablesPreparedImport` """ ret = self.client._perform_json("POST", "/projects/%s/datasets/tables-import/actions/prepare-from-keys" % ( self.project_key), body={"keys": self.keys}) future = self.client.get_future(ret["jobId"]) future.wait_for_result() return TablesPreparedImport(self.client, self.project_key, future.get_result()) class TablesPreparedImport(object): """Result of preparing a tables import. Import can now be finished""" def __init__(self, client, project_key, candidates): """Do not call this directly, use :meth:`DSSProject.init_tables_import` and then prepare""" self.client = client self.project_key = project_key self.candidates = candidates def execute(self): """ Starts executing the import in background and returns a :class:`dataikuapi.dss.future.DSSFuture` to wait on the result :returns: a future to wait on the result :rtype: :class:`dataikuapi.dss.future.DSSFuture` """ ret = self.client._perform_json("POST", "/projects/%s/datasets/tables-import/actions/execute-from-candidates" % (self.project_key), body = self.candidates) return DSSFuture.from_resp(self.client, ret) class DSSProjectSettings(object): """Settings of a DSS project""" def __init__(self, client, project_key, settings): """Do not call directly, use :meth:`DSSProject.get_settings`""" self.client = client self.project_key = project_key self.settings = settings def get_raw(self): """Gets all settings as a raw dictionary. This returns a reference to the raw settings, not a copy, so changes made to the returned object will be reflected when saving. :rtype: dict """ return self.settings def set_python_code_env(self, code_env_name): """Sets the default Python code env used by this project :param str code_env_name: Identifier of the code env to use. If None, sets the project to use the builtin Python env """ if code_env_name is None: self.settings["settings"]["codeEnvs"]["python"]["useBuiltinEnv"] = True self.settings["settings"]["codeEnvs"]["python"]["mode"] = "USE_BUILTIN_MODE" else: self.settings["settings"]["codeEnvs"]["python"]["useBuiltinEnv"] = False self.settings["settings"]["codeEnvs"]["python"]["envName"] = code_env_name self.settings["settings"]["codeEnvs"]["python"]["mode"] = "EXPLICIT_ENV" def set_r_code_env(self, code_env_name): """Sets the default R code env used by this project :param str code_env_name: Identifier of the code env to use. If None, sets the project to use the builtin R env """ if code_env_name is None: self.settings["settings"]["codeEnvs"]["r"]["useBuiltinEnv"] = True self.settings["settings"]["codeEnvs"]["r"]["mode"] = "USE_BUILTIN_MODE" else: self.settings["settings"]["codeEnvs"]["r"]["useBuiltinEnv"] = False self.settings["settings"]["codeEnvs"]["r"]["envName"] = code_env_name self.settings["settings"]["codeEnvs"]["r"]["mode"] = "EXPLICIT_ENV" def set_container_exec_config(self, config_name): """Sets the default containerized execution config used by this project :param str config_name: Identifier of the containerized execution config to use. If None, sets the project to use local execution """ if config_name is None: self.settings["settings"]["container"]["containerMode"] = "NONE" else: self.settings["settings"]["container"]["containerMode"] = "EXPLICIT_CONTAINER" self.settings["settings"]["container"]["containerConf"] = config_name def set_k8s_cluster(self, cluster, fallback_cluster=None): """Sets the Kubernetes cluster used by this project :param str cluster: Identifier of the cluster to use. May use variables expansion. If None, sets the project to use the globally-defined cluster :param str fallback_cluster: Identifier of the cluster to use if the variable used for "cluster" does not exist (defaults to **None**) """ if cluster is None: self.settings["settings"]["k8sCluster"]["clusterMode"] = "INHERIT" else: self.settings["settings"]["k8sCluster"]["clusterMode"] = "EXPLICIT_CLUSTER" self.settings["settings"]["k8sCluster"]["clusterId"] = cluster self.settings["settings"]["k8sCluster"]["defaultClusterId"] = fallback_cluster def set_cluster(self, cluster, fallback_cluster=None): """Sets the Hadoop/Spark cluster used by this project :param str cluster: Identifier of the cluster to use. May use variables expansion. If None, sets the project to use the globally-defined cluster :param str fallback_cluster: Identifier of the cluster to use if the variable used for "cluster" does not exist (defaults to **None**) """ if cluster is None: self.settings["settings"]["cluster"]["clusterMode"] = "INHERIT" else: self.settings["settings"]["cluster"]["clusterMode"] = "EXPLICIT_CLUSTER" self.settings["settings"]["cluster"]["clusterId"] = cluster self.settings["settings"]["cluster"]["defaultClusterId"] = fallback_cluster def add_exposed_object(self, object_type, object_id, target_project): """ Exposes an object from this project to another project. Does nothing if the object was already exposed to the target project :param str object_type: type of the object to expose :param str object_id: id of the object to expose :param str target_project: id of the project in which to expose the object """ found_eo = None for eo in self.settings["exposedObjects"]["objects"]: if eo["type"] == object_type and eo["localName"] == object_id: found_eo = eo break if found_eo is None: found_eo = {"type": object_type, "localName": object_id, "rules": []} self.settings["exposedObjects"]["objects"].append(found_eo) already_exists = False for rule in found_eo["rules"]: if rule["targetProject"] == target_project: already_exists = True break if not already_exists: found_eo["rules"].append({"targetProject": target_project}) def save(self): """Saves back the settings to the project""" self.client._perform_empty("PUT", "/projects/%s/settings" % (self.project_key), body=self.settings) class JobDefinitionBuilder(object): """ Helper to run a job. Do not create this class directly, use :meth:`DSSProject.new_job` """ def __init__(self, project, job_type="NON_RECURSIVE_FORCED_BUILD"): self.project = project self.definition = {'type': job_type, 'refreshHiveMetastore': False, 'outputs': []} def with_type(self, job_type): """ Sets the build type :param job_type: the build type for the job RECURSIVE_BUILD, NON_RECURSIVE_FORCED_BUILD, RECURSIVE_FORCED_BUILD, RECURSIVE_MISSING_ONLY_BUILD """ self.definition['type'] = job_type return self def with_refresh_metastore(self, refresh_metastore): """ Sets whether the hive tables built by the job should have their definitions refreshed after the corresponding dataset is built :param bool refresh_metastore: """ self.definition['refreshHiveMetastore'] = refresh_metastore return self def with_output(self, name, object_type=None, object_project_key=None, partition=None): """ Adds an item to build in this job :param name: name of the output object :param object_type: type of object to build from: DATASET, MANAGED_FOLDER, SAVED_MODEL, STREAMING_ENDPOINT (defaults to **None**) :param object_project_key: PROJECT_KEY for the project that contains the object to build (defaults to **None**) :param partition: specify partition to build (defaults to **None**) """ self.definition['outputs'].append( {'type': object_type, 'id': name, 'projectKey': object_project_key, 'partition': partition}) return self def get_definition(self): """Gets the internal definition for this job""" return self.definition def start(self): """ Starts the job, and return a :doc:`dataikuapi.dss.job.DSSJob` handle to interact with it. You need to wait for the returned job to complete :returns: a job handle :rtype: :class:`dataikuapi.dss.job.DSSJob` """ job_def = self.project.client._perform_json("POST", "/projects/%s/jobs/" % self.project.project_key, body=self.definition) return DSSJob(self.project.client, self.project.project_key, job_def['id']) def start_and_wait(self, no_fail=False): """ Starts the job, waits for it to complete and returns a :doc:`dataikuapi.dss.job.DSSJob` handle to interact with it Raises if the job failed. :param no_fail: if True, does not raise if the job failed (defaults to **False**). :returns: A job handle :rtype: :class:`dataikuapi.dss.job.DSSJob` """ job = self.start() waiter = DSSJobWaiter(job) waiter.wait(no_fail) return job dataiku-api-client-python-11.4.0/dataikuapi/dss/projectdeployer.py000066400000000000000000001130601440464736400252670ustar00rootroot00000000000000from .future import DSSFuture class DSSProjectDeployer(object): """ Handle to interact with the Project Deployer. .. important:: Do not instantiate directly, use :meth:`dataikuapi.DSSClient.get_projectdeployer()` """ def __init__(self, client): self.client = client def list_deployments(self, as_objects=True): """ List deployments on the Project Deployer. Usage example: .. code-block:: python # list all deployments with their current state for deployment in deployer.list_deployments(): status = deployment.get_status() print("Deployment %s is %s" % (deployment.id, status.get_health())) :param boolean as_objects: if True, returns a list of :class:`DSSProjectDeployerDeployment`, else returns a list of dict. :returns: list of deployments, either as :class:`DSSProjectDeployerDeployment` or as dict (with fields as in :meth:`DSSProjectDeployerDeploymentStatus.get_light()`) :rtype: list """ l = self.client._perform_json("GET", "/project-deployer/deployments") if as_objects: return [DSSProjectDeployerDeployment(self.client, x["deploymentBasicInfo"]["id"]) for x in l] else: return l def get_deployment(self, deployment_id): """ Get a handle to interact with a deployment. :param string deployment_id: identifier of a deployment :rtype: :class:`DSSProjectDeployerDeployment` """ return DSSProjectDeployerDeployment(self.client, deployment_id) def create_deployment(self, deployment_id, project_key, infra_id, bundle_id, deployed_project_key=None, project_folder_id=None, ignore_warnings=False): """ Create a deployment and return the handle to interact with it. The returned deployment is not yet started and you need to call :meth:`~DSSProjectDeployerDeployment.start_update` Usage example: .. code-block:: python # create and deploy a bundle project = 'my-project' infra = 'my-infra' bundle = 'my-bundle' deployment_id = '%s-%s-on-%s' % (project, bundle, infra) deployment = deployer.create_deployment(deployment_id, project, infra, bundle) update = deployment.start_update() update.wait_for_result() :param string deployment_id: identifier of the deployment to create :param string project_key: key of the published project :param string bundle_id: identifier of the bundle to deploy :param string infra_id: identifier of the infrastructure to use :param string deployed_project_key: The project key to use when deploying this project to the automation node. If not set, the project will be created with the same project key as the published project :param string project_folder_id: The automation node project folder id to deploy this project into. If not set, the project will be created in the root folder :param boolean ignore_warnings: ignore warnings concerning the governance status of the bundle to deploy :return: a new deployment :rtype: :class:`DSSProjectDeployerDeployment` """ settings = { "deploymentId": deployment_id, "publishedProjectKey": project_key, "infraId": infra_id, "bundleId": bundle_id } if deployed_project_key: settings["deployedProjectKey"] = deployed_project_key if project_folder_id: settings["projectFolderId"] = project_folder_id self.client._perform_json("POST", "/project-deployer/deployments", params={"ignoreWarnings": ignore_warnings}, body=settings) return self.get_deployment(deployment_id) def list_stages(self): """ List the possible stages for infrastructures. :return: list of stages. Each stage is returned as a dict with fields: * **id** : identifier of the stage * **desc** : description of the stage :rtype: list[dict] """ return self.client._perform_json("GET", "/project-deployer/stages") def list_infras(self, as_objects=True): """ List the infrastructures on the Project Deployer. Usage example: .. code-block:: python # list infrastructures that the user can deploy to for infrastructure in deployer.list_infras(as_objects=False): if infrastructure.get("canDeploy", False): print("User can deploy to %s" % infrastructure["infraBasicInfo"]["id"]) :param boolean as_objects: if True, returns a list of :class:`DSSProjectDeployerInfra`, else returns a list of dict. :return: list of infrastructures, either as :class:`DSSProjectDeployerInfra` or as dict (with fields as in :meth:`DSSProjectDeployerInfraStatus.get_raw()`) :rtype: list """ l = self.client._perform_json("GET", "/project-deployer/infras") if as_objects: return [DSSProjectDeployerInfra(self.client, x["infraBasicInfo"]["id"]) for x in l] else: return l def create_infra(self, infra_id, stage, govern_check_policy="NO_CHECK"): """ Create a new infrastructure and returns the handle to interact with it. :param string infra_id: unique identifier of the infrastructure to create :param string stage: stage of the infrastructure to create :param string govern_check_policy: what actions with Govern the the deployer will take whe bundles are deployed on this infrastructure. Possible values: PREVENT, WARN, or NO_CHECK :return: a new infrastructure :rtype: :class:`DSSProjectDeployerInfra` """ settings = { "id": infra_id, "stage": stage, "governCheckPolicy": govern_check_policy, } self.client._perform_json("POST", "/project-deployer/infras", body=settings) return self.get_infra(infra_id) def get_infra(self, infra_id): """ Get a handle to interact with an infrastructure. :param string infra_id: identifier of the infrastructure to get :rtype: :class:`DSSProjectDeployerInfra` """ return DSSProjectDeployerInfra(self.client, infra_id) def list_projects(self, as_objects=True): """ List published projects on the Project Deployer. Usage example: .. code-block:: python # list project that the user can deploy bundles from for project in deployer.list_projects(as_objects=False): if project.get("canDeploy", False): print("User can deploy to %s" % project["projectBasicInfo"]["id"]) :param boolean as_objects: if True, returns a list of :class:`DSSProjectDeployerProject`, else returns a list of dict. :return: list of published projects, either as :class:`DSSProjectDeployerProject` or as dict (with fields as in :meth:`DSSProjectDeployerProjectStatus.get_raw()`) :rtype: list """ l = self.client._perform_json("GET", "/project-deployer/projects") if as_objects: return [DSSProjectDeployerProject(self.client, x["projectBasicInfo"]["id"]) for x in l] else: return l def create_project(self, project_key): """ Create a new published project on the Project Deployer and return the handle to interact with it. :param string project_key: key of the project to create :rtype: :class:`DSSProjectDeployerProject` """ settings = { "publishedProjectKey": project_key } self.client._perform_json("POST", "/project-deployer/projects", body=settings) return self.get_project(project_key) def get_project(self, project_key): """ Get a handle to interact with a published project. :param string project_key: key of the project to get :rtype: :class:`DSSProjectDeployerProject` """ return DSSProjectDeployerProject(self.client, project_key) def upload_bundle(self, fp, project_key=None): """ Upload a bundle archive for a project. :param file-like fp: a bundle archive (should be a zip) :param string project_key: key of the published project where the bundle will be uploaded. If the project does not exist, it is created. If not set, the key of the bundle's source project is used. """ if project_key is None: params = None else: params = { "projectKey": project_key, } self.client._perform_empty("POST", "/project-deployer/projects/bundles", params=params, files={"file":fp}) ############################################### # Infrastructures ############################################### class DSSProjectDeployerInfra(object): """ An Automation infrastructure on the Project Deployer. .. important:: Do not instantiate directly, use :meth:`DSSProjectDeployer.get_infra`. """ def __init__(self, client, infra_id): self.client = client self.infra_id = infra_id @property def id(self): """ Get the unique identifier of the infrastructure. :rtype: string """ return self.infra_id def get_status(self): """ Get status information about this infrastructure. :return: the current status :rtype: :class:`DSSProjectDeployerInfraStatus` """ light = self.client._perform_json("GET", "/project-deployer/infras/%s" % (self.infra_id)) return DSSProjectDeployerInfraStatus(self.client, self.infra_id, light) def get_settings(self): """ Get the settings of this infrastructure. :rtype: :class:`DSSProjectDeployerInfraSettings` """ settings = self.client._perform_json( "GET", "/project-deployer/infras/%s/settings" % (self.infra_id)) return DSSProjectDeployerInfraSettings(self.client, self.infra_id, settings) def delete(self): """ Delete this infra. .. note:: You may only delete an infra if there are no deployments using it. """ self.client._perform_empty( "DELETE", "/project-deployer/infras/%s" % (self.infra_id)) class DSSProjectDeployerInfraSettings(object): """ The settings of an Automation infrastructure. .. important:: Do not instantiate directly, use :meth:`DSSProjectDeployerInfra.get_settings` To modify the settings, modify them in the dict returned by :meth:`get_raw()` then call :meth:`save()`. """ def __init__(self, client, infra_id, settings): self.client = client self.infra_id = infra_id self.settings = settings def get_raw(self): """ Get the raw settings of this infrastracture. This returns a reference to the raw settings, not a copy, so changes made to the returned object will be reflected when saving. :return: the settings, as a dict with fields: * **id** : unique identifier of the infrastructure * **stage** : name of the stage of the infrastructure * **governCheckPolicy** : what actions with Govern the deployer will take when bundles are deployed on this infrastructure. Possible values: PREVENT, WARN, or NO_CHECK * **autoconfigureFromNodesDirectory** : whether this infrastructure is automatically setup according to what's in the fleet * **nodeId** : when configured from a fleet, the name of the automation node in the fleet * **automationNodeUrl** : URL of the automation node that this infrastructure points to * **automationNodeExternalUrl** : externally-accessible URL of the automation node that this infrastructure points to * **adminApiKey** : API key used to communicate with the automation node * **trustAllSSLCertificates** : whether to verify SSL certificates when communicating with the automation node * **permissions** : list of permissions per group, each as a dict of: * **group** : name of the group being granted the permissions * **admin** : whether the group can administer the infrastructure * **read** : whether the group can see the deployments running on the infrastructure * **deploy** : whether the group can create deployments on the infrastructure :rtype: dict """ return self.settings def save(self): """ Save back these settings to the infrastracture. """ self.client._perform_empty( "PUT", "/project-deployer/infras/%s/settings" % (self.infra_id), body = self.settings) class DSSProjectDeployerInfraStatus(object): """ The status of an Automation infrastructure. .. important:: Do not instantiage directly, use :meth:`DSSProjectDeployerInfra.get_status` """ def __init__(self, client, infra_id, light_status): self.client = client self.infra_id = infra_id self.light_status = light_status def get_deployments(self): """ Get the deployments that are deployed on this infrastructure. :return: a list of deployments :rtype: list of :class:`DSSProjectDeployerDeployment` """ return [DSSProjectDeployerDeployment(self.client, deployment["id"]) for deployment in self.light_status["deployments"]] def get_raw(self): """ Get the raw status information. :return: the status, as a dict with fields: * **isAdmin** : whether the user can administer the infrastructure * **canDeploy** : whether the user can deploy bundles on the infrastructure * **infraBasicInfo** : summary of the infrastructure, as a dict with fields: * **id** : unique identifier of the infrastructure * **stage** : name of the stage of the infrastructure * **governCheckPolicy** : what actions with Govern the deployer will take when bundles are deployed on this infrastructure. Possible values: PREVENT, WARN, or NO_CHECK * **automationNodeUrl** : URL of the automation node that this infrastructure points to * **automationNodeExternalUrl** : externally-accessible URL of the automation node that this infrastructure points to * **deployments** : list of summaries of the deployments on the infrastructure, as a list of dict with fields: * **id** : identifier of the deployment * **infraId** : identifier of the infrastructure * **tags** : list of tags, each a string * **createdByDisplayName** : login of the user who created the deployment * **lastModifiedByDisplayName** : login of the user who last modified the deployment * **publishedProjectKey** : key of the published project of the deployment * **bundleId** : identifier of the bundle of the published project that the deployment pushes onto the automation node * **deployedProjectKey** : key of the remote project that this deployment is pushed to :rtype: dict """ return self.light_status ############################################### # Deployments ############################################### class DSSProjectDeployerDeployment(object): """ A deployment on the Project Deployer. .. important:: Do not instantiate directly, use :meth:`DSSProjectDeployer.get_deployment` """ def __init__(self, client, deployment_id): self.client = client self.deployment_id = deployment_id @property def id(self): """ Get the identifier of the deployment. :rtype: string """ return self.deployment_id def get_status(self): """ Get status information about this deployment. :rtype: dataikuapi.dss.apideployer.DSSProjectDeployerDeploymentStatus """ light = self.client._perform_json("GET", "/project-deployer/deployments/%s" % (self.deployment_id)) heavy = self.client._perform_json("GET", "/project-deployer/deployments/%s/status" % (self.deployment_id)) return DSSProjectDeployerDeploymentStatus(self.client, self.deployment_id, light, heavy) def get_governance_status(self, bundle_id=""): """ Get the governance status about this deployment. The infrastructure on which this deployment is running needs to have a Govern check policy of PREVENT or WARN. :param string bundle_id: (Optional) The ID of a specific bundle of the published project to get status from. If empty, the bundle currently used in the deployment. :return: messages about the governance status, as a dict with a **messages** field, itself a list of meassage information, each one a dict of: * **severity** : severity of the error in the message. Possible values are SUCCESS, INFO, WARNING, ERROR * **isFatal** : for ERROR **severity**, whether the error is considered fatal to the operation * **code** : a string with a well-known code documented in `DSS doc `_ * **title** : short message * **message** : the error message * **details** : a more detailed error description :rtype: dict """ return self.client._perform_json("GET", "/project-deployer/deployments/%s/governance-status" % (self.deployment_id), params={ "bundleId": bundle_id }) def get_settings(self): """ Get the settings of this deployment. :rtype: :class:`DSSProjectDeployerDeploymentSettings` """ settings = self.client._perform_json( "GET", "/project-deployer/deployments/%s/settings" % (self.deployment_id)) return DSSProjectDeployerDeploymentSettings(self.client, self.deployment_id, settings) def start_update(self): """ Start an asynchronous update of this deployment. After the update, the deployment should be matching the actual state to the current settings. :returns: a handle on the update operation :rtype: :class:`dataikuapi.dss.future.DSSFuture` """ future_response = self.client._perform_json( "POST", "/project-deployer/deployments/%s/actions/update" % (self.deployment_id)) return DSSFuture(self.client, future_response.get('jobId', None), future_response) def delete(self): """ Deletes this deployment .. note:: You may only delete a deployment if it is disabled and has been updated after disabling it. """ return self.client._perform_empty( "DELETE", "/project-deployer/deployments/%s" % (self.deployment_id)) class DSSProjectDeployerDeploymentSettings(object): """ The settings of a Project Deployer deployment. .. important:: Do not instantiate directly, use :meth:`DSSProjectDeployerDeployment.get_settings` To modify the settings, modify them in the dict returned by :meth:`get_raw()`, or change the value of :meth:`bundle_id()`, then call :meth:`save()`. """ def __init__(self, client, deployment_id, settings): self.client = client self.deployment_id = deployment_id self.settings = settings def get_raw(self): """ Get the raw settings of this deployment. This returns a reference to the raw settings, not a copy, so changes made to the returned object will be reflected when saving. :return: the settings, as a dict of: * **id** : identifier of the deployment * **infraId** : identifier of the infrastructure on which the deployment is done * **tags** : list of tags, each one a string * **publishedProjectKey** : key of the published project on the Project Deployer * **bundleId** : identifier of the bundle of the published project being deployed * **deployedProjectKey** : key of the project the deployment is deployed to on the automation node * **projectFolderId** : identifier of the project folder on the automation node * **bundleContainerSettings** : bundle settings on the automation node, as a dict of: * **remapping** : remapping settings for connections and code envs for the bundle on the automation node, as a dict with fields: * **connections** : list of remappings, each a dict of **source** and **target** fields holding connection names * **codeEnvs** : list of remappings, each a dict of **source** and **target** fields holding code env names * **codeEnvsBehavior** : defines the behavior w.r.t. code envs used by the bundle, as a dict of: * **importTimeMode** : one of INSTALL_IF_MISS, FAIL_IF_MISS or DO_NOTHING * **envImportSpecificationMode** : one of SPECIFIED or ACTUAL * **localVariables** : override to the project local variables on the automation node, as a dict * **scenariosToActivate** : dict of scenario name to boolean, controlling which scenarios of the bundle are (de)activated upon deployment * **disableAutomaticTriggers** : whether the automatic triggers in the scenario should be all deactivated after the deployment :rtype: dict """ return self.settings @property def bundle_id(self): """ Get or set the identifier of the bundle currently used by this deployment. If setting the value, you need to call :meth:`save()` afterward for the change to be effective. """ return self.settings["bundleId"] @bundle_id.setter def bundle_id(self, new_bundle_id): self.settings["bundleId"] = new_bundle_id def save(self, ignore_warnings=False): """ Save back these settings to the deployment. :param boolean ignore_warnings: whether to ignore warnings concerning the governance status of the bundle to deploy """ self.client._perform_empty( "PUT", "/project-deployer/deployments/%s/settings" % (self.deployment_id), params = { "ignoreWarnings" : ignore_warnings }, body = self.settings) class DSSProjectDeployerDeploymentStatus(object): """ The status of a deployment on the Project Deployer. .. important:: Do not instantiate directly, use :meth:`DSSProjectDeployerDeployment.get_status` """ def __init__(self, client, deployment_id, light_status, heavy_status): self.client = client self.deployment_id = deployment_id self.light_status = light_status self.heavy_status = heavy_status def get_light(self): """ Get the 'light' (summary) status. This returns a dictionary with various information about the deployment, but not the actual health of the deployment :returns: a summary, as a dict with fields: * **deploymentBasicInfo** : summary of the definition of the deployment, as a dict of: * **id** : identifier of the deployment * **infraId** : identifier of the infrastructure * **tags** : list of tags, each a string * **createdByDisplayName** : login of the user who created the deployment * **lastModifiedByDisplayName** : login of the user who last modified the deployment * **publishedProjectKey** : key of the published project of the deployment * **bundleId** : identifier of the bundle of the published project that the deployment pushes onto the automation node * **deployedProjectKey** : key of the remote project that this deployment is pushed to * **infraBasicInfo** : summary of the infrastructure on which the deployment is made, as a dict of: * **id** : unique identifier of the infrastructure * **stage** : name of the stage of the infrastructure * **governCheckPolicy** : what actions with Govern the deployer will take when bundles are deployed on this infrastructure. Possible values: PREVENT, WARN, or NO_CHECK * **automationNodeUrl** : URL of the automation node that this infrastructure points to * **automationNodeExternalUrl** : externally-accessible URL of the automation node that this infrastructure points to * **projectBasicInfo** : summary of the published project from where the bundle of this deployment originates, as a dict of: * **id** : key of the project * **name** : name of the project * **packages** : list of bundles in the published project of this deployment (see :meth:`DSSProjectDeployerProjectStatus.get_bundles()`) * **neverEverDeployed** : True if the deployment hasn't yet been deployed :rtype: dict """ return self.light_status def get_heavy(self): """ Get the 'heavy' (full) status. This returns various information about the deployment, notably its health. :return: a status, as a dict with fields: * **deploymentId** : identifier of the deployment * **health** : the current health of the deployment. Possible values: UNKNOWN, ERROR, WARNING, HEALTHY, UNHEALTHY, OUT_OF_SYNC * **healthMessages** : detailed messages of errors or warnings that occurred while checking the health * **monitoring** : information about the scenarios in the project on the automation node, as a dict of: * **hasScenarios** : whether there are scenarios in the project * **hasActiveScenarios** : whether there are active scenarios in the project * **failed** : list of names of the scenarios whose last run ended in failed state * **warning** : list of names of the scenarios whose last run ended in warning state * **successful** : list of names of the scenarios whose last run ended in success state * **aborted** : list of names of the scenarios whose last run was aborted * **running** : list of names of the scenarios currently running :rtype: dict """ return self.heavy_status def get_health(self): """ Get the health of this deployment. :returns: possible values are UNKNOWN, ERROR, WARNING, HEALTHY, UNHEALTHY, OUT_OF_SYNC :rtype: string """ return self.heavy_status["health"] def get_health_messages(self): """ Get messages about the health of this deployment :return: a dict with a **messages** field, which is a list of meassage information, each one a dict of: * **severity** : severity of the error in the message. Possible values are SUCCESS, INFO, WARNING, ERROR * **isFatal** : for ERROR **severity**, whether the error is considered fatal to the operation * **code** : a string with a well-known code documented in `DSS doc `_ * **title** : short message * **message** : the error message * **details** : a more detailed error description :rtype: dict """ return self.heavy_status["healthMessages"] ############################################### # Published Project ############################################### class DSSProjectDeployerProject(object): """ A published project on the Project Deployer. .. important:: Do not instantiate directly, use :meth:`DSSProjectDeployer.get_project` """ def __init__(self, client, project_key): self.client = client self.project_key = project_key @property def id(self): """ Get the key of the published project. :rtype: string """ return self.project_key def get_status(self): """ Get status information about this published project. This is used mostly to get information about which versions are available and which deployments are exposing this project :rtype: :class:`DSSProjectDeployerProjectStatus` """ light = self.client._perform_json("GET", "/project-deployer/projects/%s" % (self.project_key)) return DSSProjectDeployerProjectStatus(self.client, self.project_key, light) def get_settings(self): """ Get the settings of this published project. The main things that can be modified in a project settings are permissions :rtype: :class:`DSSProjectDeployerProjectSettings` """ settings = self.client._perform_json( "GET", "/project-deployer/projects/%s/settings" % (self.project_key)) return DSSProjectDeployerProjectSettings(self.client, self.project_key, settings) def delete_bundle(self, bundle_id): """ Delete a bundle from this published project. :param string bundle_id: identifier of the bundle to delete """ self.client._perform_empty( "DELETE", "/project-deployer/projects/%s/bundles/%s" % (self.project_key, bundle_id)) def delete(self): """ Delete this published project. .. note:: You may only delete a published project if there are no deployments using it. """ self.client._perform_empty( "DELETE", "/project-deployer/projects/%s" % (self.project_key)) class DSSProjectDeployerProjectSettings(object): """ The settings of a published project. .. important:: Do not instantiate directly, use :meth:`DSSProjectDeployerProject.get_settings` To modify the settings, modify them in the dict returned by :meth:`get_raw()` then call :meth:`save()`. """ def __init__(self, client, project_key, settings): self.client = client self.project_key = project_key self.settings = settings def get_raw(self): """ Get the raw settings of this published project. This returns a reference to the raw settings, not a copy, so changes made to the returned object will be reflected when saving. :return: the settings, as a dict with fields: * **id** : key of the project * **name** : name of the project * **owner** : owner of the published project (independent of the owner of the project on the design node) * **permissions** : list of permissions per group, each as a dict of: * **group** : name of the group being granted the permissions * **admin** : whether the group can administer the project * **read** : whether the group can see the published project and deployments of bundles of this project * **write** : whether the group can upload new bundles in this published project * **deploy** : whether the group can create deployments from bundles of this project * **basicImageInfo** : information about the image associated to the project (in project lists, on the project's home, etc...) :rtype: dict """ return self.settings def save(self): """ Save back these settings to the published project. """ self.client._perform_empty( "PUT", "/project-deployer/projects/%s/settings" % (self.project_key), body = self.settings) class DSSProjectDeployerProjectStatus(object): """ The status of a published project. .. important:: Do not instantiate directly, use :meth:`DSSProjectDeployerProject.get_status` """ def __init__(self, client, project_key, light_status): self.client = client self.project_key = project_key self.light_status = light_status def get_deployments(self, infra_id=None): """ Get the deployments that have been created from this published project. :param string infra_id: (optional) identifier of an infrastructure. When set, only get the deployments deployed on this infrastructure. When not set, the list contains all the deployments using this published project, across every infrastructure of the Project Deployer. :returns: a list of deployments, each a :class:`DSSProjectDeployerDeployment` :rtype: list """ if infra_id is None: return [DSSProjectDeployerDeployment(self.client, deployment["id"]) for deployment in self.light_status["deployments"]] return [DSSProjectDeployerDeployment(self.client, deployment["id"]) for deployment in self.light_status["deployments"] if infra_id == deployment["infraId"]] def get_bundles(self): """ Get the bundles that have been published on this project. Each bundle is a dict that contains at least a "id" field, which is the version identifier :returns: a list of bundles, each one a dict of: * **id** : the bundle id * **publishedOn** : timestamp of when the bundle was added to the published project * **publishedBy** : login of the user who added the bundle to the published project * **designNodeInfo** : extra information on the node from which the bundle came, as a dict of * **projectKey** : the key of the project on the source node * **nodeId** : node name in the fleet (if the design node is part of a fleet) * **url** : node url in the fleet (if the design node is part of a fleet) :rtype: list[dict] """ return self.light_status["packages"] def get_infras(self): """ Get the infrastructures that deployments of this project use. :returns: list of summaries of infrastructures, each a dict of: * **id** : unique identifier of the infrastructure * **stage** : name of the stage of the infrastructure * **governCheckPolicy** : what actions with Govern the deployer will take when bundles are deployed on this infrastructure. Possible values: PREVENT, WARN, or NO_CHECK * **automationNodeUrl** : URL of the automation node that this infrastructure points to * **automationNodeExternalUrl** : externally-accessible URL of the automation node that this infrastructure points to :rtype: list[dict] """ return self.light_status["infras"] def get_raw(self): """ Gets the raw status information. :return: the status, as a dict with fields: * **projectBasicInfo** : summary of the published project, as a dict of: * **id** : key of the project * **name** : name of the project * **isAdmin** : whether the user can administer the published project * **canDeploy** : whether the user can deploy the published project * **canWrite** : whether the user can upload bundles to the published project * **packages** : list of bundles in the published project (see :meth:`get_bundles()`) * **deployments** : list of summaries of the deployments of bundles of the published project, as a list of dict with fields: * **id** : identifier of the deployment * **infraId** : identifier of the infrastructure * **tags** : list of tags, each a string * **createdByDisplayName** : login of the user who created the deployment * **lastModifiedByDisplayName** : login of the user who last modified the deployment * **publishedProjectKey** : key of the published project of the deployment * **bundleId** : identifier of the bundle of the published project that the deployment pushes onto the automation node * **deployedProjectKey** : key of the remote project that this deployment is pushed to * **infras** : list of summaries of infrastructures, for infrastructures appearing in **deployments** (see :meth:`get_infras()`) :rtype: dict """ return self.light_status dataiku-api-client-python-11.4.0/dataikuapi/dss/projectfolder.py000066400000000000000000000235671440464736400247330ustar00rootroot00000000000000from .project import DSSProject import warnings class DSSProjectFolder(object): """ A handle for a project folder on the DSS instance. .. important:: Do not instantiate this class directly, instead use :meth:`dataikuapi.DSSClient.get_project_folder` or :meth:`dataikuapi.DSSClient.get_root_project_folder`. """ def __init__(self, client, data): self.client = client self._data = data ######################################################## # Project folder basics ######################################################## @property def id(self): """ :returns: The project folder id. :rtype: string """ return self._data["id"] @property def project_folder_id(self): """ .. caution:: Deprecated. Please use :attr:`dataikuapi.dss.projectfolder.DSSProjectFolder.id`. """ warnings.warn("DSSProjectFolder.project_folder_id is deprecated, please use id", DeprecationWarning) return self.id @property def name(self): """ :returns: The project folder name or :const:`None` for the root project folder. :rtype: string """ return self._data.get("name", None) def get_name(self): """ See :attr:`dataikuapi.dss.projectfolder.DSSProjectFolder.name`. :returns: The project folder name or :const:`None` for the root project folder. :rtype: string """ return self.name def get_path(self): """ :returns: The project folder path from the root project folder (e.g. :const:`'/'` or :const:`'/foo/bar'`). :rtype: string """ parent = self.get_parent() if parent is not None: path = parent.get_path() return ("" if path == "/" else path) + "/" + self.name else: return "/" def get_parent(self): """ :returns: A handle for the parent folder or :const:`None` for the root project folder. :rtype: :class:`dataikuapi.dss.projectfolder.DSSProjectFolder` """ parent_id = self._data.get("parentId", None) if parent_id is None: return None else: return self.client.get_project_folder(parent_id) def list_child_folders(self): """ :returns: Handles for every child project folder. :rtype: list of :class:`dataikuapi.dss.projectfolder.DSSProjectFolder` """ return [self.client.get_project_folder(child_id) for child_id in self._data["childrenIds"]] def list_project_keys(self): """ :returns: The project keys of all projects stored in this project folder. :rtype: list of string """ return self._data["projectKeys"] def list_projects(self): """ :returns: Handles for every project stored in this project folder. :rtype: list of :class:`dataikuapi.dss.project.DSSProject` """ return [self.client.get_project(pkey) for pkey in self.list_project_keys()] ######################################################## # Project folder deletion ######################################################## def delete(self): """ Delete this project folder. .. important:: This project folder must be empty, i.e. contain no project or subfolder. You must move or remove all this project folder content prior to deleting it. .. attention:: This call requires an API key with admin rights. """ return self.client._perform_empty("DELETE", "/project-folders/%s" % self.id) ######################################################## # Project folder settings ######################################################## def get_settings(self): """ :returns: A handle for this project folder settings. :rtype: :class:`dataikuapi.dss.projectfolder.DSSProjectFolderSettings` """ settings = self.client._perform_json("GET", "/project-folders/%s/settings" % self.id) return DSSProjectFolderSettings(self.client, self.id, settings) ######################################################## # Project folder subfolder creation ######################################################## def create_sub_folder(self, name): """ Create a project subfolder inside this project folder. :param str name: The name of the project subfolder to create. :returns: A handle for the created project subfolder. :rtype: :class:`dataikuapi.dss.projectfolder.DSSProjectFolder` """ params = { "name": name } pf = self.client._perform_json("POST", "/project-folders/%s/children" % self.id, params=params) return DSSProjectFolder(self.client, pf) ######################################################## # Project creation ######################################################## def create_project(self, project_key, name, owner, description=None, settings=None): """ Create a new project within this project folder. Return a handle for the created project. .. important:: The provided identifier for the new project must be globally unique. .. attention:: This call requires an API key with admin rights or the right to create a project. :param str project_key: The identifier for the new project. Must be globally unique. :param str name: The displayed name for the new project. :param str owner: The login of the new project owner. :param str description: The description for the new project. :param dict settings: The initial settings for the new project. The settings can be modified later. The exact possible settings are not documented. :returns: A handle for the created project. :rtype: :class:`dataikuapi.dss.project.DSSProject` """ return self.client.create_project(project_key, name, owner, description=description, settings=settings, project_folder_id=self.id) ######################################################## # Project folder move ######################################################## def move_to(self, destination): """ Move this project folder into another project folder. :param destination: The new parent project folder of this project folder. :type destination: :class:`dataikuapi.dss.projectfolder.DSSProjectFolder` """ params = { "destination": destination.id } self.client._perform_empty("POST", "/project-folders/%s/move" % self.id, params=params) ######################################################## # Project move ######################################################## def move_project_to(self, project_key, destination): """ Move a project from this project folder into another project folder. :param str project_key: The identifier of the project to move. :param destination: The new parent project folder of the project. :type destination: :class:`dataikuapi.dss.projectfolder.DSSProjectFolder` """ # Be nice with what people pass if isinstance(project_key, DSSProject): project_key = project_key.project_key params = { "destination": destination.id } self.client._perform_empty("POST", "/project-folders/%s/projects/%s/move" % (self.id, project_key), params=params) class DSSProjectFolderSettings(object): """ A handle for a project folder settings. .. important:: Do not instantiate this class directly, instead use :meth:`dataikuapi.dss.projectfolder.DSSProjectFolder.get_settings`. """ def __init__(self, client, project_folder_id, settings): self.client = client self.id = project_folder_id self.settings = settings def get_raw(self): """ Get the settings of the project folder as a python dict. .. important:: Returns a reference to the raw settings in opposition to a copy. Changes through the reference will be effective upon saving. :returns: The settings of the project folder as a python dict containing the keys: * **name**: the name of the project folder, * **owner**: the login of the project folder owner, * **permissions**: the list of the project folder permissions. :rtype: python dict """ return self.settings def get_name(self): """ :returns: The name of the project folder. :rtype: string """ return self.settings.get("name", None) def set_name(self, name): """ Set the name of the project folder. :param str name: The new name of the project folder. """ self.settings["name"] = name def get_owner(self): """ :returns: The login of the project folder owner. :rtype: string """ return self.settings.get("owner", None) def set_owner(self, owner): """ Set the owner of the project folder. :param str owner: The login of the new project folder owner. """ self.settings["owner"] = owner def get_permissions(self): """ Get the permissions of the project folder. .. important:: Returns a reference to the permissions in opposition to a copy. Changes through the reference will be effective upon saving. :returns: The permissions of the project folder. :rtype: list of string """ return self.settings["permissions"] def save(self): """ Save back the settings to the project folder. """ self.client._perform_empty("PUT", "/project-folders/%s/settings" % (self.id), body = self.settings) dataiku-api-client-python-11.4.0/dataikuapi/dss/projectlibrary.py000066400000000000000000000315571440464736400251220ustar00rootroot00000000000000from ..utils import DataikuException import sys if sys.version_info >= (3,0): import urllib.parse dku_quote_fn = urllib.parse.quote else: import urllib dku_quote_fn = urllib.quote class DSSLibrary(object): """ A handle to manage the library of a project It saves locally a copy of taxonomy to help navigate in the library All modifications done through this object and related library items are done locally and on remote. Note: Taxonomy modifications done outside this library are not reflected locally. You should reload the library in this case. """ def __init__(self, client, project_key): """Do not call directly, use :meth:`dataikuapi.dss.project.DSSProject.get_library`""" self.client = client self.project_key = project_key self.root = self._build_node_("/", None, self.client._perform_json("GET", "/projects/%s/libraries/contents" % (self.project_key))) def _build_node_(self, name, parent, children): """ Helper that builds the tree recursively :param: str name: the name of the library item :param: :class:`dataikuapi.dss.projectlibrary.DSSLibraryFolder` parent: the parent of the library item :param: list children: the children of the library item :returns: a new node corresponding to an item in the library :rtype: :class:`dataikuapi.dss.projectlibrary.DSSLibraryItem` """ if children is None: return DSSLibraryFile(self.client, self.project_key, name, parent) item = DSSLibraryFolder(self.client, self.project_key, name, parent, set()) for child in children: item._add_child_(self._build_node_(child["name"], item, child.get("children"))) return item def list(self, folder_path="/"): """ Lists the contents in the given library folder or on the root if no folder is given. :param: str folder_path: the folder path (optional). If no path is given, it is defaulted to the root path. :returns: the list of contents in the library folder :rtype: list of :class:`dataikuapi.dss.projectlibrary.DSSLibraryItem` """ return self.get_folder(folder_path).list() def get_file(self, path): """ Retrieves a file in the library :param: str path: the file path :returns: the file in the given path :rtype: :class:`dataikuapi.dss.projectlibrary.DSSLibraryFile` """ return self.root.get_file(path) def get_folder(self, path): """ Retrieves a folder in the library :param: str path: the folder path :returns: the folder in the given path :rtype: :class:`dataikuapi.dss.projectlibrary.DSSLibraryFolder` """ return self.root.get_folder(path) def add_file(self, file_name): """ Create a file in the library root folder :param: str file_name: the file name :returns: the new file :rtype: :class:`dataikuapi.dss.projectlibrary.DSSLibraryFile` """ return self.root.add_file(file_name) def add_folder(self, folder_name): """ Create a folder in the library root folder :param: str file_name: the file name :returns: the new folder :rtype: :class:`dataikuapi.dss.projectlibrary.DSSLibraryFolder` """ return self.root.add_folder(folder_name) class DSSLibraryItem(object): """ A handle to manage a library item """ def __init__(self, client, project_key, name, parent, children): """Do not call directly, use :meth:`dataikuapi.dss.projectlibrary.DSSLibrary.get_file()` or :meth:`dataikuapi.dss.projectlibrary.DSSLibrary.get_folder()`""" self.client = client self.project_key = project_key self.name = name self.parent = parent self.children = children @property def path(self): if self.parent is None: return self.name return self._clean_path_(self.parent.path + "/" + self.name) def is_root(self): return self.parent is None and self.name == "/" def _clean_path_(self, path): return path.replace("//", "/") def delete(self): """ Deletes this item from library """ if self.is_root(): raise Exception("Cannot delete root folder") self.client._perform_empty("DELETE", "/projects/%s/libraries/contents/%s" % (self.project_key, dku_quote_fn(self.path))) self.parent._remove_child_(self) self.parent = None def rename(self, new_name): """ Rename the folder :param: str new_name: the new name of the item """ if self.is_root(): raise Exception("Cannot rename root folder") self.client._perform_empty("POST", "/projects/%s/libraries/contents-actions/rename/" % self.project_key, body={"oldPath": self.path, "newName": new_name}) self.name = new_name def move_to(self, destination_folder): """ Move a library item to another folder :param: :class:`dataikuapi.dss.projectlibrary.DSSLibraryFolder` destination_folder: the folder where we want to move the current item """ if self.is_root(): raise DataikuException("Cannot move root folder") if destination_folder is None: raise DataikuException("Destination folder cannot be None") if not isinstance(destination_folder, DSSLibraryFolder): raise DataikuException("Destination should be a folder") self.client._perform_empty("POST", "/projects/%s/libraries/contents-actions/move" % self.project_key, body={"oldPath": self.path, "newPath": destination_folder.path}) self.parent._remove_child_(self) destination_folder._add_child_(self) self.parent = destination_folder class DSSLibraryFolder(DSSLibraryItem): """ A handle to manage a library folder """ def __init__(self, client, project_key, name, parent, children): """Do not call directly, use :class:`dataikuapi.dss.projectlibrary.DSSLibrary.get_folder`""" super(DSSLibraryFolder, self).__init__(client, project_key, name, parent, children) def get_child(self, name): """ Retrieve the sub item by its name :param: str name: the name of the sub item :returns: the sub item :rtype: :class:`dataikuapi.dss.projectlibrary.DSSLibraryItem` """ for c in self.children: if c.name == name: return c return None def add_file(self, file_name): """ Create a new file in the library folder :param: str file_name: the file name :returns: the new file :rtype: :class:`dataikuapi.dss.projectlibrary.DSSLibraryFile` """ file_name = file_name.strip() if self.is_root(): new_path = dku_quote_fn(file_name) else: new_path = dku_quote_fn(self.path + "/" + file_name) try: existing_file = self.client._perform_json("GET", "/projects/%s/libraries/contents/%s" % (self.project_key, new_path)) except: existing_file = None finally: if existing_file is not None: raise DataikuException("File %s already exists" % file_name) self.client._perform_empty("POST", "/projects/%s/libraries/contents/%s" % (self.project_key, new_path)) return self._create_library_item_(self, file_name, True) def add_folder(self, folder_name): """ Create a folder in the library :param: str folder_name: the name of the folder to add :returns: the new folder :rtype: :class:`dataikuapi.dss.projectlibrary.DSSLibraryFolder` """ folder_name = folder_name.strip() if self.is_root(): new_path = dku_quote_fn(folder_name) else: new_path = dku_quote_fn(self.path + "/" + folder_name) self.client._perform_empty("POST", "/projects/%s/libraries/folders/%s" % (self.project_key, new_path)) return self._create_library_item_(self, folder_name, False) def _create_library_item_(self, item, path, create_file): if path.startswith("/"): path = path[1:] path_split = path.split('/', 1) if path_split[0] == ".": return self._create_library_item_(item, path_split[1], create_file) if path_split[0] == "..": return self._create_library_item_(item.parent, path_split[1], create_file) if len(path_split) > 1 and path_split[1]: # Check if the path is a child's sub path for c in item.children: if c.name == path_split[0]: return self._create_library_item_(c, path_split[1], create_file) # Path element not present in folder. We should create it folder = DSSLibraryFolder(self.client, self.project_key, path_split[0], item, set()) item.children.add(folder) return self._create_library_item_(folder, path_split[1], create_file) else: # It is a direct child if create_file: new_item = DSSLibraryFile(self.client, self.project_key, path_split[0], item) else: new_item = DSSLibraryFolder(self.client, self.project_key, path_split[0], item, set()) item.children.add(new_item) return new_item def list(self): """ Gets the contents of this folder sorted by name :returns: a sorted list of items :rtype: list of :class:`dataikuapi.dss.projectlibrary.DSSLibraryItem` """ return sorted(list(self.children), key=lambda x: x.name) def _add_child_(self, child): """ Adds the given child from this folder """ self.children.add(child) def _remove_child_(self, child): """ Removes the given child from this folder """ self.children.remove(child) def get_file(self, path): """ Retrieves a file in the library :param: str path: the file path :returns: the file in the given path :rtype: :class:`dataikuapi.dss.projectlibrary.DSSLibraryFile` """ item = self._get_item_recursively_(self, path) if item is not None: if item.children is None: return item else: raise DataikuException("The item %s is a folder, not a file " % path) return None def get_folder(self, path): """ Retrieves a folder in the library :param: str path: the folder path :returns: the folder in the given path :rtype: :class:`dataikuapi.dss.projectlibrary.DSSLibraryFolder` """ item = self._get_item_recursively_(self, path) if item is not None: if item.children is not None: return item else: raise DataikuException("The item %s is a file, not a folder " % path) return None def _get_item_recursively_(self, item, path): if not path or item is None: return None if path == "/": return item if path.startswith("/"): path = path[1:] path_split = path.split('/', 1) if path_split[0] == ".": if len(path_split) > 1: return self._get_item_recursively_(item, path_split[1]) else: return item if path_split[0] == "..": if item.parent is None: raise DataikuException("Path %s is not valid" % path) if len(path_split) > 1: return self._get_item_recursively_(item.parent, path_split[1]) else: return item.parent if item.children is None: return None for c in item.children: if c.name == path_split[0]: if len(path_split) > 1: return self._get_item_recursively_(c, path_split[1]) else: return c return None class DSSLibraryFile(DSSLibraryItem): """ A handle to manage a library file """ def __init__(self, client, project_key, name, parent): """Do not call directly, use :meth:`dataikuapi.dss.projectlibrary.DSSLibrary.get_file`""" super(DSSLibraryFile, self).__init__(client, project_key, name, parent, None) def read(self): """ Get the file contents from DSS :returns: the contents of the file """ return self.client._perform_json("GET", "/projects/%s/libraries/contents/%s" % (self.project_key, dku_quote_fn(self.path)))["data"] def write(self, data): """ Updates the the contents of the file with the given data """ self.client._perform_empty("POST", "/projects/%s/libraries/contents/%s" % (self.project_key, dku_quote_fn(self.path)), raw_body=data) dataiku-api-client-python-11.4.0/dataikuapi/dss/recipe.py000066400000000000000000002747201440464736400233370ustar00rootroot00000000000000from ..utils import DataikuException from .utils import DSSTaggableObjectSettings from .discussion import DSSObjectDiscussions import json, logging, warnings from .utils import DSSTaggableObjectListItem, DSSTaggableObjectSettings try: basestring except NameError: basestring = str class DSSRecipeListItem(DSSTaggableObjectListItem): """ An item in a list of recipes. .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.list_recipes()` """ def __init__(self, client, data): super(DSSRecipeListItem, self).__init__(data) self.client = client def to_recipe(self): """ Gets a handle corresponding to this recipe. :rtype: :class:`DSSRecipe` """ return DSSRecipe(self.client, self._data["projectKey"], self._data["name"]) @property def name(self): """ Get the name of the recipe. :rtype: string """ return self._data["name"] @property def id(self): """ Get the identifier of the recipe. For recipes, the name is the identifier. :rtype: string """ return self._data["name"] @property def type(self): """ Get the type of the recipe. :return: a recipe type, for example 'sync' or 'join' :rtype: string """ return self._data["type"] class DSSRecipe(object): """ A handle to an existing recipe on the DSS instance. .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.get_recipe()` """ def __init__(self, client, project_key, recipe_name): self.client = client self.project_key = project_key self.recipe_name = recipe_name @property def id(self): """ Get the identifier of the recipe. For recipes, the name is the identifier. :rtype: string """ return self.recipe_name @property def name(self): """ Get the name of the recipe. :rtype: string """ return self.recipe_name def compute_schema_updates(self): """ Computes which updates are required to the outputs of this recipe. This method only computes which changes would be needed to make the schema of the outputs of the reicpe match the actual schema that the recipe will produce. To effectively apply these changes to the outputs, you can use the :meth:`~RequiredSchemaUpdates.apply()` on the returned object. .. note:: Not all recipe types can compute automatically the schema of their outputs. Code recipes like Python recipes, notably can't. This method raises an exception in these cases. Usage example: .. code-block:: python required_updates = recipe.compute_schema_updates() if required_updates.any_action_required(): print("Some schemas will be updated") # Note that you can call apply even if no changes are required. This will be noop required_updates.apply() :return: an object containing the required updates :rtype: :class:`RequiredSchemaUpdates` """ data = self.client._perform_json( "GET", "/projects/%s/recipes/%s/schema-update" % (self.project_key, self.recipe_name)) return RequiredSchemaUpdates(self, data) def run(self, job_type="NON_RECURSIVE_FORCED_BUILD", partitions=None, wait=True, no_fail=False): """ Starts a new job to run this recipe and wait for it to complete. Raises if the job failed. .. code-block:: python job = recipe.run() print("Job %s done" % job.id) :param string job_type: job type. One of RECURSIVE_BUILD, NON_RECURSIVE_FORCED_BUILD or RECURSIVE_FORCED_BUILD :param string partitions: if the outputs are partitioned, a partition spec. A spec is a comma-separated list of partition identifiers, and a partition identifier is a pipe-separated list of values for the partitioning dimensions :param boolean no_fail: if True, does not raise if the job failed :param boolean wait: if True, the method waits for the job complettion. If False, the method returns immediately :return: a job handle corresponding to the recipe run :rtype: :class:`dataikuapi.dss.job.DSSJob` """ project = self.client.get_project(self.project_key) outputs = project.get_flow().get_graph().get_successor_computables(self) if len(outputs) == 0: raise Exception("recipe has no outputs, can't run it") first_output = outputs[0] object_type_map = { "COMPUTABLE_DATASET": "DATASET", "COMPUTABLE_FOLDER": "MANAGED_FOLDER", "COMPUTABLE_SAVED_MODEL": "SAVED_MODEL", "COMPUTABLE_STREAMING_ENDPOINT": "STREAMING_ENDPOINT", "COMPUTABLE_MODEL_EVALUATION_STORE": "MODEL_EVALUATION_STORE" } if first_output["type"] in object_type_map: jd = project.new_job(job_type) jd.with_output(first_output["ref"], object_type=object_type_map[first_output["type"]], partition=partitions) else: raise Exception("Recipe has unsupported output type {}, can't run it".format(first_output["type"])) if wait: return jd.start_and_wait(no_fail) else: return jd.start() def delete(self): """ Delete the recipe. """ return self.client._perform_empty( "DELETE", "/projects/%s/recipes/%s" % (self.project_key, self.recipe_name)) def get_settings(self): """ Get the settings of the recipe, as a :class:`DSSRecipeSettings` or one of its subclasses. Some recipes have a dedicated class for the settings, with additional helpers to read and modify the settings Once you are done modifying the returned settings object, you can call :meth:`~DSSRecipeSettings.save` on it in order to save the modifications to the DSS recipe. :rtype: :class:`DSSRecipeSettings` or a subclass """ data = self.client._perform_json( "GET", "/projects/%s/recipes/%s" % (self.project_key, self.recipe_name)) type = data["recipe"]["type"] if type == "grouping": return GroupingRecipeSettings(self, data) elif type == "window": return WindowRecipeSettings(self, data) elif type == "sync": return SyncRecipeSettings(self, data) elif type == "pivot": return PivotRecipeSettings(self, data) elif type == "sort": return SortRecipeSettings(self, data) elif type == "topn": return TopNRecipeSettings(self, data) elif type == "distinct": return DistinctRecipeSettings(self, data) elif type == "join": return JoinRecipeSettings(self, data) elif type == "vstack": return StackRecipeSettings(self, data) elif type == "sampling": return SamplingRecipeSettings(self, data) elif type == "split": return SplitRecipeSettings(self, data) elif type == "prepare" or type == "shaker": return PrepareRecipeSettings(self, data) #elif type == "prediction_scoring": #elif type == "clustering_scoring": elif type == "download": return DownloadRecipeSettings(self, data) #elif type == "sql_query": # return WindowRecipeSettings(self, data) elif type in ["python", "r", "sql_script", "pyspark", "sparkr", "spark_scala", "shell"]: return CodeRecipeSettings(self, data) else: return DSSRecipeSettings(self, data) def get_definition_and_payload(self): """ Get the definition of the recipe. .. attention:: Deprecated. Use :meth:`get_settings` :return: an object holding both the raw definition of the recipe (the type, which inputs and outputs, engine settings...) and the payload (SQL script, Python code, join definition,... depending on type) :rtype: :class:`DSSRecipeDefinitionAndPayload` """ warnings.warn("Recipe.get_definition_and_payload is deprecated, please use get_settings", DeprecationWarning) data = self.client._perform_json( "GET", "/projects/%s/recipes/%s" % (self.project_key, self.recipe_name)) return DSSRecipeDefinitionAndPayload(self, data) def set_definition_and_payload(self, definition): """ Set the definition of the recipe. .. attention:: Deprecated. Use :meth:`get_settings` then :meth:`DSSRecipeSettings.save()` .. important:: The **definition** parameter should come from a call to :meth:`get_definition()` :param object definition: a recipe definition, as returned by :meth:`get_definition()` """ warnings.warn("Recipe.set_definition_and_payload is deprecated, please use get_settings", DeprecationWarning) definition._payload_to_str() return self.client._perform_json( "PUT", "/projects/%s/recipes/%s" % (self.project_key, self.recipe_name), body=definition.data) def get_status(self): """ Gets the status of this recipe. The status of a recipe is made of messages from checks performed by DSS on the recipe, of messages related to engines availability for the recipe, of messages about testing the recipe on the engine, ... :return: an object to interact with the status :rtype: :class:`dataikuapi.dss.recipe.DSSRecipeStatus` """ data = self.client._perform_json( "GET", "/projects/%s/recipes/%s/status" % (self.project_key, self.recipe_name)) return DSSRecipeStatus(self.client, data) def get_metadata(self): """ Get the metadata attached to this recipe. The metadata contains label, description checklists, tags and custom metadata of the recipe :return: the metadata as a dict, with fields: * **label** : label of the object (not defined for recipes) * **description** : description of the object (not defined for recipes) * **checklists** : checklists of the object, as a dict with a **checklists** field, which is a list of checklists, each a dict of fields: * **id** : identifier of the checklist * **title** : label of the checklist * **createdBy** : user who created the checklist * **createdOn** : timestamp of creation, in milliseconds * **items** : list of the items in the checklist, each a dict of * **done** : True if the item has been done * **text** : label of the item * **createdBy** : who created the item * **createdOn** : when the item was created, as a timestamp in milliseconds * **stateChangedBy** : who ticked the item as done (or not done) * **stateChangedOn** : when the item was last changed to done (or not done), as a timestamp in milliseconds * **tags** : list of tags, each a string * **custom** : custom metadata, as a dict with a **kv** field, which is a dict with any contents the user wishes * **customFields** : dict of custom field info (not defined for recipes) :rtype: dict """ return self.client._perform_json( "GET", "/projects/%s/recipes/%s/metadata" % (self.project_key, self.recipe_name)) def set_metadata(self, metadata): """ Set the metadata on this recipe. .. important:: You should only set a **metadata** object that has been retrieved using :meth:`get_metadata()`. :params dict metadata: the new state of the metadata for the recipe. """ return self.client._perform_json( "PUT", "/projects/%s/recipes/%s/metadata" % (self.project_key, self.recipe_name), body=metadata) def get_object_discussions(self): """ Get a handle to manage discussions on the recipe. :return: the handle to manage discussions :rtype: :class:`dataikuapi.dss.discussion.DSSObjectDiscussions` """ return DSSObjectDiscussions(self.client, self.project_key, "RECIPE", self.recipe_name) def get_continuous_activity(self): """ Get a handle on the associated continuous activity. .. note:: Should only be used on continuous recipes. :rtype: :class:`dataikuapi.dss.continuousactivity.DSSContinuousActivity` """ from .continuousactivity import DSSContinuousActivity return DSSContinuousActivity(self.client, self.project_key, self.recipe_name) def move_to_zone(self, zone): """ Move this object to a flow zone. :param object zone: a :class:`dataikuapi.dss.flow.DSSFlowZone` where to move the object """ if isinstance(zone, basestring): zone = self.client.get_project(self.project_key).get_flow().get_zone(zone) zone.add_item(self) class DSSRecipeStatus(object): """ Status of a recipe. .. important:: Do not instantiate directly, use :meth:`DSSRecipe.get_status` """ def __init__(self, client, data): self.client = client self.data = data def get_selected_engine_details(self): """ Get the selected engine for this recipe. This method will raise if there is no selected engine, whether it's because the present recipe type has no notion of engine, or because DSS couldn't find any viable engine for running the recipe. :return: a dict of the details of the selected recipe, with fields: * **type** : engine type * **typeLabel** : user-friendly label for the type * **variant** : engine sub-type. For example, **type** can be SPARK, and **variant** one of SPARK_SQL, SPARK_SCALA or SPARK_NATIVE * **variantLabel** : user-friendly label for the variant * **label** : user-friendly label for the engine (type and variant) * **description** : longer version of **label** * **isSelectable** : whether the engine can be selected (would be always True for the engine returned by this method) * **recommended** : whether this is the engine DSS recommends, given the inputs and outputs of the recipe, and the DSS instance setup * **statusWarnLevel** : status of the check on the engine. Possible values: OK, WARN, ERROR * **statusMessage** : optional message about the status * **statusAdditionalMessage** : optional details about **statusMessage** (only for Prepare recipes) * **canEngineAppend** : whether the engine handles appending to outputs, instead of just overwriting outputs Many engines have additional fields with more detailed information on their known abilities: * **queryBased** : whether the engine will translate the recipe to SQL form * **canAnalyticalFunctions** : whether OLAP functions are accessible (for group, window, join, ... ) * **canStddevAsAnalyticalFunctions** : whether there is an OLAP function to compute the standard deviation * **canDistinctSelect** : whether SELECT DISTINCT... is possible (for visual recipes) * **canNonEquiJoin** : whether join conditions can be other than mere equalities (for join recipes) * **canFullOuterJoin** : whether full outer join is possible (for join recipes) * **canDeduplicateJoinMatches** : whether it's possible to have unique join matches (for join recipes) * **doNotSupportLeadLagWithWindow** : whether lead and lag exist as OLAP functions * **aggregabilities** : capabilities for the aggregates in this SQL dialect, as a dict of aggregate name to a dict of capabilities * **identifierQuotingCharacter** : quoting character for identifier in SQL code * **stringQuotingCharacter** : quoting character for literals in SQL code * **lowercasesColumnNames** : whether the engine will automatically lowercase column names :rtype: dict """ if not "selectedEngine" in self.data: raise ValueError("This recipe doesn't have a selected engine") return self.data["selectedEngine"] def get_engines_details(self): """ Get details about all possible engines for this recipe. This method will raise if there is no engine, whether it's because the present recipe type has no notion of engine, or because DSS couldn't find any viable engine for running the recipe. :return: a list of dict of the details of each possible engine. See :meth:`get_selected_engine_details()` for the fields of each dict. :rtype: list[dict] """ if not "engines" in self.data: raise ValueError("This recipe doesn't have engines") return self.data["engines"] def get_status_severity(self): """ Get the overall status of the recipe. This is the final result of checking the different parts of the recipe, and depends on the recipe type. Examples of checks done include: - checking the validity of the formulas in computed columns or filters - checking if some of the input columns retrieved by joins overlap - checking against the SQL database if the generated SQL is valid :return: SUCCESS, WARNING, ERROR or INFO. None if the status has no message at all. :rtype: string """ return self.data["allMessagesForFrontend"].get("maxSeverity") def get_status_messages(self, as_objects=False): """ Returns status messages for this recipe. :param boolean as_objects: if True, return a list of :class:`dataikuapi.dss.utils.DSSInfoMessage`. If False, as a list of raw dicts. :return: if **as_objects** is True, a list of :class:`dataikuapi.dss.utils.DSSInfoMessage`, otherwise a list of message information, each one a dict of: * **severity** : severity of the error in the message. Possible values are SUCCESS, INFO, WARNING, ERROR * **isFatal** : for ERROR **severity**, whether the error is considered fatal to the operation * **code** : a string with a well-known code documented in `DSS doc `_ * **title** : short message * **message** : the error message * **details** : a more detailed error description :rtype: list """ if as_objects: return [DSSInfoMessage(message) for message in self.data["allMessagesForFrontend"].get("messages", [])] else: return self.data["allMessagesForFrontend"]["messages"] class DSSRecipeSettings(DSSTaggableObjectSettings): """ Settings of a recipe. .. important:: Do not instantiate directly, use :meth:`DSSRecipe.get_settings` """ def __init__(self, recipe, data): super(DSSRecipeSettings, self).__init__(data["recipe"]) self.recipe = recipe self.data = data self.recipe_settings = self.data["recipe"] self._str_payload = self.data.get("payload", None) self._obj_payload = None def save(self): """ Save back the recipe in DSS. """ self._payload_to_str() return self.recipe.client._perform_json( "PUT", "/projects/%s/recipes/%s" % (self.recipe.project_key, self.recipe.recipe_name), body=self.data) @property def type(self): """ Get the type of the recipe. :return: a type, like 'sync', 'python' or 'join' :rtype: string """ return self.recipe_settings["type"] @property def str_payload(self): """ The raw "payload" of the recipe. This is exactly the data persisted on disk. :return: for code recipes, the payload will be the script of the recipe. For visual recipes, the payload is a JSON of settings that are specific to the recipe type, like the definitions of the aggregations for a grouping recipe. :rtype: string """ self._payload_to_str() return self._str_payload @str_payload.setter def str_payload(self, payload): self._str_payload = payload self._obj_payload = None @property def obj_payload(self): """ The "payload" of the recipe, parsed from JSON. .. note:: Do not use on code recipes, their payload isn't JSON-encoded. :return: settings that are specific to the recipe type, like the definitions of the aggregations for a grouping recipe. :rtype: dict """ self._payload_to_obj() return self._obj_payload @property def raw_params(self): """ The non-payload settings of the recipe. :return: recipe type-specific settings that aren't stored in the payload. Typically this comprises engine settings. :rtype: dict """ return self.recipe_settings["params"] def _payload_to_str(self): if self._obj_payload is not None: self._str_payload = json.dumps(self._obj_payload) self._obj_payload = None if self._str_payload is not None: self.data["payload"] = self._str_payload def _payload_to_obj(self): if self._str_payload is not None: self._obj_payload = json.loads(self._str_payload) self._str_payload = None def get_recipe_raw_definition(self): """ Get the recipe definition. :return: the part of the recipe's settings that aren't stored in the payload, as a dict. Notable fields are: * **name** and **projectKey** : identifiers of the recipe * **type** : type of the recipe * **shortDesc** : short description of the recipe * **description** : longer description (markdown-enabled) * **tags** : list of tags, each one a string * **checklists** : list of checklists on the recipe * **doc** : free text attached to the recipe (not shown in UI) * **params** : type-specific parameters of the recipe (on top of what is in the payload) * **customMeta** : dict with a **kv** field, itself a dict * **redispatchPartitioning** : for 'sync' and 'shaker' recipe, whether the recipe re-dispatches partitions of the input to the (partitioned) output * **maxRunningActivities** : maximum number of partitions this recipe can run in parallel in a given job * **variables** : dict of recipe-specific variables * **dkuProperties** : list of properties, each a dict with **name** and **value** string fields * **labels** : (for model training recipes) list of labels to propagate to the model evaluation stores. Each label is a dict with fields **key** and **value** * **inputs** : input roles to the recipe, as a dict of role name to role, where a role is a dict with an **items** field consisting of a list of one dict per input object. Each individual input has fields: * **ref** : a dataset name or a managed folder id or a saved model id. Should be prefixed by the project key for exposed items, like in "PROJECT_KEY.dataset_name" * **deps** : for partitioned inputs, a list of partition dependencies mapping output dimensions to dimensions in this input. Each partition dependency is a dict of: * **out** : reference to the output used to compute the values for this partition dimension * **idim** : name of the partition dimension in the input * **odim** : name of the partition dimension in the output **out** * **func** : function to use to deduce input partition values from a given output partition value. Possible values are: equals, all_available, time_range, latest_available, values, custom_python * **params** : additional parameters for the dependency, for example the Python code when **func** is 'custom_python' * **values** : list of values when **func** is 'values'. Each value is a string * **expandVariables** : when **func** is 'values', whether the strings in **values** should go through variable expansion, variables being added as '${variable_name}'. * **outputs** : output roles to the recipe, as a dict of role name to role, where a role is a dict with a **items** field consisting of a list of one dict per output object. Each individual output has fields: * **ref** : a dataset name or a managed folder id or a saved model id. Should be prefixed by the project key for exposed items, like in "PROJECT_KEY.dataset_name" * **appendMode** : if True, the recipe should append into the output; if False, the recipe should overwrite the output when running :rtype: dict """ return self.recipe_settings def get_recipe_inputs(self): """ Get the inputs to this recipe. :rtype: dict """ return self.recipe_settings.get('inputs') def get_recipe_outputs(self): """ Get the outputs of this recipe. :rtype: dict """ return self.recipe_settings.get('outputs') def get_recipe_params(self): """ The non-payload settings of the recipe. :return: recipe type-specific settings that aren't stored in the payload. Typically this comprises engine settings. :rtype: dict """ return self.recipe_settings.get('params') def get_payload(self): """ The raw "payload" of the recipe. This is exactly the data persisted on disk. :return: for code recipes, the payload will be the script of the recipe. For visual recipes, the payload is a JSON of settings that are specific to the recipe type, like the definitions of the aggregations for a grouping recipe. :rtype: string """ self._payload_to_str() return self._str_payload def get_json_payload(self): """ The "payload" of the recipe, parsed from JSON. .. note:: Do not use on code recipes, their payload isn't JSON-encoded. :return: settings that are specific to the recipe type, like the definitions of the aggregations for a grouping recipe. :rtype: dict """ self._payload_to_obj() return self._obj_payload def set_payload(self, payload): """ Set the payload of this recipe. :param string payload: the payload, as a string """ self._str_payload = payload self._obj_payload = None def set_json_payload(self, payload): """ Set the payload of this recipe. :param dict payload: the payload, as a dict. Will be converted to JSON internally. """ self._str_payload = None self._obj_payload = payload def has_input(self, input_ref): """ Whether a ref is part of the recipe's inputs. :param string input_ref: a ref to an object in DSS, i.e. a dataset name or a managed folder id or a saved model id. Should be prefixed by the project key for exposed items, like in "PROJECT_KEY.dataset_name" :rtype: boolean """ inputs = self.get_recipe_inputs() for (input_role_name, input_role) in inputs.items(): for item in input_role.get("items", []): if item.get("ref", None) == input_ref: return True return False def has_output(self, output_ref): """ Whether a ref is part of the recipe's outputs. :param string output_ref: a ref to an object in DSS, i.e. a dataset name or a managed folder id or a saved model id. Should be prefixed by the project key for exposed items, like in "PROJECT_KEY.dataset_name" :rtype: boolean """ outputs = self.get_recipe_outputs() for (output_role_name, output_role) in outputs.items(): for item in output_role.get("items", []): if item.get("ref", None) == output_ref: return True return False def replace_input(self, current_input_ref, new_input_ref): """ Replaces an input of this recipe by another. If the **current_input_ref** isn't part of the recipe's inputs, this method has no effect. :param string current_input_ref: a ref to an object in DSS, i.e. a dataset name or a managed folder id or a saved model id, that is currently input to the recipe :param string new_input_ref: a ref to an object in DSS, i.e. a dataset name or a managed folder id or a saved model id, that **current_input_ref** should be replaced with. """ inputs = self.get_recipe_inputs() for (input_role_name, input_role) in inputs.items(): for item in input_role.get("items", []): if item.get("ref", None) == current_input_ref: item["ref"] = new_input_ref def replace_output(self, current_output_ref, new_output_ref): """ Replaces an output of this recipe by another. If the **current_output_ref** isn't part of the recipe's outputs, this method has no effect. :param string current_output_ref: a ref to an object in DSS, i.e. a dataset name or a managed folder id or a saved model id, that is currently output to the recipe :param string new_output_ref: a ref to an object in DSS, i.e. a dataset name or a managed folder id or a saved model id, that **current_output_ref** should be replaced with. """ outputs = self.get_recipe_outputs() for (output_role_name, output_role) in outputs.items(): for item in output_role.get("items", []): if item.get("ref", None) == current_output_ref: item["ref"] = new_output_ref def add_input(self, role, ref, partition_deps=None): """ Add an input to the recipe. For most recipes, there is only one role, named "main". Some few recipes have additional roles, like scoring recipes which have a "model" role. Check the roles known to the recipe with :meth:`get_recipe_inputs()`. :param string role: name of the role of the recipe in which to add **ref** as input :param string ref: a ref to an object in DSS, i.e. a dataset name or a managed folder id or a saved model id :param list partition_deps: if **ref** points to a partitioned object, a list of partition dependencies, one per dimension in the partitioning scheme """ if partition_deps is None: partition_deps = [] self._get_or_create_input_role(role)["items"].append({"ref": ref, "deps": partition_deps}) def add_output(self, role, ref, append_mode=False): """ Add an output to the recipe. For most recipes, there is only one role, named "main". Some few recipes have additional roles, like evaluation recipes which have a "metrics" role. Check the roles known to the recipe with :meth:`get_recipe_outputs()`. :param string role: name of the role of the recipe in which to add **ref** as input :param string ref: a ref to an object in DSS, i.e. a dataset name or a managed folder id or a saved model id :param list partition_deps: if **ref** points to a partitioned object, a list of partition dependencies, one per dimension in the partitioning scheme """ self._get_or_create_output_role(role)["items"].append({"ref": ref, "appendMode": append_mode}) def _get_or_create_input_role(self, role): inputs = self.get_recipe_inputs() if not role in inputs: role_obj = {"items": []} inputs[role] = role_obj return inputs[role] def _get_or_create_output_role(self, role): outputs = self.get_recipe_outputs() if not role in outputs: role_obj = {"items": []} outputs[role] = role_obj return outputs[role] def _get_flat_inputs(self): ret = [] for role_key, role_obj in self.get_recipe_inputs().items(): for item in role_obj["items"]: ret.append((role_key, item)) return ret def _get_flat_outputs(self): ret = [] for role_key, role_obj in self.get_recipe_outputs().items(): for item in role_obj["items"]: ret.append((role_key, item)) return ret def get_flat_input_refs(self): """ List all input refs of this recipe, regardless of the input role. :return: a list of refs, i.e. of dataset names or managed folder ids or saved model ids :rtype: list[string] """ ret = [] for role_key, role_obj in self.get_recipe_inputs().items(): for item in role_obj["items"]: ret.append(item["ref"]) return ret def get_flat_output_refs(self): """ List all output refs of this recipe, regardless of the input role. :return: a list of refs, i.e. of dataset names or managed folder ids or saved model ids :rtype: list[string] """ ret = [] for role_key, role_obj in self.get_recipe_outputs().items(): for item in role_obj["items"]: ret.append(item["ref"]) return ret class DSSRecipeDefinitionAndPayload(DSSRecipeSettings): """ Settings of a recipe. .. note:: Deprecated. Alias to :class:`DSSRecipeSettings`, use :meth:`DSSRecipe.get_settings()` instead. """ pass class RequiredSchemaUpdates(object): """ Handle on a set of required updates to the schema of the outputs of a recipe. .. important:: Do not instantiate directly, use :meth:`DSSRecipe.compute_schema_updates()` For example, changes can be new columns in the output of a Group recipe when new aggregates are activated in the recipe's settings. """ def __init__(self, recipe, data): self.recipe = recipe self.data = data self.drop_and_recreate = True self.synchronize_metastore = True def any_action_required(self): """ Whether there are changes at all. :rtype: boolean """ return self.data["totalIncompatibilities"] > 0 def apply(self): """ Apply the changes. All the updates found to be required are applied, for each of the recipe's outputs. """ results = [] for computable in self.data["computables"]: osu = { "computableType": computable["type"], # dirty "computableId": computable["type"] == "DATASET" and computable["datasetName"] or computable["id"], "newSchema": computable["newSchema"], "dropAndRecreate": self.drop_and_recreate, "synchronizeMetastore" : self.synchronize_metastore } results.append(self.recipe.client._perform_json("POST", "/projects/%s/recipes/%s/actions/updateOutputSchema" % (self.recipe.project_key, self.recipe.recipe_name), body=osu)) return results ##################################################### # Recipes creation infrastructure ##################################################### class DSSRecipeCreator(object): """ Helper to create new recipes. .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.new_recipe()` instead. """ def __init__(self, type, name, project): self.project = project self.recipe_proto = { "inputs" : {}, "outputs" : {}, "type" : type, "name" : name } self.creation_settings = { } def set_name(self, name): """ Set the name of the recipe-to-be-created. :param string name: a recipe name. Should only use alphanum letters and underscores. Cannot contain dots. """ self.recipe_proto["name"] = name def _build_ref(self, object_id, project_key=None): if project_key is not None and project_key != self.project.project_key: return project_key + '.' + object_id else: return object_id def _with_input(self, dataset_name, project_key=None, role="main"): role_obj = self.recipe_proto["inputs"].get(role, None) if role_obj is None: role_obj = { "items" : [] } self.recipe_proto["inputs"][role] = role_obj role_obj["items"].append({'ref':self._build_ref(dataset_name, project_key)}) return self def _with_output(self, dataset_name, append=False, role="main"): role_obj = self.recipe_proto["outputs"].get(role, None) if role_obj is None: role_obj = { "items" : [] } self.recipe_proto["outputs"][role] = role_obj role_obj["items"].append({'ref':self._build_ref(dataset_name, None), 'appendMode': append}) return self def _get_input_refs(self): ret = [] for role_key, role_obj in self.recipe_proto["inputs"].items(): for item in role_obj["items"]: ret.append(item["ref"]) return ret def with_input(self, object_id, project_key=None, role="main"): """ Add an existing object as input to the recipe-to-be-created. :param string dataset_name: name of the dataset, or identifier of the managed folder or identifier of the saved model :param string project_key: project containing the object, if different from the one where the recipe is created :param string role: the role of the recipe in which the input should be added. Most recipes only have one role named "main". """ return self._with_input(object_id, project_key, role) def with_output(self, object_id, append=False, role="main"): """ Add an existing object as output to the recipe-to-be-created. The output dataset must already exist. :param string object_id: name of the dataset, or identifier of the managed folder or identifier of the saved model :param boolean append: whether the recipe should append or overwrite the output when running (note: not available for all dataset types) :param string role: the role of the recipe in which the input should be added. Most recipes only have one role named "main". """ return self._with_output(object_id, append, role) def build(self): """ Create the recipe. .. note:: Deprecated. Alias to :meth:`create()` """ return self.create() def create(self): """ Creates the new recipe in the project, and return a handle to interact with it. :rtype: :class:`dataikuapi.dss.recipe.DSSRecipe` """ self._finish_creation_settings() return self.project.create_recipe(self.recipe_proto, self.creation_settings) def set_raw_mode(self): """ Activate raw creation mode. .. caution:: For advanced uses only. In this mode, the field "recipe_proto" of this recipe creator is used as-is to create the recipe, and if it exists, the value of creation_settings["rawPayload"] is used as the payload of the created recipe. No checks of existence or validity of the inputs or outputs are done, and no output is auto-created. """ self.creation_settings["rawCreation"] = True def _finish_creation_settings(self): pass class SingleOutputRecipeCreator(DSSRecipeCreator): """ Create a recipe that has a single output. .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.new_recipe()` instead. """ def __init__(self, type, name, project): DSSRecipeCreator.__init__(self, type, name, project) self.create_output_dataset = None self.output_dataset_settings = None self.create_output_folder = None self.output_folder_settings = None def with_existing_output(self, object_id, append=False): """ Add an existing object as output to the recipe-to-be-created. The output dataset must already exist. :param string object_id: name of the dataset, or identifier of the managed folder or identifier of the saved model :param boolean append: whether the recipe should append or overwrite the output when running (note: not available for all dataset types) """ assert self.create_output_dataset is None self.create_output_dataset = False self._with_output(object_id, append) return self def with_new_output(self, name, connection_id, type_option_id=None, format_option_id=None, override_sql_schema=None, partitioning_option_id=None, append=False, object_type='DATASET', overwrite=False): """ Create a new dataset or managed folder as output to the recipe-to-be-created. The dataset or managed folder is not created immediately, but when the recipe is created (ie in the create() method). Whether a dataset is created or a managed folder is created, depends on the recipe type. :param string name: name of the dataset or identifier of the managed folder :param string connection_id: name of the connection to create the dataset or managed folder on :param string type_option_id: sub-type of dataset or managed folder, for connections where the type could be ambiguous. Typically applies to SSH connections, where sub-types can be SCP or SFTP :param string format_option_id: name of a format preset relevant for the dataset type. Possible values are: CSV_ESCAPING_NOGZIP_FORHIVE, CSV_UNIX_GZIP, CSV_EXCEL_GZIP, CSV_EXCEL_GZIP_BIGQUERY, CSV_NOQUOTING_NOGZIP_FORPIG, PARQUET_HIVE, AVRO, ORC :param boolean override_sql_schema: schema to force dataset, for SQL dataset. If left empty, will be autodetected :param string partitioning_option_id: to copy the partitioning schema of an existing dataset 'foo', pass a value of 'copy:dataset:foo'. If unset, then the output will be non-partitioned :param boolean append: whether the recipe should append or overwrite the output when running (note: not available for all dataset types) :param string object_type: DATASET or MANAGED_FOLDER :param boolean overwrite: If the dataset being created already exists, overwrite it (and delete data) """ if object_type == 'DATASET': assert self.create_output_dataset is None dataset = self.project.get_dataset(name) if overwrite and dataset.exists(): dataset.delete(drop_data=True) self.create_output_dataset = True self.output_dataset_settings = {'connectionId':connection_id,'typeOptionId':type_option_id,'specificSettings':{'formatOptionId':format_option_id, 'overrideSQLSchema':override_sql_schema},'partitioningOptionId':partitioning_option_id} self._with_output(name, append) elif object_type == 'MANAGED_FOLDER': assert self.create_output_folder is None self.create_output_folder = True self.output_folder_settings = {'connectionId':connection_id,'typeOptionId':type_option_id,'partitioningOptionId':partitioning_option_id} self._with_output(name, append) return self def with_output(self, dataset_name, append=False): """ Add an existing object as output to the recipe-to-be-created. .. note:: Alias of :meth:`with_existing_output()` """ return self.with_existing_output(dataset_name, append) def _finish_creation_settings(self): self.creation_settings['createOutputDataset'] = self.create_output_dataset self.creation_settings['outputDatasetSettings'] = self.output_dataset_settings self.creation_settings['createOutputFolder'] = self.create_output_folder self.creation_settings['outputFolderSettings'] = self.output_folder_settings class VirtualInputsSingleOutputRecipeCreator(SingleOutputRecipeCreator): """ Create a recipe that has a single output and several inputs. """ def __init__(self, type, name, project): SingleOutputRecipeCreator.__init__(self, type, name, project) self.virtual_inputs = [] def with_input(self, dataset_name, project_key=None): """ Add an existing object as input to the recipe-to-be-created. :param string dataset_name: name of the dataset :param string project_key: project containing the object, if different from the one where the recipe is created """ self.virtual_inputs.append(self._build_ref(dataset_name, project_key)) return self def _finish_creation_settings(self): super(VirtualInputsSingleOutputRecipeCreator, self)._finish_creation_settings() self.creation_settings['virtualInputs'] = self.virtual_inputs ##################################################### # Per-recipe-type classes: Visual recipes ##################################################### class GroupingRecipeSettings(DSSRecipeSettings): """ Settings of a grouping recipe. .. important:: Do not instantiate directly, use :meth:`DSSRecipe.get_settings()` """ def clear_grouping_keys(self): """ Clear all grouping keys. """ self.obj_payload["keys"] = [] def add_grouping_key(self, column): """ Adds grouping on a column. :param string column: column to group on """ self.obj_payload["keys"].append({"column":column}) def set_global_count_enabled(self, enabled): """ Activate computing the count of records per group. :param boolean enabled: True if the global count should be activated """ self.obj_payload["globalCount"] = enabled def get_or_create_column_settings(self, column): """ Get a dict representing the aggregations to perform on a column. If the column has no aggregation on it yet, the dict is created and added to the settings. :param string column: name of the column to aggregate on :return: the settings of the aggregations on a particular column, as a dict with fields: * **column** : name of the column to aggregate * **min**, **max**, **count**, **countDistinct**, **sum**, **concat**, **stddev**, **avg**, **first**, **last** and **sum2** : one boolean per aggregate, indicating whether it's computed. **sum2** is the sum of squares. * **concatSeparator** : for the **concat** aggregate, a string to separate values * **concatDistinct** : for the **concat** aggregate, whether values concatenated are deduplicated * **firstLastNotNull** : for **last** and **first**, whether empty values are ignored * **orderColumn** : for **last** and **first**, name of a column on which rows in the group are sorted prior to taking the first or last row :rtype: dict """ found = None for gv in self.obj_payload["values"]: if gv["column"] == column: found = gv break if found is None: found = {"column" : column} self.obj_payload["values"].append(found) return found def set_column_aggregations(self, column, type=None, min=False, max=False, count=False, count_distinct=False, sum=False,concat=False,stddev=False,avg=False): """ Set the basic aggregations on a column. .. note:: Not all aggregations may be possible. For example string-typed columns don't have a mean or standard deviation, and some SQL databases can't compute the exact standard deviation. The method returns a reference to the settings of the column, not a copy. Modifying the dict returned by the method, then calling :meth:`DSSRecipeSettings.save()` will commit the changes. Usage example: .. code-block:: # activate the concat aggregate on a column, and set optional parameters # pertaining to concatenation settings = recipe.get_settings() column_settings = settings.set_column_aggregations("my_column_name", concat=True) column_settings["concatDistinct"] = True column_settings["concatSeparator"] = ', ' settings.save() :param string column: The column name :param string type: The type of the column (as a DSS schema type name) :param boolean min: whether the min aggregate is computed :param boolean max: whether the max aggregate is computed :param boolean count: whether the count aggregate is computed :param boolean count_distinct: whether the count distinct aggregate is computed :param boolean sum: whether the sum aggregate is computed :param boolean concat: whether the concat aggregate is computed :param boolean avg: whether the mean aggregate is computed :param boolean stddev: whether the standard deviation aggregate is computed :return: the settings of the aggregations on a the column, as a dict with fields: * **column** : name of the column to aggregate * **min**, **max**, **count**, **countDistinct**, **sum**, **concat**, **stddev**, **avg**, **first**, **last** : one boolean per aggregate, indicating whether it's computed * **concatSeparator** : for the **concat** aggregate, a string to separate values * **concatDistinct** : for the **concat** aggregate, whether values concatenated are deduplicated * **firstLastNotNull** : for **last** and **first**, whether empty values are ignored * **orderColumn** : for **last** and **first**, name of a column on which rows in the group are sorted prior to taking the first or last row :rtype dict """ cs = self.get_or_create_column_settings(column) if type is not None: cs["type"] = type cs["min"] = min cs["max"] = max cs["count"] = count cs["countDistinct"] = count_distinct cs["sum"] = sum cs["concat"] = concat cs["stddev"] = stddev return cs class GroupingRecipeCreator(SingleOutputRecipeCreator): """ Create a Group recipe. .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.new_recipe()` instead. """ def __init__(self, name, project): SingleOutputRecipeCreator.__init__(self, 'grouping', name, project) self.group_key = None def with_group_key(self, group_key): """ Set a column as the first grouping key. Only a single grouping key may be set at recipe creation time. To add more grouping keys, get the recipe settings and use :meth:`GroupingRecipeSettings.add_grouping_key()`. To have no grouping keys at all, get the recipe settings and use :meth:`GroupingRecipeSettings.clear_grouping_keys()`. :param string group_key: name of a column in the input dataset :return: self :rtype: :class:`GroupingRecipeCreator` """ self.group_key = group_key return self def _finish_creation_settings(self): super(GroupingRecipeCreator, self)._finish_creation_settings() if self.group_key is not None: self.creation_settings['groupKey'] = self.group_key class WindowRecipeSettings(DSSRecipeSettings): """ Settings of a Window recipe. .. important:: Do not instantiate directly, use :meth:`DSSRecipe.get_settings()` """ pass # TODO: Write helpers for window class WindowRecipeCreator(SingleOutputRecipeCreator): """ Create a Window recipe .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.new_recipe()` instead. """ def __init__(self, name, project): SingleOutputRecipeCreator.__init__(self, 'window', name, project) class SyncRecipeSettings(DSSRecipeSettings): """ Settings of a Sync recipe. .. important:: Do not instantiate directly, use :meth:`DSSRecipe.get_settings()` """ pass # TODO: Write helpers for sync class SyncRecipeCreator(SingleOutputRecipeCreator): """ Create a Sync recipe .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.new_recipe()` instead. """ def __init__(self, name, project): SingleOutputRecipeCreator.__init__(self, 'sync', name, project) class ContinuousSyncRecipeCreator(SingleOutputRecipeCreator): """ Create a continuous Sync recipe .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.new_recipe()` instead. """ def __init__(self, name, project): SingleOutputRecipeCreator.__init__(self, 'csync', name, project) class SortRecipeSettings(DSSRecipeSettings): """ Settings of a Sort recipe. .. important:: Do not instantiate directly, use :meth:`DSSRecipe.get_settings()` """ pass # TODO: Write helpers for sort class SortRecipeCreator(SingleOutputRecipeCreator): """ Create a Sort recipe .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.new_recipe()` instead. """ def __init__(self, name, project): SingleOutputRecipeCreator.__init__(self, 'sort', name, project) class PivotRecipeSettings(DSSRecipeSettings): """ Settings of a Pivot recipe. .. important:: Do not instantiate directly, use :meth:`DSSRecipe.get_settings()` """ pass # TODO: Write helpers for sort class PivotRecipeCreator(SingleOutputRecipeCreator): """ Create a Pivot recipe .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.new_recipe()` instead. """ def __init__(self, name, project): SingleOutputRecipeCreator.__init__(self, 'pivot', name, project) class TopNRecipeSettings(DSSRecipeSettings): """ Settings of a TopN recipe. .. important:: Do not instantiate directly, use :meth:`DSSRecipe.get_settings()` """ pass # TODO: Write helpers for topn class TopNRecipeCreator(DSSRecipeCreator): """ Create a TopN recipe .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.new_recipe()` instead. """ def __init__(self, name, project): DSSRecipeCreator.__init__(self, 'topn', name, project) class DistinctRecipeSettings(DSSRecipeSettings): """ Settings of a Distinct recipe. .. important:: Do not instantiate directly, use :meth:`DSSRecipe.get_settings()` """ pass # TODO: Write helpers for distinct class DistinctRecipeCreator(SingleOutputRecipeCreator): """ Create a Distinct recipe .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.new_recipe()` instead. """ def __init__(self, name, project): SingleOutputRecipeCreator.__init__(self, 'distinct', name, project) class PrepareRecipeSettings(DSSRecipeSettings): """ Settings of a Prepare recipe. .. important:: Do not instantiate directly, use :meth:`DSSRecipe.get_settings()` """ pass @property def raw_steps(self): """ Get the list of the steps of this prepare recipe. This method returns a reference to the list of steps, not a copy. Modifying the list then calling :meth:`DSSRecipeSettings.save()` commits the changes. :return: list of steps, each step as a dict. The precise settings for each step are not documented, but each dict has at least fields: * **metaType** : one of PROCESSOR or GROUP. If GROUP, there there is a field **steps** with a sub-list of steps. * **type** : type of the step, for example FillEmptyWithValue or ColumnRenamer (there are many types of steps) * **params** : dict of the step's own parameters. Each step type has its own parameters. * **disabled** : whether the step is disabled * **name** : label of the step * **comment** : comment on the step * **alwaysShowComment**, **mainColor** and **secondaryColor** : UI settings :rtype: list[dict] """ return self.obj_payload["steps"] def add_processor_step(self, type, params): """ Add a step in the script. :param string type: type of the step, for example FillEmptyWithValue or ColumnRenamer (there are many types of steps) :param dict params: dict of the step's own parameters. Each step type has its own parameters. """ step = { "metaType": "PROCESSOR", "type": type, "params": params } self.raw_steps.append(step) class PrepareRecipeCreator(SingleOutputRecipeCreator): """ Create a Prepare recipe .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.new_recipe()` instead. """ def __init__(self, name, project): SingleOutputRecipeCreator.__init__(self, 'shaker', name, project) class JoinRecipeSettings(DSSRecipeSettings): """ Settings of a join recipe. .. important:: Do not instantiate directly, use :meth:`DSSRecipe.get_settings()` In order to enable self-joins, join recipes are based on a concept of "virtual inputs". Every join, computed pre-join column, pre-join filter, ... is based on one virtual input, and each virtual input references an input of the recipe, by index For example, if a recipe has inputs A and B and declares two joins: - A->B - A->A (based on a computed column) There are 3 virtual inputs: * 0: points to recipe input 0 (i.e. dataset A) * 1: points to recipe input 1 (i.e. dataset B) * 2: points to recipe input 0 (i.e. dataset A) and includes the computed column * The first join is between virtual inputs 0 and 1 * The second join is between virtual inputs 0 and 2 """ pass # TODO: Write helpers for join @property def raw_virtual_inputs(self): """ Get the list of virtual inputs. This method returns a reference to the list of inputs, not a copy. Modifying the list then calling :meth:`DSSRecipeSettings.save()` commits the changes. :return: a list of virtual inputs, each one a dict of * **index** : index of the dataset of this virtual input in the recipe's list of inputs * **preFilter** : filter applied to the input, as a dict of: * **distinct** : whether the records in the input should be deduplicated * **enabled** : whether filtering is enabled * **uiData** : settings of the filter, if **enabled** is True, as a dict of: * **mode** : type of filter. Possible values: CUSTOM, SQL, '&&' (boolean AND of conditions) and '||' (boolean OR of conditions) * **conditions** : if mode is '&&' or '||', then a list of the actual filter conditions, each one a dict * **expression** : if **uiData.mode** is CUSTOM, a formula in DSS `formula language `_ . If **uiData.mode** is SQL, a SQL expression. * **computedColumns** : list of computed columns added to the input, each one a dict of: * **mode** : type of expression used to define the computations. One of GREL or SQL. * **name** : name of the column generated * **type** : name of a DSS type for the computed column * **expr** : if **mode** is CUSTOM, a formula in DSS `formula language `_ . If **mode** is SQL, a SQL expression. * **alias** : optional alias to use instead of the name of the dataset pointed by this virtual input * **autoSelectColumns** : whether to select all columns of the input * **prefix** : optional prefix to add to all columns of this input in the output schema :rtype: list[dict] """ return self.obj_payload["virtualInputs"] @property def raw_joins(self): """ Get raw list of joins. This method returns a reference to the list of joins, not a copy. Modifying the list then calling :meth:`DSSRecipeSettings.save()` commits the changes. :return: list of the join definitions, each as a dict of: * **table1** : index of the virtual input used as left-side in this join * **table2** : index of the virtual input used as right-side in this join * **type** : type of join. Possible values: INNER, LEFT, RIGHT, FULL (full outer), CROSS, ADVANCED * **outerJoinOnTheLeft** : used in ADVANCED mode only, whether to keep unmatched rows in left dataset or not * **rightLimit** : optional limiting of the number of lines that can match each line of the left-side (after taking into account all conditions), as a dict * **conditionsMode** : how the join conditions are assembled. Possible values: AND (match all conditions), OR (match at least one condition), NATURAL (for natural joins), CUSTOM * **customSQLCondition** : when **conditionsMode** is CUSTOM, the SQL expression of the join conditions * **on** : for **conditionsMode** valued AND or OR, a list of join conditions, each one a dict of: * **column1** : name of the column on the left-side * **column2** : name of the column on the right-side * **type** : type of the condition. Possible values are EQ, LTE, LT, GTE, GT, NE, WITHIN_RANGE, K_NEAREST, K_NEAREST_INFERIOR, CONTAINS, STARTS_WITH * **caseInsensitive** : for conditions on strings, whether the equality is case-sensitive * **normalizeText** : for conditions on strings, whether to lowercase and strip accents on both sides * **maxDistance** : for K_NEAREST, K_NEAREST_INFERIOR condition, the maximum allowed distance for the nearest element * **maxMatches** : for K_NEAREST, K_NEAREST_INFERIOR, the maximum number of matches (use 0 for unlimited) * **strict** : whether the limit of **maxMatches** is strict. If not, matches at the same distance are taken, even if this means going over **maxMatches** * **dateDiffUnit** : for comparison on date columns, rounding applied before comparing. Possible values are YEAR, MONTH, WEEK, DAY, HOUR, MINUTE, SECOND. :rtype: list[dict] """ return self.obj_payload["joins"] def add_virtual_input(self, input_dataset_index): """ Add a virtual input pointing to the specified input dataset of the recipe. :param int input_dataset_index: index of the dataset in the list of input_dataset_index """ self.raw_virtual_inputs.append({"index": input_dataset_index}) def add_pre_join_computed_column(self, virtual_input_index, computed_column): """ Add a computed column to a virtual input. You can use :meth:`dataikuapi.dss.utils.DSSComputedColumn.formula()` to build the computed_column object. :param int input_dataset_index: index of the dataset in the list of input_dataset_index :param dict computed_column: a computed column definition, as a dict of: * **mode** : type of expression used to define the computations. One of GREL or SQL. * **name** : name of the column generated * **type** : name of a DSS type for the computed column * **expr** : if **mode** is CUSTOM, a formula in DSS `formula language `_ . If **mode** is SQL, a SQL expression. """ self.raw_virtual_inputs[virtual_input_index]["computedColumns"].append(computed_column) def add_join(self, join_type="LEFT", input1=0, input2=1): """ Add a join between two virtual inputs. The join is initialized with no condition. Use :meth:`add_condition_to_join()` on the return value to add a join condition (for example column equality) to the join. :return: the newly added join as a dict (see :meth:`raw_joins()`) :rtype: dict """ jp = self.obj_payload if not "joins" in jp: jp["joins"] = [] join = { "conditionsMode": "AND", "on": [], "table1": input1, "table2": input2, "type": join_type } jp["joins"].append(join) return join @staticmethod def add_condition_to_join(self, join, type="EQ", column1=None, column2=None): """ Add a condition to a join. :param dict join: definition of a join :param string type: type of join condition. Possible values are EQ, LTE, LT, GTE, GT, NE, WITHIN_RANGE, K_NEAREST, K_NEAREST_INFERIOR, CONTAINS, STARTS_WITH :param string column1: name of left-side column :param string column2: name of right-side column """ cond = { "type" : type, "column1": {"name": column1, "table": join["table1"]}, "column2": {"name": column2, "table": join["table2"]}, } join["on"].append(cond) return cond def add_post_join_computed_column(self, computed_column): """ Add a post-join computed column. Use :class:`dataikuapi.dss.utils.DSSComputedColumn` to build the computed_column object. .. note:: The columns accessible to the expression of the computed column are those selected in the different joins, in their "output" form. For example if a virtual inputs 0 and 1 are joined, and column "bar" of the first input is selected with a prefix of "foo", then the computed column can use "foobar" but not "bar". :param dict computed_column: a computed column definition, as a dict of: * **mode** : type of expression used to define the computations. One of GREL or SQL. * **name** : name of the column generated * **type** : name of a DSS type for the computed column * **expr** : if **mode** is CUSTOM, a formula in DSS `formula language `_ . If **mode** is SQL, a SQL expression. """ self.obj_payload["computedColumns"].append(computed_column) def set_post_filter(self, postfilter): """ Add a post filter on the join. Use the methods on :class:`dataikuapi.dss.utils.DSSFilter` to build filter definition. :param dict postfilter: definition of a filter, as a dict of: * **distinct** : whether the records in the output should be deduplicated * **enabled** : whether filtering is enabled * **uiData** : settings of the filter, if **enabled** is True, as a dict of: * **mode** : type of filter. Possible values: CUSTOM, SQL, '&&' (boolean AND of conditions) and '||' (boolean OR of conditions) * **conditions** : if mode is '&&' or '||', then a list of the actual filter conditions, each one a dict * **expression** : if **uiData.mode** is CUSTOM, a formula in DSS `formula language `_ . If **uiData.mode** is SQL, a SQL expression. """ self.obj_payload["postFilter"] = postfilter def set_unmatched_output(self, ref, side='right', append_mode=False): """ Adds an unmatched join output :param str ref: name of the dataset :param str side: side of the unmatched output, 'right' or 'left'. :param bool append_mode: whether the recipe should append or overwrite the output when running """ if side not in ['right', 'left']: raise ValueError("Unknown side : " + side) if len(self.obj_payload["joins"]) != 1: raise DataikuException("Unmatched output can only be set on recipe that have a single join") join_type = self.obj_payload["joins"][0]["type"] if join_type not in ["RIGHT", "LEFT", "INNER"] or (side == 'left' and join_type == "LEFT") or (side == 'right' and join_type == "RIGHT"): raise DataikuException("No unmatched rows for " + side + " side with join type " + join_type) self._get_or_create_output_role("unmatchedRight" if side == 'right' else "unmatchedLeft")["items"] = [{"ref": ref, "appendMode": append_mode}] class JoinRecipeCreator(VirtualInputsSingleOutputRecipeCreator): """ Create a Join recipe. The recipe is created with default joins guessed by matching column names in the inputs. .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.new_recipe()` instead. """ def __init__(self, name, project): VirtualInputsSingleOutputRecipeCreator.__init__(self, 'join', name, project) class FuzzyJoinRecipeCreator(VirtualInputsSingleOutputRecipeCreator): """ Create a FuzzyJoin recipe The recipe is created with default joins guessed by matching column names in the inputs. .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.new_recipe()` instead. """ def __init__(self, name, project): VirtualInputsSingleOutputRecipeCreator.__init__(self, 'fuzzyjoin', name, project) class GeoJoinRecipeCreator(VirtualInputsSingleOutputRecipeCreator): """ Create a GeoJoin recipe The recipe is created with default joins guessed by matching column names in the inputs. .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.new_recipe()` instead. """ def __init__(self, name, project): VirtualInputsSingleOutputRecipeCreator.__init__(self, 'geojoin', name, project) class StackRecipeSettings(DSSRecipeSettings): """ Settings of a stack recipe. .. important:: Do not instantiate directly, use :meth:`DSSRecipe.get_settings()` """ pass # TODO: Write helpers for stack class StackRecipeCreator(VirtualInputsSingleOutputRecipeCreator): """ Create a Stack recipe .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.new_recipe()` instead. """ def __init__(self, name, project): VirtualInputsSingleOutputRecipeCreator.__init__(self, 'vstack', name, project) class SamplingRecipeSettings(DSSRecipeSettings): """ Settings of a sampling recipe. .. important:: Do not instantiate directly, use :meth:`DSSRecipe.get_settings()` """ pass # TODO: Write helpers for sampling class SamplingRecipeCreator(SingleOutputRecipeCreator): """ Create a Sample/Filter recipe .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.new_recipe()` instead. """ def __init__(self, name, project): SingleOutputRecipeCreator.__init__(self, 'sampling', name, project) class SplitRecipeSettings(DSSRecipeSettings): """ Settings of a split recipe. .. important:: Do not instantiate directly, use :meth:`DSSRecipe.get_settings()` """ pass # TODO: Write helpers for split class SplitRecipeCreator(DSSRecipeCreator): """ Create a Split recipe .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.new_recipe()` instead. """ def __init__(self, name, project): DSSRecipeCreator.__init__(self, "split", name, project) def _finish_creation_settings(self): pass class DownloadRecipeSettings(DSSRecipeSettings): """ Settings of a download recipe. .. important:: Do not instantiate directly, use :meth:`DSSRecipe.get_settings()` """ pass # TODO: Write helpers for download class DownloadRecipeCreator(SingleOutputRecipeCreator): """ Create a Download recipe .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.new_recipe()` instead. """ def __init__(self, name, project): SingleOutputRecipeCreator.__init__(self, 'download', name, project) ##################################################### # Per-recipe-type classes: Code recipes ##################################################### class CodeRecipeSettings(DSSRecipeSettings): """ Settings of a code recipe. .. important:: Do not instantiate directly, use :meth:`DSSRecipe.get_settings()` """ def get_code(self): """ Get the code of the recipe. :rtype: string """ self._payload_to_str() return self._str_payload def set_code(self, code): """ Update the code of the recipe. :param string code: the new code """ self.set_payload(code) def get_code_env_settings(self): """ Get the code env settings for this recipe. :return: settings to select the code env used by the recipe, as a dict of: * **envMode** : one of USE_BUILTIN_MODE, INHERIT (inherit from project settings and/or instance settings), EXPLICIT_ENV * **envName** : if **envMode** is EXPLICIT_ENV, the name of the code env to use :rtype: dict """ rp = self.get_recipe_params() if not "envSelection" in rp: raise ValueError("This recipe kind does not seem to take a code env selection") return rp["envSelection"] def set_code_env(self, code_env=None, inherit=False, use_builtin=False): """ Set which code env this recipe uses. Exactly one of `code_env`, `inherit` or `use_builtin` must be passed. :param string code_env: name of a code env :param boolean inherit: if True, use the project's default code env :param boolean use_builtin: if true, use the builtin code env """ rp = self.get_recipe_params() if not "envSelection" in rp: raise ValueError("This recipe kind does not seem to take a code env selection") if code_env is not None: rp["envSelection"] = {"envMode": "EXPLICIT_ENV", "envName": code_env} elif inherit: rp["envSelection"] = {"envMode": "INHERIT"} elif use_builtin: rp["envSelection"] = {"envMode": "USE_BUILTIN_MODE"} else: raise ValueError("No env setting selected") class CodeRecipeCreator(DSSRecipeCreator): """ Create a recipe running a script. .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProhect.new_recipe()` """ def __init__(self, name, type, project): DSSRecipeCreator.__init__(self, type, name, project) self.script = None def with_script(self, script): """ Set the code of the recipe-to-be-created. :param string script: code of the recipe """ self.script = script return self def with_new_output_dataset(self, name, connection, type_option_id=None, format_option_id=None, copy_partitioning_from="FIRST_INPUT", append=False, overwrite=False): """ Create a new managed dataset as output to the recipe-to-be-created. The dataset is created immediately. :param string name: name of the dataset :param string connection: name of the connection to create the dataset on :param string type_option_id: sub-type of dataset or managed folder, for connections where the type could be ambiguous. Typically applies to SSH connections, where sub-types can be SCP or SFTP :param string format_option_id: name of a format preset relevant for the dataset type. Possible values are: CSV_ESCAPING_NOGZIP_FORHIVE, CSV_UNIX_GZIP, CSV_EXCEL_GZIP, CSV_EXCEL_GZIP_BIGQUERY, CSV_NOQUOTING_NOGZIP_FORPIG, PARQUET_HIVE, AVRO, ORC :param string partitioning_option_id: to copy the partitioning schema of an existing dataset 'foo', pass a value of 'copy:dataset:foo'. If unset, then the output will be non-partitioned :param boolean append: whether the recipe should append or overwrite the output when running (note: not available for all dataset types) :param boolean overwrite: If the dataset being created already exists, overwrite it (and delete data) """ ch = self.project.new_managed_dataset_creation_helper(name) ch.with_store_into(connection, type_option_id=type_option_id, format_option_id=format_option_id) # FIXME: can't manage input folder if copy_partitioning_from == "FIRST_INPUT": inputs = self._get_input_refs() if len(inputs) == 0: logging.warn("No input declared yet, can't copy partitioning from first input") else: self.creation_settings["partitioningOptionId"] = "copy:dataset:%s" % (inputs[0]) elif copy_partitioning_from is not None: self.creation_settings["partitioningOptionId"] = "copy:%s" % copy_partitioning_from ch.create(overwrite=overwrite) self.with_output(name, append=append) return self def with_new_output_streaming_endpoint(self, name, connection, format_option_id=None, overwrite=False): """ Create a new managed streaming endpoint as output to the recipe-to-be-created. The streaming endpoint is created immediately. :param str name: name of the streaming endpoint to create :param str connection: name of the connection to create the streaming endpoint on :param str format_option_id: name of a format preset relevant for the streaming endpoint type. Possible values are: json, avro, single (kafka endpoints) or json, string (SQS endpoints). If None, uses the default :param overwrite: If the streaming endpoint being created already exists, overwrite it """ ch = self.project.new_managed_streaming_endpoint(name) ch.with_store_into(connection, format_option_id=format_option_id) ch.create(overwrite=overwrite) self.with_output(name, append=False) return self def _finish_creation_settings(self): super(CodeRecipeCreator, self)._finish_creation_settings() self.creation_settings['script'] = self.script class PythonRecipeCreator(CodeRecipeCreator): """ Create a Python recipe. .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.new_recipe()` instead. A Python recipe can be defined either by its complete code, like a normal Python recipe, or by a function signature. """ def __init__(self, name, project): DSSRecipeCreator.__init__(self, "python", name, project) _DEFAULT_RECIPE_CODE_TMPL = """ # This code is autogenerated by PythonRecipeCreator function mode import dataiku, dataiku.recipe, json from {module_name} import {fname} input_datasets = dataiku.recipe.get_inputs_as_datasets() output_datasets = dataiku.recipe.get_outputs_as_datasets() params = json.loads('{params_json}') logging.info("Reading %d input datasets as dataframes" % len(input_datasets)) input_dataframes = [ds.get_dataframe() for ds in input_datasets] logging.info("Calling user function {fname}") function_input = input_dataframes if len(input_dataframes) > 1 else input_dataframes[0] output_dataframes = {fname}(function_input, **params) if not isinstance(output_dataframes, list): output_dataframes = [output_dataframes] if not len(output_dataframes) == len(output_datasets): raise Exception("Code function {fname}() returned %d dataframes but recipe expects %d output datasets", \\ (len(output_dataframes), len(output_datasets))) output = list(zip(output_datasets, output_dataframes)) for ds, df in output: logging.info("Writing function result to dataset %s" % ds.name) ds.write_with_schema(df) """ def with_function_name(self, module_name, function_name, custom_template=None, **function_args): """ Define this recipe as being a functional recipe calling a function. With the default template, the function must take as arguments: * A list of dataframes corresponding to the dataframes of the input datasets. If there is only one input, then a single dataframe * Optional named arguments corresponding to arguments passed to the creator as kwargs The function should then return a list of dataframes, one per recipe output. If there is a single output, it is possible to return a single dataframe rather than a list. :param string module_name: name of the module where the function is defined :param string function_name: name of the function :param kwargs function_args: additional parameters to the function. :param string custom_template: template to use to create the code of the recipe. The template is formatted with '{fname}' (function name), '{module_name}' (module name) and '{params_json}' (JSON representation of **function_args**) """ script_tmpl = PythonRecipeCreator._DEFAULT_RECIPE_CODE_TMPL if custom_template is None else custom_template if function_args is None: function_args = {} code = script_tmpl.format(module_name=module_name, fname=function_name, params_json = json.dumps(function_args)) self.with_script(code) return self def with_function(self, fn, custom_template=None, **function_args): """ Define this recipe as being a functional recipe calling a function. With the default template, the function must take as arguments: * A list of dataframes corresponding to the dataframes of the input datasets. If there is only one input, then a single dataframe * Optional named arguments corresponding to arguments passed to the creator as kwargs The function should then return a list of dataframes, one per recipe output. If there is a single output, it is possible to return a single dataframe rather than a list. :param string fn: function to call :param kwargs function_args: additional parameters to the function. :param string custom_template: template to use to create the code of the recipe. The template is formatted with '{fname}' (function name), '{module_name}' (module name) and '{params_json}' (JSON representation of **function_args**) """ import inspect #TODO: add in documentation that relative imports wont work module_name = inspect.getmodule(fn).__name__ fname = fn.__name__ return self.with_function_name(module_name, fname, custom_template, **function_args) class SQLQueryRecipeCreator(SingleOutputRecipeCreator): """ Create a SQL query recipe. .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.new_recipe()` instead. """ def __init__(self, name, project): SingleOutputRecipeCreator.__init__(self, 'sql_query', name, project) ##################################################### # Per-recipe-type classes: ML recipes ##################################################### class PredictionScoringRecipeCreator(SingleOutputRecipeCreator): """ Create a new Prediction scoring recipe. .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.new_recipe()` instead. Usage example: .. code-block:: python # Create a new prediction scoring recipe outputing to a new dataset project = client.get_project("MYPROJECT") builder = project.new_recipe("prediction_scoring", "my_scoring_recipe") builder.with_input_model("saved_model_id") builder.with_input("dataset_to_score") builder.with_new_output("my_output_dataset", "myconnection") # Or for a filesystem output connection # builder.with_new_output("my_output_dataset, "filesystem_managed", format_option_id="CSV_EXCEL_GZIP") new_recipe = builder.build() """ def __init__(self, name, project): SingleOutputRecipeCreator.__init__(self, 'prediction_scoring', name, project) def with_input_model(self, model_id): """ Set the input model. :param string model_id: identifier of a saved model """ return self._with_input(model_id, self.project.project_key, "model") class EvaluationRecipeCreator(DSSRecipeCreator): """ Create a new Evaluate recipe. .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.new_recipe()` instead. Usage example: .. code-block:: python # Create a new evaluation recipe outputing to a new dataset, to a metrics dataset and/or to a model evaluation store project = client.get_project("MYPROJECT") builder = project.new_recipe("evaluation") builder.with_input_model(saved_model_id) builder.with_input("dataset_to_evaluate") builder.with_output("output_scored") builder.with_output_metrics("output_metrics") builder.with_output_evaluation_store(evaluation_store_id) new_recipe = builder.build() # Access the settings er_settings = new_recipe.get_settings() payload = er_settings.obj_payload # Change the settings payload['dontComputePerformance'] = True payload['outputProbabilities'] = False payload['metrics'] = ["precision", "recall", "auc", "f1", "costMatrixGain"] # Manage evaluation labels payload['labels'] = [dict(key="label_1", value="value_1"), dict(key="label_2", value="value_2")] # Save the settings and run the recipe er_settings.save() new_recipe.run() Outputs must exist. They can be created using the following: .. code-block:: python builder = project.new_managed_dataset("output_scored") builder.with_store_into(connection) dataset = builder.create() builder = project.new_managed_dataset("output_scored") builder.with_store_into(connection) dataset = builder.create() evaluation_store_id = project.create_model_evaluation_store("output_model_evaluation").mes_id """ def __init__(self, name, project): DSSRecipeCreator.__init__(self, 'evaluation', name, project) def with_input_model(self, model_id): """ Set the input model. :param string model_id: identifier of a saved model """ return self._with_input(model_id, self.project.project_key, "model") def with_output(self, name): """ Set the output dataset containing the scored input. :param string name: name of an existing dataset """ return self._with_output(name, role="main") def with_output_metrics(self, name): """ Set the output dataset containing the metrics. :param string name: name of an existing dataset """ return self._with_output(name, role="metrics") def with_output_evaluation_store(self, mes_id): """ Set the output model evaluation store. :param string mes_id: identifier of a model evaluation store """ return self._with_output(mes_id, role="evaluationStore") class StandaloneEvaluationRecipeCreator(DSSRecipeCreator): """ Create a new Standalone Evaluate recipe. .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.new_recipe()` instead. Usage example: .. code-block:: python # Create a new standalone evaluation of a scored dataset project = client.get_project("MYPROJECT") builder = project.new_recipe("standalone_evaluation") builder.with_input("scored_dataset_to_evaluate") builder.with_output_evaluation_store(evaluation_store_id) # Add a reference dataset (optional) to compute data drift builder.with_reference_dataset("reference_dataset") # Finish creation of the recipe new_recipe = builder.create() # Modify the model parameters in the SER settings ser_settings = new_recipe.get_settings() payload = ser_settings.obj_payload payload['predictionType'] = "BINARY_CLASSIFICATION" payload['targetVariable'] = "Survived" payload['predictionVariable'] = "prediction" payload['isProbaAware'] = True payload['dontComputePerformance'] = False # For a classification model with probabilities, the 'probas' section can be filled with the mapping of the class and the probability column # e.g. for a binary classification model with 2 columns: proba_0 and proba_1 class_0 = dict(key=0, value="proba_0") class_1 = dict(key=1, value="proba_1") payload['probas'] = [class_0, class_1] # Change the 'features' settings for this standalone evaluation # e.g. reject the features that you do not want to use in the evaluation feature_passengerid = dict(name="Passenger_Id", role="REJECT", type="TEXT") feature_ticket = dict(name="Ticket", role="REJECT", type="TEXT") feature_cabin = dict(name="Cabin", role="REJECT", type="TEXT") payload['features'] = [feature_passengerid, feature_ticket, feature_cabin] # To set the cost matrix properly, access the 'metricParams' section of the payload and set the cost matrix weights: payload['metricParams'] = dict(costMatrixWeights=dict(tpGain=0.4, fpGain=-1.0, tnGain=0.2, fnGain=-0.5)) # Save the recipe and run the recipe # Note that with this method, all the settings that were not explicitly set are instead set to their default value. ser_settings.save() new_recipe.run() Output model evaluation store must exist. It can be created using the following: .. code-block:: python evaluation_store_id = project.create_model_evaluation_store("output_model_evaluation").mes_id """ def __init__(self, name, project): DSSRecipeCreator.__init__(self, 'standalone_evaluation', name, project) def with_output_evaluation_store(self, mes_id): """ Set the output model evaluation store. :param string mes_id: identifier of a model evaluation store """ return self._with_output(mes_id, role="main") def with_reference_dataset(self, dataset_name): """ Set the dataset to use as a reference in data drift computation. :param string dataset_name: name of a dataset """ return self._with_input(dataset_name, self.project.project_key, role="reference") class ClusteringScoringRecipeCreator(SingleOutputRecipeCreator): """ Create a new Clustering scoring recipe,. .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.new_recipe()` instead. Usage example: .. code-block:: python # Create a new prediction scoring recipe outputing to a new dataset project = client.get_project("MYPROJECT") builder = project.new_recipe("clustering_scoring", "my_scoring_recipe") builder.with_input_model("saved_model_id") builder.with_input("dataset_to_score") builder.with_new_output("my_output_dataset", "myconnection") # Or for a filesystem output connection # builder.with_new_output("my_output_dataset, "filesystem_managed", format_option_id="CSV_EXCEL_GZIP") new_recipe = builder.build() """ def __init__(self, name, project): SingleOutputRecipeCreator.__init__(self, 'clustering_scoring', name, project) def with_input_model(self, model_id): """ Set the input model. :param string model_id: identifier of a saved model """ return self._with_input(model_id, self.project.project_key, "model") dataiku-api-client-python-11.4.0/dataikuapi/dss/savedmodel.py000066400000000000000000000637521440464736400242140ustar00rootroot00000000000000 import tempfile from dataikuapi.dss.utils import DSSDatasetSelectionBuilder from .discussion import DSSObjectDiscussions from .managedfolder import DSSManagedFolder from .metrics import ComputedMetrics from .ml import DSSMLTask from .ml import DSSTrainedClusteringModelDetails from .ml import DSSTrainedPredictionModelDetails from ..utils import _make_zipfile, dku_basestring_type try: basestring except NameError: basestring = str class DSSSavedModel(object): """ Handle to interact with a saved model on the DSS instance. .. important:: Do not create this class directly, instead use :meth:`dataikuapi.dss.DSSProject.get_saved_model` :param client: an api client to connect to the DSS backend :type client: :class:`dataikuapi.dssclient.DSSClient` :param str project_key: identifier of the project containing the model :param str sm_id: identifier of the saved model """ def __init__(self, client, project_key, sm_id): self.client = client self.project = client.get_project(project_key) self.project_key = project_key self.sm_id = sm_id @property def id(self): """ Returns the identifier of the saved model :rtype: str """ return self.sm_id def get_settings(self): """ Returns the settings of this saved model. :return: settings of this saved model :rtype: :class:`dataikuapi.dss.savedmodel.DSSSavedModelSettings` """ data = self.client._perform_json( "GET", "/projects/%s/savedmodels/%s" % (self.project_key, self.sm_id)) return DSSSavedModelSettings(self, data) ######################################################## # Versions ######################################################## def list_versions(self): """ Gets the versions of this saved model. :return: a list of the versions, as a dict of object. Each object contains at least an "id" parameter, which can be passed to :meth:`get_metric_values`, :meth:`get_version_details` and :meth:`set_active_version` :rtype: list """ return self.client._perform_json( "GET", "/projects/%s/savedmodels/%s/versions" % (self.project_key, self.sm_id)) def get_active_version(self): """ Gets the active version of this saved model. :return: a dict representing the active version or None if no version is active. The dict contains at least an "id" parameter, which can be passed to :meth:`get_metric_values`, :meth:`get_version_details` and :meth:`set_active_version`. :rtype: dict or None """ filtered = [x for x in self.list_versions() if x["active"]] if len(filtered) == 0: return None else: return filtered[0] def get_version_details(self, version_id): """ Gets details for a version of a saved model :param str version_id: identifier of the version, as returned by :meth:`list_versions` :return: details of this trained model :rtype: :class:`dataikuapi.dss.ml.DSSTrainedPredictionModelDetails` """ details = self.client._perform_json( "GET", "/projects/%s/savedmodels/%s/versions/%s/details" % (self.project_key, self.sm_id, version_id)) snippet = self.client._perform_json( "GET", "/projects/%s/savedmodels/%s/versions/%s/snippet" % (self.project_key, self.sm_id, version_id)) if "facts" in details: return DSSTrainedClusteringModelDetails(details, snippet, saved_model=self, saved_model_version=version_id) else: return DSSTrainedPredictionModelDetails(details, snippet, saved_model=self, saved_model_version=version_id) def set_active_version(self, version_id): """ Sets a particular version of the saved model as the active one. :param str version_id: identifier of the version, as returned by :meth:`list_versions` """ self.client._perform_empty( "POST", "/projects/%s/savedmodels/%s/versions/%s/actions/setActive" % (self.project_key, self.sm_id, version_id)) def delete_versions(self, versions, remove_intermediate=True): """ Deletes version(s) of the saved model. :param list[str] versions: list of versions to delete :param bool remove_intermediate: If True, also removes intermediate versions. In the case of a partitioned model, an intermediate version is created every time a partition has finished training. (defaults to **True**) """ if not isinstance(versions, list): versions = [versions] body = { "versions": versions, "removeIntermediate": remove_intermediate } self.client._perform_empty( "POST", "/projects/%s/savedmodels/%s/actions/delete-versions" % (self.project_key, self.sm_id), body=body) def get_origin_ml_task(self): """ Fetches the last ML task that has been exported to this saved model. :return: origin ML task or None if the saved model does not have an origin ml task :rtype: :class:`dataikuapi.dss.ml.DSSMLTask` or None """ fmi = self.get_settings().get_raw().get("lastExportedFrom") if fmi is not None: return DSSMLTask.from_full_model_id(self.client, fmi, project_key=self.project_key) def import_mlflow_version_from_path(self, version_id, path, code_env_name="INHERIT", container_exec_config_name="NONE", set_active=True, binary_classification_threshold=0.5): """ Creates a new version for this saved model from a path containing a MLFlow model. .. important:: Requires the saved model to have been created using :meth:`dataikuapi.dss.project.DSSProject.create_mlflow_pyfunc_model`. :param str version_id: identifier of the version, as returned by :meth:`list_versions` :param str path: absolute path on the local filesystem - must be a folder, and must contain a MLFlow model :param str code_env_name: Name of the code env to use for this model version. The code env must contain at least mlflow and the package(s) corresponding to the used MLFlow-compatible frameworks. * If value is "INHERIT", the default active code env of the project will be used. (defaults to **INHERIT**) :param str container_exec_config_name: Name of the containerized execution configuration to use for running the evaluation process. * If value is "INHERIT", the container execution configuration of the project will be used. * If value is "NONE", local execution will be used (no container) (defaults to **INHERIT**) :param bool set_active: sets this new version as the active version of the saved model (defaults to **True**) :param float binary_classification_threshold: for binary classification, defines the actual threshold for the imported version (defaults to **0.5**) :return: external model version handler in order to interact with the new MLFlow model version :rtype: :class:`dataikuapi.dss.savedmodel.ExternalModelVersionHandler` """ # TODO: Add a check that it's indeed a MLFlow model folder import shutil import os archive_temp_dir = tempfile.mkdtemp() try: archive_filename = _make_zipfile(os.path.join(archive_temp_dir, "tmpmodel.zip"), path) with open(archive_filename, "rb") as fp: self.client._perform_empty( "POST", "/projects/{project_id}/savedmodels/{saved_model_id}/versions/{version_id}".format( project_id=self.project_key, saved_model_id=self.sm_id, version_id=version_id ), params={"codeEnvName": code_env_name, "containerExecConfigName": container_exec_config_name, "setActive": set_active, "binaryClassificationThreshold": binary_classification_threshold}, files={"file": (archive_filename, fp)}) return self.get_external_model_version_handler(version_id) finally: shutil.rmtree(archive_temp_dir) def import_mlflow_version_from_managed_folder(self, version_id, managed_folder, path, code_env_name="INHERIT", container_exec_config_name="INHERIT", set_active=True, binary_classification_threshold=0.5): """ Creates a new version for this saved model from a managed folder. .. important:: Requires the saved model to have been created using :meth:`dataikuapi.dss.project.DSSProject.create_mlflow_pyfunc_model`. :param str version_id: identifier of the version, as returned by :meth:`list_versions` :param managed_folder: managed folder, or identifier of the managed folder :type managed_folder: :class:`dataikuapi.dss.managedfolder.DSSManagedFolder` or str :param str path: path of the MLflow folder in the managed folder :param str code_env_name: Name of the code env to use for this model version. The code env must contain at least mlflow and the package(s) corresponding to the used MLFlow-compatible frameworks. * If value is "INHERIT", the default active code env of the project will be used. (defaults to **INHERIT**) :param str container_exec_config_name: Name of the containerized execution configuration to use for running the evaluation process. * If value is "INHERIT", the container execution configuration of the project will be used. * If value is "NONE", local execution will be used (no container) (defaults to **INHERIT**) :param bool set_active: sets this new version as the active version of the saved model (defaults to **True**) :param float binary_classification_threshold: for binary classification, defines the actual threshold for the imported version (defaults to **0.5**) :return: external model version handler in order to interact with the new MLFlow model version :rtype: :class:`dataikuapi.dss.savedmodel.ExternalModelVersionHandler` """ # TODO: Add a check that it's indeed a MLFlow model folder folder_ref = None if isinstance(managed_folder, DSSManagedFolder): folder_ref = "{}.{}".format(managed_folder.project_key, managed_folder.id) elif isinstance(managed_folder, dku_basestring_type): folder_ref = managed_folder else: raise Exception("managed_folder should either be a string representing the identifier of the managed folder" " or an instance of dataikuapi.dss.managedfolder.DSSManagedFolder") self.client._perform_empty( "POST", "/projects/{project_id}/savedmodels/{saved_model_id}/versions/{version_id}".format( project_id=self.project_key, saved_model_id=self.sm_id, version_id=version_id ), params={ "folderRef": folder_ref, "path": path, "codeEnvName": code_env_name, "containerExecConfigName": container_exec_config_name, "setActive": set_active, "binaryClassificationThreshold": binary_classification_threshold }, files={"file": (None, None)} # required for backend-mandated multipart request ) return self.get_external_model_version_handler(version_id) def create_proxy_model_version(self, version_id, protocol, configuration): """ EXPERIMENTAL. Creates a new version of a proxy model. This is an experimental API, subject to change. .. important:: Requires the saved model to have been created using :meth:`dataikuapi.dss.project.DSSProject.create_proxy_model`. :param str version_id: identifier of the version, as returned by :meth:`list_versions` :param str protocol: one of ["KServe", "DSS_API_NODE"] :param dict configuration: A dictionary containing the required params for the selected protocol :return: external model version handler in order to interact with the new Proxy model version :rtype: :class:`dataikuapi.dss.savedmodel.ExternalModelVersionHandler` """ import json # Set the type in configuration assert isinstance(configuration, dict), "configuration should be a dictionary containing the required params for the selected protocol" configuration["protocol"] = protocol self.client._perform_empty( "POST", "/projects/{project_id}/savedmodels/{saved_model_id}/versions/{version_id}?containerExecConfigName={containerExecConfigName}".format( project_id=self.project_key, saved_model_id=self.sm_id, version_id=version_id, containerExecConfigName="NONE" ), params={"configuration": json.dumps(configuration)}, files={"file": (None, None)} # required for backend-mandated multipart request ) return self.get_external_model_version_handler(version_id) def get_external_model_version_handler(self, version_id): """ Returns a handler to interact with an external model version (MLflow or Proxy model) :param str version_id: identifier of the version, as returned by :meth:`list_versions` :return: external model version handler :rtype: :class:`dataikuapi.dss.savedmodel.ExternalModelVersionHandler` """ return ExternalModelVersionHandler(self, version_id) ######################################################## # Metrics ######################################################## def get_metric_values(self, version_id): """ Gets the values of the metrics on the version of this saved model :param str version_id: identifier of the version, as returned by :meth:`list_versions` :return: a list of metric objects and their value :rtype: list """ return ComputedMetrics(self.client._perform_json( "GET", "/projects/%s/savedmodels/%s/metrics/%s" % (self.project_key, self.sm_id, version_id))) ######################################################## # Misc ######################################################## def get_zone(self): """ Gets the flow zone of this saved model :return: the saved model's flow zone :rtype: :class:`dataikuapi.dss.flow.DSSFlowZone` """ return self.project.get_flow().get_zone_of_object(self) def move_to_zone(self, zone): """ Moves this object to a flow zone :param object zone: flow zone where the object should be moved :type zone: :class:`dataikuapi.dss.flow.DSSFlowZone` """ if isinstance(zone, basestring): zone = self.project.get_flow().get_zone(zone) zone.add_item(self) def share_to_zone(self, zone): """ Shares this object to a flow zone :param object zone: flow zone where the object should be shared :type zone: :class:`dataikuapi.dss.flow.DSSFlowZone` """ if isinstance(zone, basestring): zone = self.project.get_flow().get_zone(zone) zone.add_shared(self) def unshare_from_zone(self, zone): """ Unshares this object from a flow zone :param object zone: flow zone from which the object shouldn't be shared :type zone: :class:`dataikuapi.dss.flow.DSSFlowZone` """ if isinstance(zone, basestring): zone = self.project.get_flow().get_zone(zone) zone.remove_shared(self) def get_usages(self): """ Gets the recipes referencing this model :return: a list of usages :rtype: list """ return self.client._perform_json("GET", "/projects/%s/savedmodels/%s/usages" % (self.project_key, self.sm_id)) def get_object_discussions(self): """ Gets a handle to manage discussions on the saved model :return: the handle to manage discussions :rtype: :class:`dataikuapi.dss.discussion.DSSObjectDiscussions` """ return DSSObjectDiscussions(self.client, self.project_key, "SAVED_MODEL", self.sm_id) ######################################################## # Deletion ######################################################## def delete(self): """Deletes the saved model""" return self.client._perform_empty("DELETE", "/projects/%s/savedmodels/%s" % (self.project_key, self.sm_id)) class MLFlowVersionSettings: """ Handle for the settings of an imported MLFlow model version. .. important:: Do not create this class directly, instead use :meth:`dataikuapi.dss.savedmodel.ExternalModelVersionHandler.get_settings` :param version_handler: handler to interact with an external model version :type version_handler: :class:`dataikuapi.dss.savedmodel.ExternalModelVersionHandler` :param dict data: raw settings of the imported MLFlow model version """ def __init__(self, version_handler, data): self.version_handler = version_handler self.data = data @property def raw(self): """The raw settings of the imported MLFlow model version""" return self.data def save(self): """Saves the settings of this MLFlow model version""" self.version_handler.saved_model.client._perform_empty("PUT", "/projects/%s/savedmodels/%s/versions/%s/external-ml/metadata" % (self.version_handler.saved_model.project_key, self.version_handler.saved_model.sm_id, self.version_handler.version_id), body=self.data) class ExternalModelVersionHandler: """ Handler to interact with an External model version (MLflow import of Proxy model). .. important:: Do not create this class directly, instead use :meth:`dataikuapi.dss.savedmodel.DSSSavedModel.get_external_model_version_handler` :param saved_model: the saved model object :type saved_model: :class:`dataikuapi.dss.savedmodel.DSSSavedModel` :param str version_id: identifier of the version, as returned by :meth:`dataikuapi.dss.savedmodel.DSSSavedModel.list_versions` """ def __init__(self, saved_model, version_id): self.saved_model = saved_model self.version_id = version_id def get_settings(self): """ Returns the settings of the MLFlow model version :return: settings of the MLFlow model version :rtype: :class:`dataikuapi.dss.savedmodel.MLFlowVersionSettings` """ metadata = self.saved_model.client._perform_json("GET", "/projects/%s/savedmodels/%s/versions/%s/external-ml/metadata" % (self.saved_model.project_key, self.saved_model.sm_id, self.version_id)) return MLFlowVersionSettings(self, metadata) def set_core_metadata(self, target_column_name, class_labels=None, get_features_from_dataset=None, features_list=None, output_style="AUTO_DETECT", container_exec_config_name="NONE"): """ Sets metadata for this MLFlow model version In addition to ``target_column_name``, one of ``get_features_from_dataset`` or ``features_list`` must be passed in order to be able to evaluate performance :param str target_column_name: name of the target column. Mandatory in order to be able to evaluate performance :param class_labels: List of strings, ordered class labels. Mandatory in order to be able to evaluate performance on classification models :type class_labels: list or None :param get_features_from_dataset: mame of a dataset to get feature names from :type get_features_from_dataset: str or None :param features_list: list of ``{"name": "feature_name", "type": "feature_type"}`` :type features_list: list or None :param str container_exec_config_name: name of the containerized execution configuration to use for running the evaluation process. * If value is "INHERIT", the container execution configuration of the project will be used. * If value is "NONE", local execution will be used (no container) (defaults to **None**) """ if features_list is not None and get_features_from_dataset is not None: raise Exception("The information of the features should come either from the features_list or get_features_from_dataset, but not both.") metadata = self.saved_model.client._perform_json("GET", "/projects/%s/savedmodels/%s/versions/%s/external-ml/metadata" % (self.saved_model.project_key, self.saved_model.sm_id, self.version_id)) if target_column_name is not None: metadata["targetColumnName"] = target_column_name if class_labels is not None: metadata["classLabels"] = [{"label": label} for label in class_labels] if get_features_from_dataset is not None: metadata["gatherFeaturesFromDataset"] = get_features_from_dataset # TODO: add support for get_features_from_signature=False, # if get_features_from_signature: # raise Exception("Get features from signature is not yet implemented") if features_list is not None: for feature in features_list: if not ("name" in feature and "type" in feature): raise Exception("The features_list should be a list of {'name': 'feature_name', 'type': 'feature_type'}") metadata["features"] = features_list self.saved_model.client._perform_empty( "PUT", "/projects/{project_key}/savedmodels/{sm_id}/versions/{version_id}/external-ml/metadata?containerExecConfigName={container_exec_config_name}".format( project_key=self.saved_model.project_key, sm_id=self.saved_model.sm_id, version_id=self.version_id, container_exec_config_name=container_exec_config_name), body=metadata) def evaluate(self, dataset_ref, container_exec_config_name="INHERIT", selection=None, use_optimal_threshold=True): """ Evaluates the performance of this model version on a particular dataset. After calling this, the "result screens" of the MLFlow model version will be available (confusion matrix, error distribution, performance metrics, ...) and more information will be available when calling: :meth:`dataikuapi.dss.savedmodel.DSSSavedModel.get_version_details` Evaluation is available only for models having BINARY_CLASSIFICATION, MULTICLASS or REGRESSION as prediction type. See :meth:`DSSProject.create_mlflow_pyfunc_model`. .. important:: :meth:`set_core_metadata` must be called before you can evaluate a dataset :param dataset_ref: Evaluation dataset to use :type dataset_ref: str or :class:`dataikuapi.dss.dataset.DSSDataset` or :class:`dataiku.Dataset` :param str container_exec_config_name: Name of the containerized execution configuration to use for running the evaluation process. * If value is "INHERIT", the container execution configuration of the project will be used. * If value is "NONE", local execution will be used (no container) (defaults to **INHERIT**) :param selection: Sampling parameter for the evaluation. * Example 1: ``DSSDatasetSelectionBuilder().with_head_sampling(100)`` * Example 2: ``{"samplingMethod": "HEAD_SEQUENTIAL", "maxRecords": 100}`` (defaults to **None**) :type selection: dict or :class:`DSSDatasetSelectionBuilder` or None :param bool use_optimal_threshold: Choose between optimized or actual threshold. Optimized threshold has been computed according to the metric set on the saved model setting (i.e. ``prediction_metrics_settings['thresholdOptimizationMetric']``) (defaults to **True**) """ sampling_param = selection.build() if isinstance( selection, DSSDatasetSelectionBuilder) else selection if hasattr(dataset_ref, 'name'): dataset_ref = dataset_ref.name req = { "datasetRef": dataset_ref, "containerExecConfigName": container_exec_config_name, "samplingParam": sampling_param } post = ("/projects/{project_id}/savedmodels/{saved_model_id}/versions/{version_id}/external-ml/actions/evaluate" "?useOptimalThreshold={use_optimal_threshold}").format( project_id=self.saved_model.project_key, saved_model_id=self.saved_model.sm_id, version_id=self.version_id, use_optimal_threshold=use_optimal_threshold) self.saved_model.client._perform_empty("POST", post, body=req) class DSSSavedModelSettings: """ Handle on the settings of a saved model. .. important:: Do not create this class directly, instead use :meth:`dataikuapi.dss.DSSSavedModel.get_settings` :param saved_model: the saved model object :type saved_model: :class:`dataikuapi.dss.savedmodel.DSSSavedModel` :param dict settings: the settings of the saved model """ def __init__(self, saved_model, settings): self.saved_model = saved_model self.settings = settings def get_raw(self): """ Returns the raw settings of the saved model :return: the raw settings of the saved model :rtype: dict """ return self.settings @property def prediction_metrics_settings(self): """ Returns the metrics-related settings :rtype: dict """ return self.settings["miniTask"]["modeling"]["metrics"] def save(self): """Saves the settings of this saved model""" self.saved_model.client._perform_empty("PUT", "/projects/%s/savedmodels/%s" % (self.saved_model.project_key, self.saved_model.sm_id), body=self.settings) dataiku-api-client-python-11.4.0/dataikuapi/dss/scenario.py000066400000000000000000001540011440464736400236600ustar00rootroot00000000000000from datetime import datetime import time, warnings from ..utils import DataikuException from .discussion import DSSObjectDiscussions from .utils import DSSTaggableObjectListItem from dateutil.tz import tzlocal class DSSScenario(object): """ A handle to interact with a scenario on the DSS instance. .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.get_scenario` """ def __init__(self, client, project_key, id): self.client = client self.id = id self.project_key = project_key def abort(self): """ Abort the scenario. This method does nothing if the scenario is not currently running. """ return self.client._perform_json( "POST", "/projects/%s/scenarios/%s/abort" % (self.project_key, self.id)) def run_and_wait(self, params=None, no_fail=False): """ Request a run of the scenario and wait the end of the run to complete. The method requests a new run, which will start after a few seconds. :param dict params: additional parameters that will be passed to the scenario through trigger params (defaults to `{}`) :param boolean no_fail: if False, raises if the run doesn't end with a SUCCESS outcome :return: a handle on the run :rtype: :class:`DSSScenarioRun` """ if params is None: params = {} trigger_fire = self.run(params) scenario_run = trigger_fire.wait_for_scenario_run(no_fail) waiter = DSSScenarioRunWaiter(scenario_run, trigger_fire) return waiter.wait(no_fail) def run(self, params=None): """ Request a run of the scenario. The method requests a new run, which will start after a few seconds. .. note:: This method returns a trigger fire, NOT a Scenario run object. The trigger fire may ultimately result in a run or not. Usage example: .. code-block:: python scenario = project.get_scenario("myscenario") trigger_fire = scenario.run() # When you call `run` a scenario, the scenario is not immediately # started. Instead a "manual trigger" fires. # # This trigger fire can be cancelled if the scenario was already running, # or if another trigger fires. Thus, the scenario run is not available # immediately, and we must "wait" for it scenario_run = trigger_fire.wait_for_scenario_run() # Now the scenario is running. We can wait for it synchronously with # scenario_run.wait_for_completion(), but if we want to do other stuff # at the same time, we can use refresh while True: scenario_run.refresh() if scenario_run.running: print("Scenario is still running ...") else: print("Scenario is not running anymore") break time.sleep(5) :params dict params: additional parameters that will be passed to the scenario through trigger params (defaults to `{}`) :return: a request for a run, as a trigger fire object :rtype: :class:`DSSTriggerFire` """ if params is None: params = {} trigger_fire = self.client._perform_json( "POST", "/projects/%s/scenarios/%s/run" % (self.project_key, self.id), body=params) return DSSTriggerFire(self, trigger_fire) def get_last_runs(self, limit=10, only_finished_runs=False): """ Get the list of the last runs of the scenario. :param int limit: maximum number of last runs to retrieve :param boolean only_finished_runs: if True, currently running runs are not returned. :return: a list of :class:`DSSScenarioRun` :rtype: list """ runs = self.client._perform_json( "GET", "/projects/%s/scenarios/%s/get-last-runs" % (self.project_key, self.id), params={ 'limit' : limit, 'onlyFinishedRuns' : only_finished_runs }) return [DSSScenarioRun(self.client, run) for run in runs] def get_runs_by_date(self, from_date, to_date=datetime.now()): """ Get the list of the runs of the scenario in a given date range. :param datetime from_date: start of the date range to retrieve runs for, inclusive :param datetime to_date: end of the date range to retrieve runs for, exclusive :return: a list of :class:`DSSScenarioRun` :rtype: list """ def as_date(d): if isinstance(d, datetime): return d.strftime("%Y-%m-%d") else: return d runs = self.client._perform_json( "GET", "/projects/%s/scenarios/%s/get-runs-by-date" % (self.project_key, self.id), params={ 'fromDate': as_date(from_date), 'toDate': as_date(to_date) }) return [DSSScenarioRun(self.client, run) for run in runs] def get_last_finished_run(self): """ Get the last run that completed. The run may be successful or failed, or even aborted. :rtype: :class:`DSSScenarioRun` """ lr = [sr for sr in self.get_last_runs() if not sr.running] if len(lr) == 0: raise ValueError("No scenario run completed") return lr[0] def get_last_successful_run(self): """ Get the last run that completed successfully. :rtype: :class:`DSSScenarioRun` """ lr = self.get_last_runs(only_finished_runs=True) if len(lr) == 0: raise ValueError("No scenario run completed successfully") return lr[0] def get_current_run(self): """ Get the current run of the scenario. If the scenario is not running at the moment, returns None. :rtype: :class:`DSSScenarioRun` """ last_run = self.get_last_runs(1) if len(last_run) == 0: return None last_run = last_run[0] if 'result' in last_run.run: return None # has a result means it's done else: return last_run def get_run(self, run_id): """ Get a handle to a run of the scenario. :param string run_id: identifier of the run. :rtype: :class:`DSSScenarioRun` """ run_details = self.client._perform_json( "GET", "/projects/%s/scenarios/%s/%s/" % (self.project_key, self.id, run_id)) return DSSScenarioRun(self.client, run_details["scenarioRun"]) def get_status(self): """ Get the status of this scenario. :rtype: :class:`DSSScenarioStatus` """ data = self.client._perform_json("GET", "/projects/%s/scenarios/%s/light" % (self.project_key, self.id)) return DSSScenarioStatus(self, data) def get_settings(self): """ Get the settings of this scenario. :return: a :class:`StepBasedScenarioSettings` for step-based scenarios, or a :class:`PythonScriptBasedScenarioSettings` for scenarios defined by a python script. :rtype: :class:`DSSScenarioSettings` """ data = self.client._perform_json("GET", "/projects/%s/scenarios/%s" % (self.project_key, self.id)) if data["type"] == "step_based": return StepBasedScenarioSettings(self.client, self, data) else: payload = self.client._perform_json( "GET", "/projects/%s/scenarios/%s/payload" % (self.project_key, self.id)).get('script', '') return PythonScriptBasedScenarioSettings(self.client, self, data, payload) def get_average_duration(self, limit=3): """ Get the average duration of the last runs of this scenario. The duration is computed on successful runs only, that is, on runs that ended with SUCCESS or WARNING satus. If there are not enough runs to perform the average, returns None :param int limit: number of last runs to average on :return: the average duration of the last runs, in seconds :rtype: float """ last_runs = self.get_last_runs(limit=limit, only_finished_runs=True) if len(last_runs) < limit: return None return sum([run.get_duration() for run in last_runs]) / len(last_runs) def delete(self): """ Delete this scenario. """ return self.client._perform_json( "DELETE", "/projects/%s/scenarios/%s" % (self.project_key, self.id)) def get_object_discussions(self): """ Get a handle to manage discussions on the scenario. :returns: the handle to manage discussions :rtype: :class:`dataikuapi.dss.discussion.DSSObjectDiscussions` """ return DSSObjectDiscussions(self.client, self.project_key, "SCENARIO", self.id) ######################################################## # Advanced ######################################################## def get_trigger_fire(self, trigger_id, trigger_run_id): """ Get a trigger fire object. .. caution:: Advanced usages only (see :meth:`run()`) :param string trigger_id: identifier of the trigger, in the scenario's settings :param string trigger_run_id: identifier of the run of the trigger :rtype: :class:`DSSTriggerFire` """ trigger_fire = self.client._perform_json( "GET", "/projects/%s/scenarios/trigger/%s/%s" % (self.project_key, self.id, trigger_id), params={ 'triggerRunId' : trigger_run_id }) return DSSTriggerFire(self, trigger_fire) ######################################################## # Deprecated ######################################################## def get_definition(self, with_status=True): """ Get the definition of the scenario. .. attention:: Deprecated, use :meth:`get_settings` and :meth:`get_status` :param bool with_status: if True, get only the run status of the scenario. If False, get the raw definition of the scenario. :return: if **with_status** is False, the scenario's definition as returned by :meth:`DSSScenarioSettings.get_raw()`. If **with_status** is True, a summary of the scenario as returned by :meth:`DSSScenarioStatus.get_raw()` :rtype: dict """ warnings.warn("DSSScenario.get_definition is deprecated, please use get_settings", DeprecationWarning) suffix = '/light' if with_status else '' return self.client._perform_json( "GET", "/projects/%s/scenarios/%s%s" % (self.project_key, self.id, suffix)) def set_definition(self, definition, with_status=True): """ Update the definition of this scenario. .. attention:: Deprecated, use :meth:`get_settings` and :meth:`DSSScenarioSettings.save` :param dict definition: a scenario definition obtained by calling :meth:`get_definition()`, then modified :param bool with_status: should be the same as the value passed to :meth:`get_definition()`. If True, the only fields that can be modified are active, checklists, description, shortDesc and tags """ warnings.warn("DSSScenario.set_definition is deprecated, please use get_settings", DeprecationWarning) suffix = '/light' if with_status else '' return self.client._perform_json( "PUT", "/projects/%s/scenarios/%s%s" % (self.project_key, self.id, suffix), body = definition) def get_payload(self, extension='py'): """ Get the payload of the scenario. .. attention:: Deprecated, use :meth:`get_settings` and :meth:`get_status` :param string extension: the type of script. Default is 'py' for python """ warnings.warn("DSSScenario.get_payload is deprecated, please use get_settings", DeprecationWarning) return self.client._perform_json( "GET", "/projects/%s/scenarios/%s/payload" % (self.project_key, self.id), body = {'extension' : extension}).get('script', '') # used to have a with_status parameter, that was unused and had a default value, so was likely never used by callers def set_payload(self, script, extension='py'): """ Update the payload of this scenario. .. attention:: Deprecated, use :meth:`get_settings` and :meth:`DSSScenarioSettings.save` :param string script: the new value of the script :param string extension: the type of script. Default is 'py' for python """ warnings.warn("DSSScenario.set_payload is deprecated, please use get_settings", DeprecationWarning) return self.client._perform_json( "PUT", "/projects/%s/scenarios/%s/payload" % (self.project_key, self.id), body = {'script' : script, 'extension' : extension}) class DSSScenarioStatus(object): """ Status of a scenario. .. important:: Do not instantiate directly, use :meth:`DSSScenario.get_status()` """ def __init__(self, scenario, data): self.scenario = scenario self.data = data def get_raw(self): """ Get the raw status data. :return: the status, as a dict with fields: * **projectKey** and **id** : identifiers of the scenario * **name** : name of the scenario, can be distinct from **id** * **shortDesc** : short description of the scenario * **description** : longer description of the scenario, can use markdown * **tags** : list of tags, each one a string * **type** : type of scenario. Possible values are 'custom_python' and 'step_based' * **active** : whether the scenario runs its automatic triggers * **automationLocal** : on automation nodes, flag indicating that the scenario was created on the automation node, as opposed to having been imported via a bundle. * **runAsUser** : login of the user that the scenario runs as. If empty, runs as the last user who modified the scenario * **interest** : watches and stars on the scenario, as a dict * **createdBy** : user who created the scenario, as a dict with fields **login** and **displayName** * **createdOn** : timestamp of creation of the scenario, in milliseconds * **lastModifiedBy** : user who last modified the scenario, as a dict with fields **login** and **displayName** * **lastModifiedOn** : timestamp of last modification of the scenario, in milliseconds * **triggerDigestItems** : list of summaries of the triggers, each one a dict: * **name** : name of the trigger * **description** : description of the trigger * **active** : whether the trigger is active * **nextRun** : for active time-based triggers, the expected timestamp of the next scenario run it will initiate * **triggerDigest** : friendly display of the concatenation of the contents of **triggerDigestItems** * **nextRun** : expected timestamp of the next scenario run that active time-based triggers will initiate * **running** : whether the scenario is currently running * **start** : if the scenario is running, the timestamp of the beginning of the run * **futureId** : identifier of the thread running the scenario in the DSS backend. Can be used to instantiate a :class:`dataikuapi.dss.future.DSSFuture` * **trigger** : if the scenario is running, the trigger fire that initiated it, as a dict (see :meth:DSSTriggerFire.get_raw()`) :rtype: dict """ return self.data @property def running(self): """ Whether the scenario is currently running :rtype: boolean """ return self.data["running"] @property def next_run(self): """ Time at which the scenario is expected to run next. This expected time is computed based on the only triggers for which forecasts are possible, that is, the active time-based triggers. May be None if there is no such trigger. This is an approximate indication as scenario run may be delayed, especially in the case of multiple triggers or high load. :rtype: :class:`datetime.datetime` """ if not "nextRun" in self.data or self.data["nextRun"] == 0: return None return datetime.fromtimestamp(self.data["nextRun"] / 1000) class DSSScenarioSettings(object): """ Settings of a scenario. .. important:: Do not instantiate directly, use :meth:`DSSScenario.get_settings()` """ def __init__(self, client, scenario, data): self.client = client self.scenario = scenario self.data = data def get_raw(self): """ Get the raw definition of the scenario. This method returns a reference to the settings, not a copy. Modifying the settings then calling :meth:`save()` saves the changes made. :return: the scenario, as a dict with fields: * **projectKey** and **id** : identifiers of the scenario * **name** : name of the scenario, can be distinct from **id** * **type** : type of scenario. Possible values are 'custom_python' and 'step_based' * **shortDesc** : short description of the scenario * **description** : longer description of the scenario, can use markdown * **tags** : list of tags, each one a string * **checklists** : definition of checklists on the scenario, as a dict * **params** : type-specific parameters of the scenario, as dict * **active** : whether the scenario runs its automatic triggers * **automationLocal** : on automation nodes, flag indicating that the scenario was created on the automation node, as opposed to having been imported via a bundle. * **runAsUser** : login of the user that the scenario runs as. If empty, runs as the last user who modified the scenario * **delayedTriggersBehavior** : settings for the behavior of triggers firing while the scenario is running, as a dict of: * **delayWhileRunning** : if True, run the scenario again after the current run is done. If False, dismiss the trigger fire * **squashDelayedTriggers** : if True, when several triggers fire during a scenario run, run of the scenario only once afterwards. If False, run as many times as there were trigger fires. * **suppressTriggersWhileRunning** : if True, triggers simply don't evaluate while the scenario runs * **triggers** : list of the automatic triggers, see :meth:`raw_triggers()` * **reporters** : list of reporters on the scenario, see :meth:`raw_reporters()` :rtype: dict """ return self.data @property def active(self): """ Whether this scenario is currently active, i.e. its auto-triggers are executing. :rtype: boolean """ return self.data["active"] @active.setter def active(self, active): self.data["active"] = active @property def run_as(self): """ Get the login of the user the scenario runs as. None means that the scenario runs as the last user who modified the scenario. Only administrators may set a non-None value. :rtype: string """ return self.data.get("runAsUser", None) @run_as.setter def run_as(self, run_as): self.data["runAsUser"] = run_as @property def effective_run_as(self): """ Get the effective 'run as' of the scenario. If the value returned by :meth:`run_as()` is not None, then that value. Otherwise, this will be the login of the last user who modified the scenario. .. note:: If this method returns None, it means that it was not possible to identify who this scenario should run as. This scenario is probably not currently functioning. :rtype: string """ # Note: this logic must match the one in ScenarioBaseService if "runAsUser" in self.data: return self.data["runAsUser"] elif "versionTag" in self.data: return self.data["versionTag"]["lastModifiedBy"]["login"] elif "creationTag" in self.data: return self.data["creationTag"]["lastModifiedBy"]["login"] else: return None @property def raw_triggers(self): """ Get the list of automatic triggers. This method returns a reference to the settings, not a copy. Modifying the settings then calling :meth:`save()` saves the changes made. :return: list of the automatic triggers, each one a dict of: * **id** : identifier of the trigger within the scenario * **type** : type of the trigger * **name** : label of the trigger, distinct from **id** (which is often auto-generated) * **description** : description of the trigger * **active** : if False, the trigger is disabled * **delay** : (deprecated) delay between the time when the trigger fires, and the time a scenario run is requested * **graceDelaySettings** : additional settings for the delaying of the scenario run w.r.t. the trigger fire, as a dict of: * **delay** : delay between the time when the trigger fires, and the time a scenario run is requested * **checkAgainAfterGraceDelay** : if True, the trigger is evaluated again after the delay, and if the trigger fires again, the delay is reset * **params** : type-specific parameters of the trigger :rtype: list[dict] """ return self.data["triggers"] @property def raw_reporters(self): """ Get the list of reporters. This method returns a reference to the settings, not a copy. Modifying the settings then calling :meth:`save()` saves the changes made. :return: list of reporters on the scenario, each one a dict of: * **id** : identifier of the reporter * **name** : label of the reporter * **description** : description of the reporter * **active** : if False, the reporter is disabled and will not send messages * **runConditionEnabled** : if True, the reporter only runs if the **runCondition** evaluates to True * **runCondition** : a condition to check to send the message * **phase** : when the reporter is run. Possible values are START and END (of the scenario run) * **messaging** : definition of how to send the message as a dict of: * **type** : type of message to send, for example 'mail-scenario' or 'slack-scenario' * **configuration** : type-specific parameters for how to produce and send the message, as a dict :rtype: list[dict] """ return self.data["reporters"] def add_periodic_trigger(self, every_minutes=5): """ Add a trigger that runs the scenario every X minutes. :param int every_minutes: interval between activations of the trigger, in minutes """ trigger = {"active": True, "type": "temporal", "params": { "frequency": "Minutely", "repeatFrequency": every_minutes }} self.raw_triggers.append(trigger) def add_hourly_trigger(self, minute_of_hour=0, year=None, month=None, day=None, starting_hour=0, repeat_every=1, timezone="SERVER"): """ Add a trigger that runs the scenario every X hours. :param int repeat_every: interval between activations of the trigger, in hours :param int minute_of_hour: minute in the hour when the trigger should run :param int year: year part of the date/time before which the trigger won't run :param int month: month part of the date/time before which the trigger won't run :param int day: day part of the date/time before which the trigger won't run :param int starting_hour: hour part of the date/time before which the trigger won't run :param string timezone: timezone in which the start date/time is expressed. Can be a time zone name like "Europe/Paris" or "SERVER" for the time zone of the DSS server """ starting_date =self.__get_starting_date__(year=year, month=month, day=day) trigger = {"active": True, "type": "temporal", "params": { "frequency": "Hourly", "hour": starting_hour, "minute": minute_of_hour, "startingFrom": starting_date, "repeatFrequency": repeat_every, "timezone": timezone }} self.raw_triggers.append(trigger) def add_daily_trigger(self, hour=2, minute=0, days=None, year=None, month=None, day=None, repeat_every=1, timezone="SERVER"): """ Add a trigger that runs the scenario every X days. :param int repeat_every: interval between activations of the trigger, in days :param list days: if None, the trigger runs every **repeat_every** other day. If set to a list of day names, the trigger runs every **repeat_every** other week, on the designated days. The day names are Monday, Tuesday, Wednesday, Thursday, Friday, Saturday, Sunday :param int hour: hour in the day when the trigger should run :param int minute: minute in the hour when the trigger should run :param int year: year part of the date/time before which the trigger won't run :param int month: month part of the date/time before which the trigger won't run :param int day: day part of the date/time before which the trigger won't run :param string timezone: timezone in which the start date/time is expressed. Can be a time zone name like "Europe/Paris" or "SERVER" for the time zone of the DSS server """ starting_date =self.__get_starting_date__(year=year, month=month, day=day) if days is None: trigger = {"active": True, "type": "temporal", "params": { "frequency": "Daily", "hour": hour, "minute": minute, "startingFrom": starting_date, "repeatFrequency": repeat_every, "timezone": timezone }} else: trigger = {"active": True, "type": "temporal", "params": { "frequency": "Weekly", "hour": hour, "minute": minute, "startingFrom": starting_date, "daysOfWeek": days, "repeatFrequency": repeat_every, "timezone": timezone }} self.raw_triggers.append(trigger) def add_monthly_trigger(self, day=1, hour=2, minute=0, year=None, month=None, run_on="ON_THE_DAY", repeat_every=1, timezone="SERVER"): """ Add a trigger that runs the scenario every X months. :param int repeat_every: interval between activations of the trigger, in months :parma string run_on: when in the month the trigger should run. Possible values are ON_THE_DAY, LAST_DAY_OF_THE_MONTH, FIRST_WEEK, SECOND_WEEK, THIRD_WEEK, FOURTH_WEEK, LAST_WEEK. :param int day: day in the day when the trigger should run, when **run_on** is ON_THE_DAY or None :param int hour: hour in the day when the trigger should run, when **run_on** is ON_THE_DAY or None :param int minute: minute in the hour when the trigger should run, when **run_on** is ON_THE_DAY or None :param int minute_of_hour: position in the hour of the firing of the trigger :param int year: year part of the date/time before which the trigger won't run :param int month: month part of the date/time before which the trigger won't run :param string timezone: timezone in which the start date/time is expressed. Can be a time zone name like "Europe/Paris" or "SERVER" for the time zone of the DSS server """ starting_date = self.__get_starting_date__(year=year, month=month, day=day, default_is_today=False) trigger = {"active": True, "type": "temporal", "params": { "frequency": "Monthly", "startingFrom": starting_date, "hour": hour, "minute": minute, "monthlyRunOn": run_on, "repeatFrequency": repeat_every, "timezone": timezone }} self.raw_triggers.append(trigger) def __get_starting_date__(self, year, month, day, default_is_today=True): today = datetime.now(tzlocal()) if day is None or not isinstance(day, int) or day < 1 or day > 31: if default_is_today: day = today.day else: # Will be use to avoid Monthly trigger on the 31 of February if the day is not set day = 1 if month is None or not isinstance(month, int) or month < 1 or month > 12: if default_is_today: month = today.month else: # Will be use to avoid Monthly trigger on the 31 of February if the month is not set month = 1 if year is None or not isinstance(year, int): year = today.year start_date = today.replace(year=year, month=month, day=day, hour=0, minute=0, second=0, microsecond=0) return start_date.strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3] + start_date.strftime('%z') def save(self): """ Saves the settings to the scenario """ self.client._perform_json("PUT", "/projects/%s/scenarios/%s" % (self.scenario.project_key, self.scenario.id), body = self.data) class StepBasedScenarioSettings(DSSScenarioSettings): """ Settings of a step-based scenario. .. important:: Do not instantiate directly, use :meth:`DSSScenario.get_settings()`. """ @property def raw_steps(self): """ Returns raw definition of steps. This method returns a reference to the settings, not a copy. Modifying the settings then calling :meth:`save()` saves the changes made. :return: a list of scenario steps, each one a dict with fields: * **id** : identifier of the step (unique in the scenario) * **name** : label of the step * **description** : description of the step * **type** : type of the step. There are many types, commonly used ones are build_flowitem, custom_python or exec_sql * **params** : type-specific parameters for the step, as a dict * **runConditionType** : type of condition for the step to run. Possible values are: * DISABLED : never run the step * RUN_ALWAYS : always run the step, regardless of the current state of the scenario run (even FAILED) * RUN_IF_STATUS_MATCH : run the step if the scenario run's current state is in **runConditionStatuses** * RUN_CONDITIONALLY : run the step if **runConditionExpression** evaluates to True * **runConditionStatuses** : list of scenario run states. Possible states are SUCCESS, WARNING, FAILED, ABORTED * **runConditionExpression** : a `GREL formula `_ . Can use variables, notably the current run state as '${outcome}' * **maxRetriesOnFail** : how many times to retry the step if it ends in FAILED * **delayBetweenRetries** : delay between retries, in seconds * **resetScenarioStatus** : if True, the state of the scenario run before this step is ignored when computing the state post-step :rtype: list[dict] """ return self.data["params"]["steps"] class PythonScriptBasedScenarioSettings(DSSScenarioSettings): """ Settings of a scenario defined by a Python script. .. important:: Do not instantiate directly, use :meth:`DSSScenario.get_settings()`. """ def __init__(self, client, scenario, data, script): super(PythonScriptBasedScenarioSettings, self).__init__(client, scenario, data) self.script = script @property def code(self): """ Get the Python script of the scenario :rtype: string """ return self.script @code.setter def code(self, script): self.script = script def save(self): """ Saves the settings to the scenario. """ super(PythonScriptBasedScenarioSettings, self).save() self.client._perform_json("PUT", "/projects/%s/scenarios/%s/payload" % (self.scenario.project_key, self.scenario.id), body = {'script' : self.script}) class DSSScenarioRun(object): """ A handle containing basic info about a past run of a scenario. .. important:: Do not instantiate directly, use :meth:`DSSScenario.get_run()`, :meth:`DSSScenario.get_current_run()` or :meth:`DSSScenario.get_last_runs()` This handle can also be used to fetch additional information about the run. """ def __init__(self, client, run): self.client = client self.run = run @property def id(self): """ Get the identifier of this run. :rtype: string """ return self.run["runId"] def refresh(self): """ Refresh the details of the run. For ongoing scenario runs, this updates the set of outcomes of the steps and their results. .. note:: This method performs another API call """ updated_run_details = self.client._perform_json("GET", "/projects/%s/scenarios/%s/%s/" % \ (self.run["scenario"]["projectKey"], self.run["scenario"]["id"], self.run["runId"])) self.run = updated_run_details["scenarioRun"] def wait_for_completion(self, no_fail=False): """ Wait for the scenario run to complete. If the scenario run is already finished, this method returns immediately. :param boolean no_fail: if False, raises an exception if scenario fails """ while self.running: self.refresh() time.sleep(5) if self.outcome != 'SUCCESS' and no_fail == False: raise DataikuException("Scenario run returned status %s" % outcome) @property def running(self): """ Whether this scenario run is currently running. :rtype: boolean """ return not "result" in self.run @property def outcome(self): """ The outcome of this scenario run, if available. :return: one of SUCCESS, WARNING, FAILED, or ABORTED :rtype: string """ if not "result" in self.run: raise ValueError("outcome not available for this scenario run. Maybe still running?") return self.run["result"]["outcome"] @property def trigger(self): """ Get the trigger that triggered this scenario run. :return: the definition of a trigger, as in the :meth:`DSSScenarioSettings.raw_triggers()` list :rtype: dict """ return self.run["trigger"]["trigger"] def get_info(self): """ Get the raw information of the scenario run. :return: the scenario run, as a dict with fields: * **scenario** : the scenario of this run, as a dict with the same fields as :meth:`DSSScenarioSettings.get_raw()`. In particular, the dict has fields: * **projectKey** and **id** : identifiers of the scenario * **name** : label of the scenario (can be distinct from **id**) * **active** : whether the automatic triggers of this scenario run * **runAsUser** : user that the scenario is configured to run as. If None, le last user that modified the scenario * **type** : type of scenario (step_based or custom_python) * **runId** : identifier of the run of the scenario * **start** : timestamp of the scenario run beginning, in milliseconds * **end** : timestamp of the scenario run end, in milliseconds. Will be 0 if the scenario run is still ongoing * **trigger** : definition of the trigger fire that initiated the scenario run (see :meth:`DSSTriggerFire.get_raw()`) * **result** : if the scenario run is finished, the detailed outcome of the run, as a dict of: * **type** : type of result (should be SCENARIO_DONE in this context) * **outcome** : one of SUCCESS, WARNING, FAILED, or ABORTED * **target** : definition of which object the result applies to, as a dict. A sub-field **type** indicates which kind of object the **target** is, and additional type-specific fields hold identifiers of the object * **start** : timestamp of the beginning of the operation that produced the result, in milliseconds (the scenario run in this context) * **end** : timestamp of the end of the operation that produced the result, in milliseconds (the scenario run in this context) * **thrown** : if the **outcome** is FAILED, the error that caused the failure, as a dict (see :meth:`DSSStepRunDetails.first_error_details()`) * **logTail** : if the **outcome** is FAILED, the last lines of the log of the operation that produced the result, as a dict of: * **totalLines** : number of lines in the full log (not just in this log tail) * **lines** : list of lines, each a string * **status** : detected log level of each line, as a list of int * **runAsUser** : information about the user that the scenario run effectively runs as, as a dict of: * **authSource** : type of user in DSS. One of USER_FROM_UI, PERSONAL_API_KEY, CONFIGURABLE_API_KEY_GLOBAL, CONFIGURABLE_API_KEY_PROJECT * **apiKey** : for API_KEY authentication sources, the API key as a dict, with at least a **key** field * **realUserLogin** : when **authSource** is USER_FROM_UI, the login of the user * **variables** : variables passed from parent scenario, in situations where a scenario A runs a sub-scenario B, as a dict * **clustersUsed** : list of clusters created by start or attach cluster steps, as a list of dicts with **key** (for the cluster id) and **value** (for the name of the variable in which the cluster id is stored) fields * **reportersStates** : the results of running the reporters of the scenario, as a list of dict. Only reporters that have completed are listed. Sub-fields are: * **reporterName** : label of the reporter * **messagingType** : type of reporter * **activated** : if False, the reporter's run condition prevented it from executing * **error** : if the reporter failed, a dict with the error details. * **messages** : messages outputted while executing the reporter, as a dict with a **messages** field which is a list of messages * **started** : when the reporter started executing, in milliseconds * **ended** : when the reporter was done executing, in milliseconds :rtype: dict """ return self.run def get_details(self): """ Get the full details of the scenario run. This includes notably the individual step runs inside the scenario run. .. note:: This method performs another API call :return: full details on a scenario run, as a dict with fields: * **scenarioRun** : the run definition and base status, as a dict (see :meth:`get_info()`) * **stepRuns** : details about each step that has executed so far, as a list of dicts (see :meth:`DSSScenarioRunDetails.steps()`) :rtype: dict """ raw_data = self.client._perform_json( "GET", "/projects/%s/scenarios/%s/%s/" % (self.run['scenario']['projectKey'], self.run['scenario']['id'], self.run['runId'])) details = DSSScenarioRunDetails(raw_data) if "stepRuns" in details: structured_steps = [] for step in details["stepRuns"]: structured_steps.append(DSSStepRunDetails(step)) details["stepRuns"] = structured_steps return details def get_start_time(self): """ Get the start time of the scenario run. :rtype: :class:`datetime.datetime` """ return datetime.fromtimestamp(self.run['start'] / 1000) start_time = property(get_start_time) def get_end_time(self): """ Get the end time of the scenario run, if it completed, else raises. :rtype: :class:`datetime.datetime` """ if "end" in self.run and self.run["end"] > 0: return datetime.fromtimestamp(self.run['end'] / 1000) else: raise ValueError("Scenario run has not completed") end_time = property(get_end_time) def get_duration(self): """ Get the duration of this run (in fractional seconds). If the run is still running, get the duration since it started. :rtype: float """ end_time = datetime.now() if self.run['end'] > 0: end_time = datetime.fromtimestamp(self.run['end'] / 1000) duration = (end_time - self.get_start_time()).total_seconds() return duration duration = property(get_duration) class DSSScenarioRunDetails(dict): """ Details of a scenario run, notably the outcome of its steps. .. important:: Do not instantiate directly, see :meth:`DSSScenarioRun.get_details()` """ def __init__(self, data): super(DSSScenarioRunDetails, self).__init__(data) @property def steps(self): """ Get the list of runs of the steps. Only completed or ongoing steps are included. .. note:: When the instance of :class:`DSSScenarioRunDetails` was obtained via :meth:`DSSScenarioRun.get_details()`, then the returned list is made of instances of :class:`DSSStepRunDetails`. :return: a list of step runs, each as a dict with fields: * **scenarioRun** : the scenario run to which this step run belongs (see :meth:`DSSScenarioRun.get_info()`) * **step** : definition of the step that is running, as a dict (see :meth:`StepBasedScenarioSettings.raw_steps()`) * **runId** : unique identifier of the step run within the scenario run * **retryIndex** : how many times the step was retried before this step run * **start** : timestamp of the step run beginning, in milliseconds * **end** : timestamp of the step run end, in milliseconds. Will be 0 if the scenario run is still ongoing * **result** : result of the step itself, as a dict: * **type** : type of result (should be SCENARIO_DONE in this context) * **outcome** : one of SUCCESS, WARNING, FAILED, or ABORTED * **target** : definition of which object the result applies to, as a dict. A sub-field **type** indicates which kind of object the **target** is, and additional type-specific fields hold identifiers of the object * **start** : timestamp of the beginning of the operation that produced the result, in milliseconds (the scenario run in this context) * **end** : timestamp of the end of the operation that produced the result, in milliseconds (the scenario run in this context) * **thrown** : if the **outcome** is FAILED, the error that caused the failure, as a dict (see :meth:`DSSStepRunDetails.first_error_details()`) * **logTail** : if the **outcome** is FAILED, the last lines of the log of the operation that produced the result, as a dict of: * **totalLines** : number of lines in the full log (not just in this log tail) * **lines** : list of lines, each a string * **status** : detected log level of each line, as a list of int * **additionalReportItems** : additional results that the step can produce, as a list of dict with the structure of **result**. Can be datasets built, jobs runs, ... :rtype: list[dict] """ return self["stepRuns"] @property def last_step(self): """ Get the last step run. :return: a step run, as a dict. See :meth:`steps()`. :rtype: dict """ return self["stepRuns"][len(self["stepRuns"]) - 1] @property def first_error_details(self): """ Get the details of the first error if this run failed. This will not always be able to find the error details (it returns None in that case) :return: a serialized exception, as in :meth:`DSSStepRunDetails.first_error_details()` :rtype: dict """ for step in self.steps: step_error = step.first_error_details if step_error is not None: return step_error class DSSStepRunDetails(dict): """ Details of a run of a step in a scenario run. .. important:: Do not instantiate directly, see :meth:`DSSScenarioRunDetails.steps()` on an instance of :class:`DSSScenarioRunDetails` obtained via :meth:`DSSScenarioRun.get_details()` """ def __init__(self, data): super(DSSStepRunDetails, self).__init__(data) @property def outcome(self): """ Get the outcome of the step run/ :return: one of SUCCESS, WARNING, FAILED, or ABORTED :rtype: string """ return self["result"]["outcome"] @property def job_ids(self): """ Get the list of DSS job ids that were run as part of this step. :return: a list of job ids, each one a string :rtype: list[string] """ return [ri["jobId"] for ri in self["additionalReportItems"] if ri["type"] == "JOB_EXECUTED"] @property def first_error_details(self): """ Try to get the details of the first error if this step failed. This will not always be able to find the error details (it returns None in that case) :return: a serialized exception, as a with fields: * **clazz** : class name of the exception * **title** : short message of the exception * **message** : message of the exception * **stack** : stacktrace of the exception, as a single string * **code** : well-known error code, as listed in `the doc `_ * **fixability** : type of action to take in order to remediate the issue, if possible. For example USER_CONFIG_DATASET, ADMIN_SETTINGS_SECURITY, ... :rtype: dict """ if self.outcome == 'FAILED': step_thrown = self.get('result').get('thrown', None) if step_thrown is not None: return step_thrown for item in self['additionalReportItems']: if item.get("outcome", None) == 'FAILED': item_thrown = item.get('thrown', None) if item_thrown is not None: return item_thrown class DSSScenarioRunWaiter(object): """ Helper to wait for a scenario to run to complete. """ def __init__(self, scenario_run, trigger_fire): self.trigger_fire = trigger_fire self.scenario_run = scenario_run def wait(self, no_fail=False): """ Wait for the scenario run completion. :param boolean no_fail: if False, raises if the run doesn't end with a SUCCESS outcome :return: the final state of the scenario run (see :meth:`DSSScenarioRun.get_info()`) :rtype: dict """ while not self.scenario_run.run.get('result', False): self.scenario_run = self.trigger_fire.get_scenario_run() time.sleep(5) outcome = self.scenario_run.run.get('result', None).get('outcome', 'UNKNOWN') if outcome == 'SUCCESS' or no_fail: return self.scenario_run else: raise DataikuException("Scenario run returned status %s" % outcome) class DSSTriggerFire(object): """ A handle representing the firing of a trigger on a scenario. .. important:: Do not instantiate directly, use :meth:`DSSScenario.run` """ def __init__(self, scenario, trigger_fire): self.client = scenario.client self.project_key = scenario.project_key self.scenario_id = scenario.id self.trigger_id = trigger_fire['trigger']['id'] self.run_id = trigger_fire['runId'] self.trigger_fire = trigger_fire def get_raw(self): """ Get the definition of the trigger fire event. :return: the trigger fire, as a dict of: * **trigger** : definition of the trigger which fired, as a dict (see :meth:`DSSScenarioSettings.raw_triggers()`) * **runId** : identifier of the trigger fire. Can be used with :meth:`get_trigger_fire()` * **timestamp** : when the trigger fired (precedes the scenario run start) * **triggerState** : current internal state of the trigger, as a string * **params** : parameters passed by the trigger to the scenario (depend on trigger type), as a dict * **triggerAuthCtx** : for manual runs, identity of the user who started the scenario, as a dict """ def wait_for_scenario_run(self, no_fail=False): """ Poll for the run of the scenario that the trigger fire should initiate. This methos waits for the run of the sceanrio that this trigger activation launched to be available, or for the trigger fire to be cancelled (possibly cancelled by another trigger firing). :param boolean no_fail: if True, return None if the trigger fire is cancelled, else raise :return: a handle on a scenario run, or None :rtype: :class:`DSSScenarioRun` """ scenario_run = None refresh_trigger_counter = 0 while scenario_run is None: refresh_trigger_counter += 1 if refresh_trigger_counter == 10: refresh_trigger_counter = 0 if self.is_cancelled(refresh=refresh_trigger_counter == 0): if no_fail: return None else: raise DataikuException("Scenario run has been cancelled") scenario_run = self.get_scenario_run() time.sleep(5) return scenario_run def get_scenario_run(self): """ Get the run of the scenario that this trigger fire launched. May return None if the scenario run started from this trigger has not yet been created. :return: a handle on a scenario run, or None :rtype: :class:`DSSScenarioRun` """ run = self.client._perform_json( "GET", "/projects/%s/scenarios/%s/get-run-for-trigger" % (self.project_key, self.scenario_id), params= { 'triggerId' : self.trigger_id, 'triggerRunId' : self.run_id }) if 'scenarioRun' not in run: return None else: return DSSScenarioRun(self.client, run['scenarioRun']) def is_cancelled(self, refresh=False): """ Whether the trigger fire has been cancelled :param refresh: get the state of the trigger from the backend """ if refresh == True: self.trigger_fire = self.client._perform_json( "GET", "/projects/%s/scenarios/trigger/%s/%s" % (self.project_key, self.scenario_id, self.trigger_id), params={ 'triggerRunId' : self.run_id }) return self.trigger_fire["cancelled"] class DSSScenarioListItem(DSSTaggableObjectListItem): """ An item in a list of scenarios. .. important:: Do not instantiate directly, use :meth:`dataikuapi.dss.project.DSSProject.list_scenarios()` """ def __init__(self, client, data): super(DSSScenarioListItem, self).__init__(data) self.client = client def to_scenario(self): """ Get a handle corresponding to this scenario. :rtype: :class:`DSSScenario` """ return DSSScenario(self.client, self._data["projectKey"], self._data["id"]) @property def id(self): """ Get the identifier of the scenario. :rtype: string """ return self._data["name"] @property def running(self): """ Whether the scenario is currently running. :rtype: boolean """ return self._data["running"] @property def start_time(self): """ Get the start time of the scenario run. :return: timestap of the scenario run start, or None if it's not running at the moment. :rtype: :class:`datetime.datetime` """ if self._data['start'] <= 0: return None else: return datetime.fromtimestamp(self._data['start'] / 1000) dataiku-api-client-python-11.4.0/dataikuapi/dss/sqlquery.py000066400000000000000000000106251440464736400237450ustar00rootroot00000000000000from ..utils import DataikuException from ..utils import DataikuUTF8CSVReader from ..utils import DataikuStreamedHttpUTF8CSVReader import json class DSSSQLQuery(object): """ A connection to a database or database-like on which queries can be run through DSS. .. important:: Do not create this class directly, instead use :meth:`dataikuapi.DSSClient.sql_query` Usage example: .. code-block:: python # run some query on a connection query = client.sql_query('select * from "public"."SOME_TABLE"', connection='some_postgres_connection') n = 0 for row in query.iter_rows(): n += 1 if n < 10: print("row %s : %s" % (n, row)) query.verify() print("Returned %s rows" % n) """ def __init__(self, client, query, connection, database, dataset_full_name, pre_queries, post_queries, type, extra_conf, script_steps, script_input_schema, script_output_schema, script_report_location, read_timestamp_without_timezone_as_string, read_date_as_string, project_key): self.client = client self.streaming_session = self.client._perform_json( "POST", "/sql/queries/", body = { "query" : query, "preQueries" : pre_queries, "postQueries" : post_queries, "connection" : connection, "database" : database, "datasetFullName" : dataset_full_name, "projectKey" : project_key, "type" : type, "extraConf" : extra_conf, "scriptSteps" : script_steps, "scriptInputSchema" : script_input_schema, "scriptOutputSchema" : script_output_schema, "scriptReportLocation" : script_report_location, "readTimestampWithoutTimezoneAsString" : read_timestamp_without_timezone_as_string, "readDateAsString" : read_date_as_string }) self.queryId = self.streaming_session['queryId'] def get_schema(self): """ Get the query's result set's schema. The schema made of DSS column types, and built from mapping database types to DSS types. The actual type in the database can be found in the `originalType` field (`originalSQLType` in BigQuery) :return: a schema, as a dict with a `columns` array, in which each element is a column, itself as a dict of * **name** : the column name * **type** : the column type (smallint, int, bigint, float, double, boolean, date, string) * **length** : the string length * **comment** : the column name * **originalType** : type of the column in the database :rtype: dict """ return self.streaming_session['schema'] def iter_rows(self): """ Get an iterator on the query's results. :return: an iterator over the rows, each row being a tuple of values. The order of values in the tuples is the same as the order of columns in the schema returned by :meth:`~get_schema()`. The values are cast to python types according to the types in :meth:`~get_schema()` :rtype: iterator[list] """ csv_stream = self.client._perform_raw( "GET", "/sql/queries/%s/stream" % (self.queryId), params = { "format" : "tsv-excel-noheader" }) return DataikuStreamedHttpUTF8CSVReader(self.get_schema(), csv_stream).iter_rows() def verify(self): """ Verify that reading results completed successfully. When using the :meth:`~iter_rows()` method, and the iterator stops returning rows, there is no way to tell whether there are no more rows because the query didn't return more rows, or because an error in the query, or in the fetching of its results, happened. You should thus call :meth:`~verify()` after the iterator is done, because it will raise an Expcetion if an error happened. :raises: Exception """ resp = self.client._perform_empty( "GET", "/sql/queries/%s/finish-streaming" % (self.queryId)) # exception raising is done in _perform_empty()dataiku-api-client-python-11.4.0/dataikuapi/dss/statistics.py000066400000000000000000000246021440464736400242520ustar00rootroot00000000000000from .utils import DSSDatasetSelectionBuilder from .future import DSSFuture class DSSStatisticsWorksheet(object): """ A handle to interact with a worksheet. .. important:: Do not create this class directly, instead use :meth:`dataikuapi.dss.dataset.DSSDataset.get_statistics_worksheet` """ def __init__(self, client, project_key, dataset_name, worksheet_id): self.client = client self.project_key = project_key self.dataset_name = dataset_name self.worksheet_id = worksheet_id def delete(self): """ Deletes the worksheet. """ self.client._perform_empty( "DELETE", "/projects/%s/datasets/%s/statistics/worksheets/%s" % (self.project_key, self.dataset_name, self.worksheet_id)) def get_settings(self): """ Gets the worksheet settings. :returns: a handle for the worksheet settings :rtype: :class:`DSSStatisticsWorksheetSettings` """ worksheet_json = self.client._perform_json( "GET", "/projects/%s/datasets/%s/statistics/worksheets/%s" % ( self.project_key, self.dataset_name, self.worksheet_id) ) return DSSStatisticsWorksheetSettings(self.client, self.project_key, self.dataset_name, self.worksheet_id, worksheet_json) def run_worksheet(self, wait=True): """ Computes the result of the whole worksheet. When `wait` is `True`, the method waits for the computation to complete and returns the corresponding result, otherwise it returns immediately a handle for a :class:`~dataikuapi.dss.future.DSSFuture` result. :param bool wait: a flag to wait for the computation to complete (defaults to **True**) :returns: the corresponding card result or a handle for the future result :rtype: :class:`~DSSStatisticsCardResult` or :class:`~dataikuapi.dss.future.DSSFuture` """ root_card = self.get_settings().get_raw()['rootCard'] return self.run_card(root_card, wait=wait) def run_card(self, card, wait=True): """ Computes the result of a card in the context of the worksheet. When `wait` is `True`, the method waits for the computation to complete and returns the corresponding result, otherwise it returns immediately a handle for a :class:`~dataikuapi.dss.future.DSSFuture` result. .. note:: The card does not need to belong to the worksheet. :param card: the card to compute :type card: :class:`~DSSStatisticsCardSettings` or dict obtained from :meth:`DSSStatisticsCardSettings.get_raw` :param bool wait: a flag to wait for the computations to complete (defaults to **True**) :returns: the corresponding card result or a handle for the future result :rtype: :class:`DSSStatisticsCardResult` or :class:`~dataikuapi.dss.future.DSSFuture` """ card = DSSStatisticsCardSettings._from_card_or_dict(self.client, card) future_response = self.client._perform_json( "POST", "/projects/%s/datasets/%s/statistics/worksheets/%s/actions/run-card" % ( self.project_key, self.dataset_name, self.worksheet_id), body=card.get_raw() ) future = DSSFuture(self.client, future_response.get("jobId", None), future_response, result_wrapper=lambda raw_result: DSSStatisticsCardResult(raw_result)) return future.wait_for_result() if wait else future def run_computation(self, computation, wait=True): """ Runs a computation in the context of the worksheet. When `wait` is `True`, the method waits for the computation to complete and returns the corresponding result, otherwise it returns immediately a handle for a :class:`~dataikuapi.dss.future.DSSFuture` result. :param computation: the computation to perform :type computation: :class:`DSSStatisticsComputationSettings` or dict obtained from :meth:`DSSStatisticsComputationSettings.get_raw` :param bool wait: a flag to wait for the computations to complete (defaults to **True**) :returns: the corresponding computation result or a handle for the future result :returns: :class:`DSSStatisticsComputationResult` or :class:`~dataikuapi.dss.future.DSSFuture` """ computation = DSSStatisticsComputationSettings._from_computation_or_dict( computation) future_response = self.client._perform_json( "POST", "/projects/%s/datasets/%s/statistics/worksheets/%s/actions/run-computation" % ( self.project_key, self.dataset_name, self.worksheet_id), body=computation.get_raw() ) future = DSSFuture(self.client, future_response.get("jobId", None), future_response, result_wrapper=lambda raw_result: DSSStatisticsComputationResult(raw_result)) return future.wait_for_result() if wait else future class DSSStatisticsWorksheetSettings(object): """ A handle to interact with the worksheet settings. .. important:: Do not create this class directly, instead use :meth:`DSSStatisticsWorksheet.get_settings` """ def __init__(self, client, project_key, dataset_name, worksheet_id, worksheet_definition): self._worksheet_definition = worksheet_definition self.client = client self.project_key = project_key self.dataset_name = dataset_name self.worksheet_id = worksheet_id def add_card(self, card): """ Adds a new card to the worksheet. :param card: the card to add :type card: :class:`DSSStatisticsCardSettings` or dict obtained from :meth:`DSSStatisticsCardSettings.get_raw` """ card = DSSStatisticsCardSettings._from_card_or_dict(self.client, card) self._worksheet_definition['rootCard']['cards'].append(card.get_raw()) def list_cards(self): """ Lists the cards for the worksheet. :returns: a list of card handles :rtype: list of :class:`DSSStatisticsCardSettings` """ return [DSSStatisticsCardSettings(self.client, card_definition) for card_definition in self._worksheet_definition['rootCard']['cards']] def get_raw(self): """ Gets a reference to the raw representation of the worksheet settings. :returns: the worksheet settings :rtype: dict """ return self._worksheet_definition def set_sampling_settings(self, selection): """ Sets the worksheet sampling settings. :param selection: the sampling settings :type selection: :class:`~dataikuapi.dss.utils.DSSDatasetSelectionBuilder` or dict obtained from :meth:`get_raw_sampling_settings` """ raw_selection = selection.build() if isinstance( selection, DSSDatasetSelectionBuilder) else selection self._worksheet_definition['dataSpec']['datasetSelection'] = raw_selection def get_raw_sampling_settings(self): """ Gets a reference to the raw representation of the worksheet sampling settings. :returns: the sampling settings :rtype: dict """ return self._worksheet_definition['dataSpec']['datasetSelection'] def save(self): """ Saves the settings of the worksheet. """ self._worksheet_definition = self.client._perform_json( "PUT", "/projects/%s/datasets/%s/statistics/worksheets/%s" % ( self.project_key, self.dataset_name, self.worksheet_id), body=self._worksheet_definition ) class DSSStatisticsCardSettings(object): """ A handle to interact with the card settings. """ def __init__(self, client, card_definition): self.client = client self._card_definition = card_definition def get_raw(self): """ Gets a reference to the raw representation of the card settings. :returns: the card settings :rtype: dict """ return self._card_definition def compile(self): """ Gets the computation used to compute the card result. :returns: the computation settings :rtype: :class:`DSSStatisticsComputationSettings` """ computation_json = self.client._perform_json( "POST", "/statistics/cards/compile", body=self._card_definition ) return DSSStatisticsComputationSettings(computation_json) @staticmethod def _from_card_or_dict(client, card_or_dict): if isinstance(card_or_dict, DSSStatisticsCardSettings): card_or_dict = card_or_dict.get_raw() return DSSStatisticsCardSettings(client, card_or_dict) class DSSStatisticsCardResult(object): """ A handle to interact with the computed result of a :class:`DSSStatisticsCardSettings`. """ def __init__(self, card_result): self._card_result = card_result def get_raw(self): """ Gets a reference to the raw representation of the card result. :returns: the card result :rtype: dict """ return self._card_result class DSSStatisticsComputationSettings(object): """ A handle to interact with the computation settings. """ def __init__(self, computation_definition): self._computation_definition = computation_definition def get_raw(self): """ Gets a reference to the raw representation of the computation settings. :returns: the computation settings :rtype: dict """ return self._computation_definition @staticmethod def _from_computation_or_dict(computation_or_dict): if isinstance(computation_or_dict, DSSStatisticsComputationSettings): computation_or_dict = computation_or_dict.get_raw() return DSSStatisticsComputationSettings(computation_or_dict) class DSSStatisticsComputationResult(object): """ A handle to interact with the computed result of a :class:`DSSStatisticsComputationSettings`. """ def __init__(self, computation_result): self._computation_result = computation_result def get_raw(self): """ Gets a reference to the raw representation of the computation result. :returns: the computation result :rtype: dict """ return self._computation_result dataiku-api-client-python-11.4.0/dataikuapi/dss/streaming_endpoint.py000066400000000000000000000674631440464736400257650ustar00rootroot00000000000000from ..utils import DataikuException from .future import DSSFuture import json, warnings, logging from .utils import DSSTaggableObjectListItem from .future import DSSFuture from .discussion import DSSObjectDiscussions from . import recipe try: basestring except NameError: basestring = str class DSSStreamingEndpointListItem(DSSTaggableObjectListItem): """ An item in a list of streaming endpoints. .. important:: Do not instantiate this class, use :meth:`.DSSProject.list_streaming_endpoints()` instead """ def __init__(self, client, data): super(DSSStreamingEndpointListItem, self).__init__(data) self.client = client def __repr__(self): return u"<{} {}.{}>".format(self.__class__.__name__, self._data["projectKey"], self._data["id"]) def to_streaming_endpoint(self): """ Get a handle on the corresponding streaming endpoint. :rtype: :class:`DSSStreamingEndpoint` """ return DSSStreamingEndpoint(self.client, self._data["projectKey"], self._data["id"]) @property def name(self): """ Get the streaming endpoint name. :rtype: string """ return self._data["id"] @property def id(self): """ Get the streaming endpoint identifier. .. note:: For streaming endpoints, the identifier is equal to its name :rtype: string """ return self._data["id"] @property def type(self): """ Get the streaming endpoint type. :returns: the type is the same as the type of the connection that the streaming endpoint uses, ie `Kafka`, `SQS`, `KDBPlus` or `httpsse` :rtype: string """ return self._data["type"] @property def schema(self): """ Get the schema of the streaming endpoint. Usage example: .. code-block:: python # list all endpoints containing columns with a 'PII' in comment for endpoint in p.list_streaming_endpoints(): column_list = endpoint.schema['columns'] pii_columns = [c for c in column_list if 'PII' in c.get("comment", "")] if len(pii_columns) > 0: print("Streaming endpoint %s contains %s PII columns" % (endpoint.id, len(pii_columns))) :returns: a schema, as a dict with a `columns` array, in which each element is a column, itself as a dict of * **name** : the column name * **type** : the column type (smallint, int, bigint, float, double, boolean, date, string) * **length** : the string length * **comment** : user comment about the column :rtype: dict """ return self._data["schema"] @property def connection(self): """ Get the connection on which this streaming endpoint is attached. :returns: a connection name, or None for HTTP SSE endpoints :rtype: string """ if not "params" in self._data: return None return self._data["params"].get("connection", None) def get_column(self, column): """ Get a column in the schema, by its name. :param string column: name of column to find :returns: a dict of the column settings or None if column does not exist. Fields are: * **name** : the column name * **type** : the column type (smallint, int, bigint, float, double, boolean, date, string) * **length** : the string length * **comment** : user comment about the column :rtype: dict """ matched = [col for col in self.schema["columns"] if col["name"] == column] return None if len(matched) == 0 else matched[0] class DSSStreamingEndpoint(object): """ A streaming endpoint on the DSS instance. """ def __init__(self, client, project_key, streaming_endpoint_name): self.client = client self.project = client.get_project(project_key) self.project_key = project_key self.streaming_endpoint_name = streaming_endpoint_name @property def name(self): """ Get the streaming endpoint name. :rtype: string """ return self.streaming_endpoint_name @property def id(self): """ Get the streaming endpoint identifier. .. note:: For streaming endpoints, the identifier is equal to its name :rtype: string """ return self.streaming_endpoint_name ######################################################## # Streaming Endpoint deletion ######################################################## def delete(self): """ Delete the streaming endpoint from the flow, and objects using it (recipes or continuous recipes) .. attention:: This call doesn't delete the underlying streaming data. For example for a Kafka streaming endpoint the topic isn't deleted """ return self.client._perform_empty( "DELETE", "/projects/%s/streamingendpoints/%s" % (self.project_key, self.streaming_endpoint_name), params = { }) ######################################################## # Streaming Endpoint definition ######################################################## def get_settings(self): """ Get the settings of this streaming endpoint. Know subclasses of :class:`DSSStreamingEndpointSettings` include :class:`KafkaStreamingEndpointSettings` and :class:`HTTPSSEStreamingEndpointSettings` You must use :meth:`~DSSStreamingEndpointSettings.save()` on the returned object to make your changes effective on the streaming endpoint. .. code-block:: python # Example: changing the topic on a kafka streaming endpoint streaming_endpoint = project.get_streaming_endpoint("my_endpoint") settings = streaming_endpoint.get_settings() settings.set_connection_and_topic(None, "country") settings.save() :returns: an object containing the settings :rtype: :class:`DSSStreamingEndpointSettings` or a subclass """ data = self.client._perform_json("GET", "/projects/%s/streamingendpoints/%s" % (self.project_key, self.streaming_endpoint_name)) if data["type"] in ['kafka']: return KafkaStreamingEndpointSettings(self, data) elif data["type"] in ['httpsse']: return HTTPSSEStreamingEndpointSettings(self, data) else: return DSSStreamingEndpointSettings(self, data) def exists(self): """ Whether this streaming endpoint exists in DSS. :rtype: boolean """ try: self.get_schema() return True except Exception as e: return False ######################################################## # Streaming Endpoint metadata ######################################################## def get_schema(self): """ Get the schema of the streaming endpoint. :returns: a schema, as a dict with a `columns` array, in which each element is a column, itself as a dict of * **name** : the column name * **type** : the column type (smallint, int, bigint, float, double, boolean, date, string) * **comment** : user comment about the column :rtype: dict """ return self.client._perform_json( "GET", "/projects/%s/streamingendpoints/%s/schema" % (self.project_key, self.streaming_endpoint_name)) def set_schema(self, schema): """ Set the schema of the streaming endpoint. Usage example: .. code-block:: python # copy schema of the input of a continuous recipe to its output recipe = p.get_recipe('my_recipe_name') recipe_settings = recipe.get_settings() input_endpoint = p.get_streaming_endpoint(recipe_settings.get_flat_input_refs()[0]) output_endpoint = p.get_streaming_endpoint(recipe_settings.get_flat_output_refs()[0]) output_endpoint.set_schema(input_endpoint.get_schema()) :param dict schema: a schema, as a dict with a `columns` array, in which each element is a column, itself as a dict of * **name** : the column name * **type** : the column type (smallint, int, bigint, float, double, boolean, date, string) * **comment** : user comment about the column """ return self.client._perform_json( "PUT", "/projects/%s/streamingendpoints/%s/schema" % (self.project_key, self.streaming_endpoint_name), body=schema) ######################################################## # Misc ######################################################## def get_zone(self): """ Get the flow zone of this streaming endpoint. :returns: a flow zone :rtype: :class:`dataikuapi.dss.flow.DSSFlowZone` """ return self.project.get_flow().get_zone_of_object(self) def move_to_zone(self, zone): """ Move this object to a flow zone. :param object zone: a :class:`dataikuapi.dss.flow.DSSFlowZone` where to move the object, or its identifier """ if isinstance(zone, basestring): zone = self.project.get_flow().get_zone(zone) zone.add_item(self) def share_to_zone(self, zone): """ Share this object to a flow zone. :param object zone: a :class:`dataikuapi.dss.flow.DSSFlowZone` where to share the object, or its identifier """ if isinstance(zone, basestring): zone = self.project.get_flow().get_zone(zone) zone.add_shared(self) def unshare_from_zone(self, zone): """ Unshare this object from a flow zone. :param object zone: a :class:`dataikuapi.dss.flow.DSSFlowZone` from where to unshare the object, or its identifier """ if isinstance(zone, basestring): zone = self.project.get_flow().get_zone(zone) zone.remove_shared(self) def get_usages(self): """ Get the recipes referencing this streaming endpoint. Usage example: .. code-block:: python for usage in streaming_endpoint.get_usages(): if usage["type"] == 'RECIPE_INPUT': print("Used as input of %s" % usage["objectId"]) :returns: a list of usages, each one a dict of: * **type** : the type of usage, either "RECIPE_INPUT" or "RECIPE_OUTPUT" * **objectId** : name of the recipe or continuous recipe * **objectProjectKey** : project of the recipe or continuous recipe :rtype: list[dict] """ return self.client._perform_json("GET", "/projects/%s/streamingendpoints/%s/usages" % (self.project_key, self.streaming_endpoint_name)) ######################################################## # Discussions ######################################################## def get_object_discussions(self): """ Get a handle to manage discussions on the streaming endpoint. :returns: the handle to manage discussions :rtype: :class:`dataikuapi.dss.discussion.DSSObjectDiscussions` """ return DSSObjectDiscussions(self.client, self.project_key, "STREAMING_ENDPOINT", self.streaming_endpoint_name) ######################################################## # Test / Autofill ######################################################## def test_and_detect(self, infer_storage_types=False, limit=10, timeout=60): """ Used internally by :meth:`autodetect_settings`. It is not usually required to call this method .. attention:: Only Kafka and HTTP-SSE streaming endpoints are handled .. note:: Schema inferrence is done on the captured rows from the underlying stream. If no record is captured, for example because no message is posted to the stream while the capture is done, then no schema inferrence can take place. :returns: a future object on the result of the detection. The future's result is a dict with fields: * **table** : the captured rows * **schemaDetection** : the result of the schema inference. Notable sub-fields are * **detectedSchema** : the inferred schema * **detectedButNotInSchema** : list of column names found in the capture, but not yet in the endpoint's schema * **inSchemaButNotDetected** : list of column names present in the endpoint's schema, but not found in the capture :rtype: :class:`dataikuapi.dss.future.DSSFuture` """ settings = self.get_settings() if settings.type in ['kafka']: future_resp = self.client._perform_json("POST", "/projects/%s/streamingendpoints/%s/actions/testAndDetectSettings/kafka"% (self.project_key, self.streaming_endpoint_name), body = { "inferStorageTypes" : infer_storage_types , "limit" : limit, "timeout" : timeout }) return DSSFuture(self.client, future_resp.get('jobId', None), future_resp) elif settings.type in ['httpsse']: future_resp = self.client._perform_json("POST", "/projects/%s/streamingendpoints/%s/actions/testAndDetectSettings/httpsse"% (self.project_key, self.streaming_endpoint_name), body = { "inferStorageTypes" : infer_storage_types , "limit" : limit, "timeout" : timeout }) return DSSFuture(self.client, future_resp.get('jobId', None), future_resp) else: raise ValueError("don't know how to test/detect on streaming endpoint type:%s" % settings.type) def autodetect_settings(self, infer_storage_types=False, limit=10, timeout=60): """ Detect an appropriate schema for this streaming endpoint using Dataiku detection engine. The detection bases itself on a capture of the stream that this streaming endpoint represents. First a number of messages are captured from the stream, within the bounds passed as parameters, then a detection of the columns and their types is done on the captured messages. If no message is send on the stream during the capture, no column is inferred. .. attention:: Only Kafka and HTTP-SSE streaming endpoints are handled .. note:: Format-related settings are not automatically inferred. For example the method will not detect whether Kafka messages are JSON-encoded or Avro-encoded Usage example: .. code-block:: python # create a kafka endpoint on a json stream and detect its schema e = project.create_kafka_streaming_endpoint('test_endpoint', 'kafka_connection', 'kafka-topic') s = e.autodetect_settings(infer_storage_types=True) s.save() :param boolean infer_storage_types: if True, DSS will try to guess types of the columns. If False, all columns will be assumed to be strings (default: False) :param int limit: max number of rows to use for the autodetection (default: 10) :param int timeout: max duration in seconds of the stream capture to use for the autodetection (default: 60) :returns: streaming endpoint settings with an updated schema that you can :meth:`DSSStreamingEndpointSettings.save`. :rtype: :class:`DSSStreamingEndpointSettings` or a subclass """ settings = self.get_settings() if settings.type in ['kafka']: future = self.test_and_detect(infer_storage_types, limit, timeout) result = future.wait_for_result() if result.get("schemaDetection", {}).get("detectedSchema") is None: logging.warn("No schema detected, leaving existing schema") else: settings.get_raw()["schema"] = result["schemaDetection"]["detectedSchema"] return settings elif settings.type in ['httpsse']: future = self.test_and_detect(infer_storage_types, limit, timeout) result = future.wait_for_result() if result.get("schemaDetection", {}).get("detectedSchema") is None: logging.warn("No schema detected, leaving existing schema") else: settings.get_raw()["schema"] = result["schemaDetection"]["detectedSchema"] return settings else: raise ValueError("don't know how to test/detect on streaming endpoint type:%s" % settings.type) def get_as_core_streaming_endpoint(self): """ Get the streaming endpoint as a handle for use inside DSS. :returns: the streaming endpoint :rtype: :class:`dataiku.StreamingEndpoint` """ import dataiku return dataiku.StreamingEndpoint("%s.%s" % (self.project_key, self.streaming_endpoint_name)) ######################################################## # Creation of recipes ######################################################## def new_code_recipe(self, type, code=None, recipe_name=None): """ Start the creation of a new code recipe taking this streaming endpoint as input. Usage example: .. code-block:: python # create a continuous python recipe from an endpoint recipe_creator = endpoint.new_code_recipe('cpython', 'some code here', 'compute_something') recipe_creator.with_new_output_streaming_endpoint('something', 'my_kafka_connection', 'avro') recipe = recipe_creator.create() :param string type: type of the recipe. Can be 'cpython', 'streaming_spark_scala' or 'ksql'; the non-continuous type 'python' is also possible :param string code: (optional) The script of the recipe :param string recipe_name: (optional) base name for the new recipe. :returns: an object to create a new recipe :rtype: :class:`dataikuapi.dss.recipe.CodeRecipeCreator` """ builder = recipe.CodeRecipeCreator(recipe_name, type, self.project) builder.with_input(self.streaming_endpoint_name) if code is not None: builder.with_script(code) return builder def new_recipe(self, type, recipe_name=None): """ Start the creation of a new recipe taking this streaming endpoint as input. This method can create non-code recipes like Sync or continuous Sync recipes. For more details, please see :meth:`dataikuapi.dss.project.DSSProject.new_recipe`. Usage example: .. code-block:: python # create a continuous sync from an endpoint to a dataset recipe_creator = endpoint.new_recipe('csync', 'compute_my_dataset_name') recipe_creator.with_new_output('my_dataset_name', 'filesystem_managed') recipe = recipe_creator.create() :param string type: type of the recipe. Possible values are 'csync', 'cpython', 'python', 'ksql', 'streaming_spark_scala' :param string recipe_name: (optional) base name for the new recipe. :returns: A new DSS Recipe Creator handle :rtype: :class:`dataikuapi.dss.recipe.DSSRecipeCreator` or a subclass """ builder = self.project.new_recipe(type=type, name=recipe_name) builder.with_input(self.streaming_endpoint_name) return builder class DSSStreamingEndpointSettings(object): """ Base settings class for a DSS streaming endpoint. .. important:: Do not instantiate this class directly, use :meth:`DSSStreamingEndpoint.get_settings` Use :meth:`save` to save your changes """ def __init__(self, streaming_endpoint, settings): self.streaming_endpoint = streaming_endpoint self.settings = settings def get_raw(self): """ Get the streaming endpoint settings as a dict. :returns: the settings. Top-level fields are: * **id** : the name (used as identifier) of the streaming endpoint * **type** : the type of the connection underlying the streaming endpoint (Kafka, SQS, ...) * **params** : the type-specific parameters, as a dict. Depend on the connection type * **tags** : list of tags, each a string * **schema** : the columns of the streaming endpoing, as a dict with a `columns` array, in which each element is a column, itself as a dict of * **name** : the column name * **type** : the column type (smallint, int, bigint, float, double, boolean, date, string) * **length** : the string length * **comment** : user comment about the column :rtype: dict """ return self.settings def get_raw_params(self): """ Get the type-specific (Kafka/ HTTP-SSE/ Kdb+...) params as a dict. :returns: the type-specific params. Each type defines a set of fields; commonly found fields are : * **connection** : name of the connection used by the streaming endpoint * **topic** or **queueName** : name of the Kafka topic or SQS queue corresponding to this streaming endpoint :rtype: dict """ return self.settings["params"] @property def type(self): """ Get the type of streaming system that the streaming endpoint uses. :return: a type of streaming system. Possible values: 'kafka', 'SQS', 'kdbplustick', 'httpsse' :rtype: string """ return self.settings["type"] def add_raw_schema_column(self, column): """ Add a column in the schema :param dict column: a dict defining the column. It should contain a "name" field and a "type" field (with a DSS type, like bigint, double, string, date, boolean, ...), optionally a "length" field for string-typed columns and a "comment" field. """ self.settings["schema"]["columns"].append(column) def save(self): """ Save the changes to the settings on the streaming endpoint. """ self.streaming_endpoint.client._perform_empty( "PUT", "/projects/%s/streamingendpoints/%s" % (self.streaming_endpoint.project_key, self.streaming_endpoint.streaming_endpoint_name), body=self.settings) class KafkaStreamingEndpointSettings(DSSStreamingEndpointSettings): """ Settings for a Kafka streaming endpoint. This class inherits from :class:`DSSStreamingEndpointSettings`. .. important:: Do not instantiate this class directly, use :meth:`DSSStreamingEndpoint.get_settings` Use :meth:`save` to save your changes """ def __init__(self, streaming_endpoint, settings): super(KafkaStreamingEndpointSettings, self).__init__(streaming_endpoint, settings) def set_connection_and_topic(self, connection=None, topic=None): """ Change the connection and topic of an endpoint. :param string connection: (optional) name of a Kafka connection in DSS :param string topic: (optional) name of a Kafka topic. Can contain DSS variables """ if connection is not None: self.settings["params"]["connection"] = connection if topic is not None: self.settings["params"]["topic"] = topic class HTTPSSEStreamingEndpointSettings(DSSStreamingEndpointSettings): """ Settings for a HTTP-SSE streaming endpoint. This class inherits from :class:`DSSStreamingEndpointSettings`. .. important:: Do not instantiate this class directly, use :meth:`DSSStreamingEndpoint.get_settings` Use :meth:`save` to save your changes """ def __init__(self, streaming_endpoint, settings): super(HTTPSSEStreamingEndpointSettings, self).__init__(streaming_endpoint, settings) def set_url(self, url): """ Change the URL of the endpoint. :param string url: url to connect to """ self.settings["params"]["url"] = url class DSSManagedStreamingEndpointCreationHelper(object): """ Utility class to help create a new streaming endpoint. .. note:: Do not instantiate directly, use :meth:`.DSSProject.new_managed_streaming_endpoint()` instead .. important:: Only Kafka and SQS endpoints support managed streaming endpoints """ def __init__(self, project, streaming_endpoint_name, streaming_endpoint_type): self.project = project self.streaming_endpoint_name = streaming_endpoint_name self.streaming_endpoint_type = streaming_endpoint_type self.creation_settings = {} def get_creation_settings(self): """ Get the settings for the creation of the new managed streaming endpoint. .. note:: Modifying the values in the creation settings directly is discouraged, use :meth:`with_store_into()` instead. :returns: the settings as a dict :rtype: dict """ return self.creation_settings def with_store_into(self, connection, format_option_id = None): """ Set the DSS connection underlying the new streaming endpoint. :param string connection: name of the connection to store into :param string format_option_id: (optional) identifier of a serialization format option. For Kafka endpoints, possible values are : * **json** : messages are JSON strings * **single** : messages are handled as a single typed field * **avro** : messages are Kafka-Avro messages (ie an avro message padded with a field indicating the avro schema version in the kafka schema registry) For SQS endpoints, possible values are * **json** : messages are JSON strings * **string** : messages are raw strings :returns: self """ self.creation_settings["connectionId"] = connection if format_option_id is not None: self.creation_settings["formatOptionId"] = format_option_id return self def create(self, overwrite=False): """ Execute the creation of the streaming endpoint according to the selected options :param boolean overwrite: If the streaming endpoint being created already exists, delete it first :returns: an object corresponding to the newly created streaming endpoint :rtype: :class:`DSSStreamingEndpoint` """ if overwrite and self.already_exists(): self.project.get_streaming_endpoint(self.streaming_endpoint_name).delete() self.project.client._perform_json("POST", "/projects/%s/streamingendpoints/managed" % self.project.project_key, body = { "id": self.streaming_endpoint_name, "type": self.streaming_endpoint_type, "creationSettings": self.creation_settings }) return DSSStreamingEndpoint(self.project.client, self.project.project_key, self.streaming_endpoint_name) def already_exists(self): """ Whether the desired name for the new streaming endpoint is already used by another streaming endpoint :rtype: boolean """ streaming_endpoint = self.project.get_streaming_endpoint(self.streaming_endpoint_name) try: streaming_endpoint.get_schema() return True except Exception as e: return Falsedataiku-api-client-python-11.4.0/dataikuapi/dss/tools/000077500000000000000000000000001440464736400226425ustar00rootroot00000000000000dataiku-api-client-python-11.4.0/dataikuapi/dss/tools/__init__.py000066400000000000000000000000001440464736400247410ustar00rootroot00000000000000dataiku-api-client-python-11.4.0/dataikuapi/dss/tools/codegen.py000066400000000000000000000410521440464736400246220ustar00rootroot00000000000000import logging import re from ..dataset import DSSDataset from ..recipe import CodeRecipeSettings class _IndentContext(object): def __init__(self, flow_code_generator): self.flow_code_generator = flow_code_generator def __enter__(self): self.flow_code_generator.cur_indent += 1 def __exit__(self, b, c, d): self.flow_code_generator.cur_indent -= 1 def slugify(name): return re.sub(r"[^A-Za-z0-9_]", "_", name) class FlowCodeGenerator(object): def __init__(self): self.code = "" self.cur_indent = 0 self.remove_metrics_on_datasets = False self.remove_display_width_on_prepare = False def set_options(self, remove_metrics_on_datasets=False, remove_display_width_on_prepare=False): self.remove_metrics_on_datasets = remove_metrics_on_datasets self.remove_display_width_on_prepare = remove_display_width_on_prepare pass def generate_code_for_dataset(self, dataset): entrypoint_name = "create_dataset_%s" % dataset.dataset_name self._generate_code_for_dataset(dataset, entrypoint_name) return self.code def generate_code_for_recipe(self, recipe): entrypoint_name = "create_recipe_%s" % recipe.recipe_name self._generate_code_for_recipe(recipe, entrypoint_name) return self.code def generate_code_for_project(self, project, entrypoint_name = None): self.gen("import json") self.gen("") if entrypoint_name is None: entrypoint_name = "create_flow_for_project" self.gen("def %s(project):" % entrypoint_name) flow_graph = project.get_flow().get_graph() flow_items = flow_graph.get_items_in_traversal_order(as_type="object") entrypoints_to_call = [] with _IndentContext(self) as ic: for item in flow_items: if isinstance(item, DSSDataset): entrypoint_name = "create_dataset_%s" % slugify(item.dataset_name) self._generate_code_for_dataset(item, entrypoint_name) else: entrypoint_name = "create_recipe_%s" % slugify(item.recipe_name) self._generate_code_for_recipe(item, entrypoint_name) entrypoints_to_call.append(entrypoint_name) self.gen("") self.gen("# Actual creation of the Flow from the individual functions") for ep in entrypoints_to_call: self.gen("%s(project)" % ep) return self.code def _generate_code_for_dataset(self, dataset, entrypoint_name): self.gen("def %s(project):" % entrypoint_name) settings = dataset.get_settings() raw = settings.get_raw() templates = dataset.client._perform_json("GET", "/projects/X/datasets/templates") do_not_copy = [ "projectKey", "name", "type", "versionTag", "creationTag", "schema" ] self.gen(" # Base dataset params") if raw["type"] == "UploadedFiles": self.gen(" dataset = project.create_upload_dataset(\"%s\")" % dataset.dataset_name) self.gen(" settings = dataset.get_settings()") self.lf() self.codegen_object_fields_explicit(settings.get_raw(), templates["dataset"], ["params"], "settings.get_raw()") do_not_copy.append("params") elif raw["type"] in DSSDataset.FS_TYPES: srcp = raw["params"] self.gen(" dataset = project.create_fslike_dataset(\"%s\", \"%s\", \"%s\", \"%s\")" % \ (dataset.dataset_name, raw["type"], srcp["connection"], srcp.get("path", "/"))) self.gen(" settings = dataset.get_settings()") self.lf() self.codegen_object_fields(srcp, templates["abstractFSConfig"], ["connection", "path"], "settings.get_raw_params()") do_not_copy.append("params") elif raw["type"] in DSSDataset.SQL_TYPES: self.gen(" dataset = project.create_dataset(\"%s\", \"%s\")" % (dataset.dataset_name, raw["type"])) self.gen(" settings = dataset.get_settings()") self.lf() srcp = raw["params"] if srcp.get("mode", None) == "table": self.gen(" settings.set_table(%s, %s, %s)" % \ (self.objstr(srcp.get("connection")), self.objstr(srcp.get("schema")), self.objstr(srcp.get("table")))) self.codegen_object_fields(srcp, templates["abstractSQLConfig"], ["mode", "connection", "schema", "table"], "settings.get_raw_params()") do_not_copy.append("params") else: self.gen(" dataset = project.create_dataset(\"%s\", \"%s\")" % (dataset.dataset_name, raw["type"])) self.gen(" settings = dataset.get_settings()") self.codegen_object_fields_explicit(settings.get_raw(), templates["dataset"], ["params"], "settings.get_raw()") do_not_copy.append("params") # Copy of format params if "formatType" in raw: self.lf() self.gen(" # Format params") handled = False if raw["formatType"] == "csv": csv_format = raw["formatParams"] self.gen(" settings.set_csv_format(separator=\"%s\", style=\"%s\", skip_rows_before=%d, header_row=%s, skip_rows_after=%d)"%\ (csv_format.get("separator", None), csv_format["style"], csv_format["skipRowsBeforeHeader"],\ csv_format["parseHeaderRow"], csv_format["skipRowsAfterHeader"])) self.codegen_object_fields(csv_format, templates["csvFormat"], ["separator", "style", "skipRowsBeforeHeader", "parseHeaderRow", "skipRowsAfterHeader", "probableNumberOfRecords"], "settings.get_raw_format_params()") else: self.codegen_object_fields_explicit(settings.get_raw(), templates["dataset"], ["formatType", "formatParams"], "settings.get_raw()") do_not_copy.extend(["formatType", "formatParams"]) self.lf() self.gen(" # Schema") for column in settings.get_raw()["schema"]["columns"]: self.gen(" settings.add_raw_schema_column(%s)" % column) if not self.remove_metrics_on_datasets: self.lf() self.gen(" # Metrics") self.codegen_object_fields(settings.get_raw()["metrics"], templates["dataset"]["metrics"], [], "settings.get_raw()[\"metrics\"]") do_not_copy.append("metrics") self.lf() self.gen(" # Other dataset params") self.codegen_object_fields(settings.get_raw(), templates["dataset"], do_not_copy, "settings.get_raw()") self.lf() self.gen(" settings.save()") self.lf() def _generate_code_for_recipe(self, recipe, entrypoint_name): logging.info("Codegen for recipe %s" % entrypoint_name) self.gen("def %s(project):" % entrypoint_name) settings = recipe.get_settings() raw = settings.get_recipe_raw_definition() templates = recipe.client._perform_json("GET", "/projects/%s/recipes/templates" % recipe.project_key) template = {"tags":[], "optionalDependencies": False, "redispatchPartitioning": False, "maxRunningActivities": 0, "neverRecomputeExistingPartitions" : False, "customFields":{}, "customMeta": {"kv":{}}, "checklists" : {"checklists":[]}} do_not_copy = [ "projectKey", "name", "type", "versionTag", "creationTag", "inputs", "outputs" ] self.gen(" # Create the recipe as a blank recipe") self.gen(" builder = project.new_recipe(\"%s\", \"%s\")" % (raw["type"], recipe.recipe_name)) self.gen(" builder.set_raw_mode()") self.gen(" recipe = builder.create()") self.lf() self.gen(" # Setup the recipe") self.gen(" settings = recipe.get_settings()" ) self.lf() self.gen(" # Recipe inputs/outputs") for (input_role, input_item) in settings._get_flat_inputs(): if len(input_item["deps"]) > 0: self.gen(" settings.add_input(%s, %s, %s)" % (self.objstr(input_role), self.objstr(input_item["ref"]), self.objstr(input_item["deps"]))) else: self.gen(" settings.add_input(%s, %s)" % (self.objstr(input_role), self.objstr(input_item["ref"]))) for (output_role, output_item) in settings._get_flat_outputs(): self.gen(" settings.add_output(%s, %s, %s)" % (self.objstr(output_role), self.objstr(output_item["ref"]), self.objstr(output_item["appendMode"]))) self.lf() types_with_obj_payload = ["join", "grouping", "shaker"] # A bit of "classical cleanup" # Remove the dirty "map" in Spark read params if settings.type in types_with_obj_payload and "engineParams" in settings.obj_payload: rp = settings.obj_payload["engineParams"].get("spark", {}).get("readParams", {}) if rp.get("mode", "?") == "AUTO": rp["map"] = {} if raw is not None and "params" in raw and "engineParams" in raw["params"]: rp = raw["params"]["engineParams"].get("spark", {}).get("readParams", {}) if rp.get("mode", "?") == "AUTO": rp["map"] = {} # And some per-type cleanup def cleanup_grouping(): for grouping_key in settings.obj_payload.get("keys", []): for item in ["count", "last", "min", "max", "sum", "countDistinct", "stddev", "avg", "concat", "first"]: if item in grouping_key and grouping_key[item] == False: del grouping_key[item] for aggregation in settings.obj_payload.get("values", []): for item in ["count", "last", "min", "max", "sum", "countDistinct", "stddev", "avg", "concat", "first", "concatDistinct", "$idx", "sum2", "firstLastNotNull"]: if item in aggregation and aggregation.get(item, None) == False: del aggregation[item] def cleanup_join(): for vi in settings.raw_virtual_inputs: if not vi["preFilter"]["enabled"]: del vi["preFilter"] def cleanup_shaker(): if self.remove_display_width_on_prepare: if "columnWidthsByName" in settings.obj_payload: del settings.obj_payload["columnWidthsByName"] cleanup_by_type = { "grouping": cleanup_grouping, "join": cleanup_join, "shaker": cleanup_shaker } if settings.type in cleanup_by_type: cleanup_by_type[settings.type]() # Output payload, either globally for code if isinstance(settings, CodeRecipeSettings): code = settings.get_code() self.gen(" # Recipe code") self.gen(" settings.set_payload(%s)" % self.payloadstr(code)) # per-field for recipes with JSON payload elif settings.type in types_with_obj_payload: prefix_by_type = { "join": "Join details", "shaker": "Prepare script" } self.gen(" # %s" % (prefix_by_type.get(settings.type, "Recipe payload"))) payload = settings.obj_payload payload_template = templates["payloadsByType"][settings.type] self.gen(" settings.set_payload(\"{}\")") self.codegen_object_fields(payload, payload_template, [], "settings.obj_payload") # Or as string for others elif len(settings.get_payload()) > 0: self.gen(" # Recipe payload") self.gen(" settings.set_payload(%s)" % self.payloadstr(settings.get_payload())) # Then params if settings.type in templates["paramsByType"] and "params" in raw: self.lf() self.gen(" # Type-specific parameters") self.codegen_object_fields(raw["params"], templates["paramsByType"][settings.type], [], "settings.raw_params") do_not_copy.append("params") # And finally other recipe fields that are not params self.lf() self.gen(" # Other parameters") self.codegen_object_fields(raw, template, do_not_copy, "settings.get_recipe_raw_definition()") self.lf() self.gen(" settings.save()") self.lf() # Helpers def gen(self, code): self.code += "%s%s\n" % (" " * (4 * self.cur_indent), code) def lf(self): self.code += "\n" def payloadstr(self, payload): if payload.endswith("\n"): return "\"\"\"%s\"\"\"" % payload else: return "\"\"\"%s\n\"\"\"" % payload def objstr(self, obj): return ObjectFieldFormatter(self.cur_indent + 1).format(obj) def codegen_object_fields_explicit(self, object, template, copy, prefix): for key in copy: if not key in object: continue self.codegen_object_field(object, key, template, prefix) def codegen_object_fields(self, object, template, do_not_copy, prefix): for key in object.keys(): if key in do_not_copy: continue self.codegen_object_field(object, key, template, prefix) def codegen_object_field(self, object, key, template, prefix): value_for_key = object[key] default_value_for_key = template.get(key, None) if default_value_for_key is not None and value_for_key == default_value_for_key: #print("Skipping value equal to default: %s" % key) return else: #print("Not equal for %s" % key) #print("Template: %s" % default_value_for_key) #print("Real: %s" % value_for_key) self.gen(" %s[\"%s\"] = %s" % ( prefix, key, self.objstr(value_for_key))) class ObjectFieldFormatter(object): def __init__(self, base_indent): self.formatters = { dict: self.__class__.format_dict, list: self.__class__.format_list, } self.sp = " " self.base_indent = base_indent self.no_pretty_level = 0 def format(self, value, depth=0): base_repr = repr(value) if len(base_repr) < 25: # This entire value is very small, don't bother pretty-priting it return base_repr elif type(value) in self.formatters: return self.formatters[type(value)](self, value, depth) else: return base_repr def format_dict(self, value, depth): indent = self.base_indent + depth if depth > 2 or self.no_pretty_level > 0: return repr(value) else: items = [ "\n" + self.sp * (indent + 1) + repr(key) + ': ' + self.format(value[key], depth + 1) for key in value ] return '{%s}' % (','.join(items) + "\n" + self.sp * indent) def format_list(self, value, depth): indent = self.base_indent + depth if depth > 2 or self.no_pretty_level > 0 or len(value) == 0: return repr(value) else: # Big array, don't pretty-print inner if len(value) > 3: self.no_pretty_level += 1 items = [ "\n" + self.sp * (indent + 1) + self.format(item, depth + 1) for item in value ] if len(value) > 3: self.no_pretty_level -= 1 return '[%s]' % (','.join(items) + "\n" + self.sp * indent)dataiku-api-client-python-11.4.0/dataikuapi/dss/utils.py000066400000000000000000000416661440464736400232310ustar00rootroot00000000000000class DSSDatasetSelectionBuilder(object): """Builder for a "dataset selection". In DSS, a dataset selection is used to select a part of a dataset for processing. Depending on the location where it is used, a selection can include: * Sampling * Filtering by partitions (for partitioned datasets) * Filtering by an expression * Selection of columns * Ordering Please see the sampling documentation of DSS for a detailed explanation of the sampling methods. """ def __init__(self): self.selection = {} def build(self): """ :returns: the built selection dict :rtype: dict """ return self.selection def with_head_sampling(self, limit): """ Sets the sampling to 'first records' mode :param int limit: Maximum number of rows in the sample """ self.selection["samplingMethod"] = "HEAD_SEQUENTIAL" self.selection["maxRecords"] = limit return self def with_all_data_sampling(self): """Sets the sampling to 'no sampling, all data' mode""" self.selection["samplingMethod"] = "FULL" return self def with_random_fixed_nb_sampling(self, nb): """ Sets the sampling to 'Random sampling, fixed number of records' mode :param int nb: Maximum number of rows in the sample """ self.selection["samplingMethod"] = "RANDOM_FIXED_NB" self.selection["maxRecords"] = nb return self def with_selected_partitions(self, ids): """ Sets partition filtering on the given partition identifiers. .. warning:: The dataset to select must be partitioned. :param list ids: list of selected partitions """ self.selection["partitionSelectionMethod"] = "SELECTED" self.selection["selectedPartitions"] = ids return self class DSSComputedColumn(object): @staticmethod def formula(name, formula, type="double"): """ Create a computed column with a formula. :param string name: a name for the computed column :param string formula: formula to compute values, using the `GREL language `_ :param string type: name of a DSS type for the values of the column :return: a computed column as a dict :rtype: dict """ return {"expr": formula, "mode": "GREL", "name": name, "type": type} import sys if sys.version_info > (3,4): from enum import Enum else: class Enum(object): pass class DSSFilterOperator(Enum): EMPTY_ARRAY = "empty array" """ Test if an array is empty. """ NOT_EMPTY_ARRAY = "not empty array" """ Test if an array is not empty. """ CONTAINS_ARRAY = "array contains" """ Test if an array contains a value. """ NOT_EMPTY = "not empty" """ Test if a value is not empty and not null. """ EMPTY = "is empty" """ Test if a value is empty or null. """ NOT_EMPTY_STRING = "not empty string" """ Test if a string is not empty. """ EMPTY_STRING = "empty string" """ Test if a string is empty. """ IS_TRUE = "true" """ Test if a boolean is true. """ IS_FALSE = "false" """ Test if a boolean is false. """ EQUALS_STRING = "== [string]" """ Test if a string is equal to a given value. """ EQUALS_CASE_INSENSITIVE_STRING = "== [string]i" """ Test if a string is equal to a given value, ignoring case. """ NOT_EQUALS_STRING = "!= [string]" """ Test if a string is not equal to a given value. """ SAME = "== [NaNcolumn]" """ Test if two columns have the same value when formatted to string. """ DIFFERENT = "!= [NaNcolumn]" """ Test if two columns have different values when formatted to string. """ EQUALS_NUMBER = "== [number]" """ Test if a number is equal to a given value. """ NOT_EQUALS_NUMBER = "!= [number]" """ Test if a number is not equal to a given value. """ GREATER_NUMBER = "> [number]" """ Test if a number is greater than a given value. """ LESS_NUMBER = "< [number]" """ Test if a number is less than a given value. """ GREATER_OR_EQUAL_NUMBER = ">= [number]" """ Test if a number is greater or equal to a given value. """ LESS_OR_EQUAL_NUMBER = "<= [number]" """ Test if a number is less or equal to a given value. """ EQUALS_DATE = "== [date]" """ Test if a date/time is equal to a given date/time (rounded). """ GREATER_DATE = "> [date]" """ Test if a date/time is greater than a given date/time. """ GREATER_OR_EQUAL_DATE = ">= [date]" """ Test if a date/time is greater or equal than a given date/time. """ LESS_DATE = "< [date]" """ Test if a date/time is less than a given date/time. """ LESS_OR_EQUAL_DATE = "<= [date]" """ Test if a date/time is less or equal than a given date/time. """ BETWEEN_DATE = ">< [date]" """ Test if a date/time is between two given date/times. """ EQUALS_COL = "== [column]" """ Test if two columns have the same (typed) value. """ NOT_EQUALS_COL = "!= [column]" """ Test if two columns have different (typed) values. """ GREATER_COL = "> [column]" """ Test if one column is greater than another. """ LESS_COL = "< [column]" """ Test if one column is less than another. """ GREATER_OR_EQUAL_COL = ">= [column]" """ Test if one column is greater or equal than another. """ LESS_OR_EQUAL_COL = "<= [column]" """ Test if one column is less or equal than another. """ CONTAINS_STRING = "contains" """ Test if a column contains a given string. """ REGEX = "regex" """ Test if a column matches a regular expression. """ class DSSFilter(object): """ Helper class to build filter objects for use in visual recipes. """ @staticmethod def of_single_condition(column, operator, string=None, num=None, date=None, time=None, date2=None, time2=None, unit=None): """ Create a simple filter on a column. Which of the 'string', 'num', 'date', 'time', 'date2' and 'time2' parameter holds the literal to filter against depends on the filter operator. :param string column: name of a column to filter (left operand) :param string operator: type of filter applied to the column, one of the values in the :class:`DSSFilterOperator` enum :param string string: string literal for the right operand :param string num: numeric literal for the right operand :param string date: date part literal for the right operand :param string time: time part literal for the right operand :param string date2: date part literal for the right operand of BETWEEN_DATE :param string time2: time part literal for the right operand of BETWEEN_DATE :param string unit: date/time rounding for date operations. Possible values are YEAR, MONTH, WEEK, DAY, HOUR, MINUTE, SECOND """ return { "enabled": True, "uiData": { 'conditions': [DSSFilter.condition(column, operator, string, num, date, time, date2, time2, unit)], "mode": "&&" } } @staticmethod def of_and_conditions(conditions): """ Create a filter as an intersection of conditions. The resulting filter keeps rows that match all the conditions in the list. Conditions are for example the output of :meth:`condition()`. :param list conditions: a list of conditions :return: a filter, as a dict :rtype: dict """ return { "enabled": True, "uiData": { 'conditions': conditions, "mode": "&&" } } @staticmethod def of_or_conditions(conditions): """ Create a filter as an union of conditions. The resulting filter keeps rows that match any of the conditions in the list. Conditions are for example the output of :meth:`condition()`. :param list conditions: a list of conditions :return: a filter, as a dict :rtype: dict """ return { "enabled": True, "uiData": { 'conditions': conditions, "mode": "||" } } @staticmethod def of_formula(formula): """ Create a filter that applies a GREL formula. The resulting filter evaluates the formula and keeps rows for which the formula returns a True value. :param string formula: a `GREL formula `_ :return: a filter, as a dict :rtype: dict """ return { "enabled": True, "uiData": { "mode": "CUSTOM" }, "expression": formula } @staticmethod def of_sql_expression(sql_expression): """ Create a filter that applies a SQL expression. The resulting filter evaluates the sql expression and keeps rows for which the sql expression returns a True value. :param string sql_expression: a SQL expression :return: a filter, as a dict :rtype: dict """ return { "enabled": True, "uiData": { "mode": "SQL" }, "expression": sql_expression } @staticmethod def condition(column, operator, string=None, num=None, date=None, time=None, date2=None, time2=None, unit=None): """ Create a condition on a column for a filter. Which of the 'string', 'num', 'date', 'time', 'date2' and 'time2' parameter holds the literal to filter against depends on the filter operator. :param string column: name of a column to filter (left operand) :param string operator: type of filter applied to the column, one of the values in the :class:`DSSFilterOperator` enum :param string string: string literal for the right operand :param string num: numeric literal for the right operand :param string date: date part literal for the right operand :param string time: time part literal for the right operand :param string date2: date part literal for the right operand of BETWEEN_DATE :param string time2: time part literal for the right operand of BETWEEN_DATE :param string unit: date/time rounding for date operations. Possible values are YEAR, MONTH, WEEK, DAY, HOUR, MINUTE, SECOND """ if isinstance(operator, DSSFilterOperator): operator = operator.value cond = { "input": column, "operator": operator } if string is not None: cond["string"] = string if num is not None: cond["num"] = num if date is not None: cond["date"] = date if time is not None: cond["time"] = time if date2 is not None: cond["date2"] = date2 if time2 is not None: cond["time2"] = time2 if unit is not None: cond["unit"] = unit return cond class DSSFilterBuilder(object): """ Builder for a "filter". In DSS, a filter is used to define a subset of rows for processing. """ def __init__(self): self.filter = {"enabled": False, "distinct": False, "expression": None, "uiData": {"mode": "CUSTOM"}} def build(self): """ :returns: the built filter :rtype: dict """ return self.filter def with_distinct(self): """Sets the filter to deduplicate""" self.filter["distinct"] = True return self def with_formula(self, expression): """ Sets the formula (DSS formula) used to filter rows :param str expression: the DSS formula """ self.filter["enabled"] = True self.filter["expression"] = expression self.filter["uiData"]["mode"] = "CUSTOM" return self class AnyLoc(object): def __init__(self, project_key, object_id): self.project_key = project_key self.object_id = object_id def __eq__(self, obj): return isinstance(obj, AnyLoc) and obj.project_key == self.project_key and obj.object_id == self.object_id @staticmethod def from_ref(context_project_key, ref): if ref.find(".") >= 0: elts = ref.split(".") return AnyLoc(elts[0], elts[1]) else: return AnyLoc(context_project_key, ref) @staticmethod def from_full(ref): if ref.find(".") >= 0: elts = ref.split(".") return AnyLoc(elts[0], elts[1]) else: raise Exception("Cannot parse object id, it's not a full id") class DSSTaggableObjectListItem(dict): """An item in a list of taggable objects. Do not instantiate this class""" def __init__(self, data): super(DSSTaggableObjectListItem, self).__init__(data) self._data = data @property def tags(self): return self._data["tags"] class DSSTaggableObjectSettings(object): def __init__(self, taggable_object_data): self._tod = taggable_object_data @property def tags(self): """The tags of the object, as a list of strings""" return self._tod["tags"] @tags.setter def tags(self, tags): self._tod["tags"] = tags @property def description(self): """The description of the object as a string""" return self._tod.get("description", None) @description.setter def description(self, description): self._tod["description"] = description @property def short_description(self): """The short description of the object as a string""" return self._tod.get("shortDesc", None) @short_description.setter def short_description(self, short_description): self._tod["shortDesc"] = short_description @property def custom_fields(self): """The custom fields of the object as a dict. Returns None if there are no custom fields""" return self._tod.get("customFields", None) @custom_fields.setter def custom_fields(self, custom_fields): self._tod["customFields"] = custom_fields class DSSInfoMessages(object): """ Contains a list of :class:`dataikuapi.dss.utils.DSSInfoMessage`. .. important:: Do not instantiate this class. """ def __init__(self, data): self._data = data def __repr__(self): return "DSSInfoMessages(has_messages: {}, max_severity: {}, has_error: {}, has_warning: {}, has_success: {}, messages: {})".format( self.has_messages, self.max_severity, self.has_error, self.has_warning, self.has_success, "[...]" if self.has_messages else "[]" ) @property def messages(self): """The messages as a list of :class:`dataikuapi.dss.utils.DSSInfoMessage`""" return [DSSInfoMessage(message) for message in self._data.get("messages", [])] @property def has_messages(self): """`True` if there is any message""" return self._data.get("anyMessage", False) @property def has_error(self): """`True` if there is any error message""" return self._data.get("error", False) @property def max_severity(self): """The max severity of the messages""" return self._data.get("maxSeverity", None) @property def has_success(self): """`True` if there is any success message""" return self._data.get("success", False) @property def has_warning(self): """`True` if there is any warning message""" return self._data.get("warning", False) class DSSInfoMessage(object): """ A message with a code, a title, a severity and a content. .. important:: Do not instantiate this class. """ def __init__(self, data): self._data = data def __str__(self): return "{severity} - {title} - {details}".format( severity=self.severity, title=self.title, details=self.details ) @property def severity(self): """The severity of the message""" return self._data.get("severity", None) @property def code(self): """The code of the message""" return self._data.get("code", None) @property def details(self): """The details of the message""" return self._data.get("details", None) @property def title(self): """The title of the message""" return self._data.get("title", None) @property def message(self): """The full message""" return self._data.get("message", None) dataiku-api-client-python-11.4.0/dataikuapi/dss/wiki.py000066400000000000000000000401231440464736400230170ustar00rootroot00000000000000from .discussion import DSSObjectDiscussions from ..utils import DataikuException import sys import copy import re if sys.version_info >= (3,0): import urllib.parse dku_quote_fn = urllib.parse.quote else: import urllib dku_quote_fn = urllib.quote class DSSWiki(object): """ A handle to manage the wiki of a project .. important:: Do not instantiate this class directly, instead use :meth:`dataikuapi.dss.project.DSSProject.get_wiki` """ def __init__(self, client, project_key): self.client = client self.project_key = project_key def get_settings(self): """ Get wiki settings :returns: a handle to manage the wiki settings (taxonomy, home article) :rtype: :class:`dataikuapi.dss.wiki.DSSWikiSettings` """ return DSSWikiSettings(self.client, self.project_key, self.client._perform_json("GET", "/projects/%s/wiki/" % (self.project_key))) def get_article(self, article_id_or_name): """ Get a wiki article :param str article_id_or_name: reference to the article, it can be its ID or its name :returns: a handle to manage the Article :rtype: :class:`dataikuapi.dss.wiki.DSSWikiArticle` """ return DSSWikiArticle(self.client, self.project_key, article_id_or_name) def __flatten_taxonomy__(self, taxonomy): """ Private recursive method to get the flatten list of article IDs from the taxonomy :param list taxonomy: :returns: list of articles :rtype: list of :class:`dataikuapi.dss.wiki.DSSWikiArticle` """ article_list = [] for article in taxonomy: article_list.append(self.get_article(article['id'])) article_list += self.__flatten_taxonomy__(article['children']) return article_list def list_articles(self): """ Get a list of all the articles in form of :class:`dataikuapi.dss.wiki.DSSWikiArticle` objects :returns: list of articles :rtype: list of :class:`dataikuapi.dss.wiki.DSSWikiArticle` """ return self.__flatten_taxonomy__(self.get_settings().get_taxonomy()) def create_article(self, article_name, parent_id=None, content=None): """ Create a wiki article and return a handle to interact with it. :param str article_name: the article name :param str parent_id: the parent article ID (or None if the article has to be at root level, defaults to **None**) :param str content: the article content (defaults to **None**) :returns: the created article :rtype: :class:`dataikuapi.dss.wiki.DSSWikiArticle` """ body = { "projectKey": self.project_key, "name": article_name, "parent": parent_id } result = self.client._perform_json("POST", "/projects/%s/wiki/" % (self.project_key), body=body) article = DSSWikiArticle(self.client, self.project_key, result['article']['id']) # set article content if given if content is not None: article_data = article.get_data() article_data.set_body(content) article_data.save() return article def get_export_stream(self, paper_size="A4", export_attachment=False): """ Download the whole wiki of the project in PDF format as a binary stream. .. warning:: You need to close the stream after download. Failure to do so will result in the DSSClient becoming unusable. :param str paper_size: the format of the exported page, can be one of 'A4', 'A3', 'US_LETTER' or 'LEDGER' (defaults to **A4**) :param bool export_attachment: export the attachments of the article(s) in addition to the pdf in a zip file (defaults to **False**) :returns: the exported pdf or zip file as a stream """ body = { "paperSize": paper_size, "exportAttachment": export_attachment } return self.client._perform_raw("POST", "/projects/%s/wiki/actions/export" % (self.project_key), body=body) def export_to_file(self, path, paper_size="A4", export_attachment=False): """ Download the whole wiki of the project in PDF format into the given output file. :param str path: the path of the file where the pdf or zip file will be downloaded :param str paper_size: the format of the exported page, can be one of 'A4', 'A3', 'US_LETTER' or 'LEDGER' (defaults to **A4**) :param bool export_attachment: export the attachments of the article(s) in addition to the pdf in a zip file (defaults to **False**) """ with self.get_export_stream(paper_size=paper_size, export_attachment=export_attachment) as stream: with open(path, 'wb') as f: for chunk in stream.iter_content(chunk_size=10000): if chunk: f.write(chunk) f.flush() class DSSWikiSettings(object): """ Global settings for the wiki, including taxonomy. Call save() to save """ def __init__(self, client, project_key, settings): """Do not instantiate this class directly, instead use :meth:`dataikuapi.dss.wiki.DSSWiki.get_settings`""" self.client = client self.project_key = project_key self.settings = settings def get_taxonomy(self): """ Get the taxonomy. The taxonomy is an array listing at top level the root article IDs and their children in a tree format. Every existing article of the wiki has to be in the taxonomy once and only once. For instance: .. code-block:: python [ { 'id': 'article1', 'children': [] }, { 'id': 'article2', 'children': [ { 'id': 'article3', 'children': [] } ] } ] .. note :: Note that this is a direct reference, not a copy, so modifications to the returned object will be reflected when saving :returns: The taxonomy :rtype: list """ return self.settings["taxonomy"] if "taxonomy" in self.settings else [] def __retrieve_article_in_taxonomy__(self, taxonomy, article_id, remove=False): """ Private recursive method that get the sub tree structure from the taxonomy for a specific article :param list taxonomy: the current level of taxonomy :param str article_id: the article to retrieve :param bool remove: either remove the sub tree structure or not (defaults to **False**) :returns: the sub tree structure at a specific article level :rtype: dict """ idx = 0 for tax_article in taxonomy: if tax_article["id"] == article_id: ret = taxonomy.pop(idx) if remove else taxonomy[idx] return ret children_ret = self.__retrieve_article_in_taxonomy__(tax_article["children"], article_id, remove) if children_ret is not None: return children_ret idx += 1 return None def move_article_in_taxonomy(self, article_id, parent_article_id=None): """ An helper to update the taxonomy by moving an article with its children as a child of another article :param str article_id: the main article ID :param str parent_article_id: the new parent article ID or None for root level (defaults to **None**) """ old_taxonomy = copy.deepcopy(self.settings["taxonomy"]) tax_article = self.__retrieve_article_in_taxonomy__(self.settings["taxonomy"], article_id, True) if tax_article is None: raise DataikuException("Article not found: %s" % (article_id)) if parent_article_id is None: self.settings["taxonomy"].append(tax_article) else: tax_parent_article = self.__retrieve_article_in_taxonomy__(self.settings["taxonomy"], parent_article_id, False) if tax_parent_article is None: self.settings["taxonomy"] = old_taxonomy raise DataikuException("Parent article not found (or is one of the article descendants): %s" % (parent_article_id)) tax_parent_article["children"].append(tax_article) def set_taxonomy(self, taxonomy): """ Set the taxonomy :param list taxonomy: the taxonomy """ self.settings["taxonomy"] = taxonomy def get_home_article_id(self): """ Get the home article ID :returns: The home article ID :rtype: str """ return self.settings["homeArticleId"] if "homeArticleId" in self.settings else None def set_home_article_id(self, home_article_id): """ Set the home article ID :param str home_article_id: the home article ID """ self.settings["homeArticleId"] = home_article_id def save(self): """ Save the current settings to the backend """ self.settings = self.client._perform_json("PUT", "/projects/%s/wiki/" % (self.project_key), body=self.settings) class DSSWikiArticle(object): """ A handle to manage an article """ def __init__(self, client, project_key, article_id_or_name): """Do not instantiate this class directly, instead use :meth:`dataikuapi.dss.wiki.DSSWiki.get_article`""" self.client = client self.project_key = project_key # Retrieve the real article id article_data = self.client._perform_json("GET", "/projects/%s/wiki/%s" % (project_key, article_id_or_name)) self.article_id = article_data["article"]['id'] # encode in UTF-8 if its python2 and unicode if sys.version_info < (3,0) and isinstance(self.article_id, unicode): self.article_id = self.article_id.encode('utf-8') def get_data(self): """ Get article data handle :returns: the article data handle :rtype: :class:`dataikuapi.dss.wiki.DSSWikiArticleData` """ article_data = self.client._perform_json("GET", "/projects/%s/wiki/%s" % (self.project_key, self.article_id)) return DSSWikiArticleData(self.client, self.project_key, self.article_id, article_data) def upload_attachement(self, fp, filename): """ Upload and attach a file to the article. .. note :: Note that the type of file will be determined by the filename extension :param file fp: A file-like object that represents the upload file :param str filename: The attachement filename """ clean_filename = re.sub(r'[^A-Za-z0-9 ._-]+', '', filename) self.client._perform_json("POST", "/projects/%s/wiki/%s/upload" % (self.project_key, dku_quote_fn(self.article_id)), files={"file":(clean_filename, fp)}) def get_uploaded_file(self, upload_id): """ Download an attachment of the article .. warning:: You need to close the stream after download. Failure to do so will result in the DSSClient becoming unusable. :param str upload_id: The attachement upload id :returns: The attachment file as a stream :rtype: :class:`requests.Response` """ return self.client._perform_raw("GET", "/projects/%s/wiki/%s/uploads/%s" % (self.project_key, self.article_id, upload_id)) def get_export_stream(self, paper_size="A4", export_children=False, export_attachment=False): """ Download the article in PDF format as a binary stream. .. warning:: You need to close the stream after download. Failure to do so will result in the DSSClient becoming unusable. :param str paper_size: the format of the exported page, can be one of 'A4', 'A3', 'US_LETTER' or 'LEDGER' (defaults to **A4**) :param bool export_children: export the children of the article in the pdf (defaults to **False**) :param bool export_attachment: export the attachments of the article(s) in addition to the pdf in a zip file (defaults to **False**) :returns: the exported pdf or zip file as a stream :rtype: :class:`requests.Response` """ body = { "paperSize": paper_size, "exportChildren": export_children, "exportAttachment": export_attachment } return self.client._perform_raw("POST", "/projects/%s/wiki/%s/actions/export" % (self.project_key, self.article_id), body=body) def export_to_file(self, path, paper_size="A4", export_children=False, export_attachment=False): """ Download the article in PDF format into the given output file. :param str path: the path of the file where the pdf or zip file will be downloaded :param str paper_size: the format of the exported page, can be one of 'A4', 'A3', 'US_LETTER' or 'LEDGER' (defaults to **A4**) :param bool export_children: export the children of the article in the pdf (defaults to **False**) :param bool export_attachment: export the attachments of the article(s) in addition to the pdf in a zip file (defaults to **False**) """ with self.get_export_stream(paper_size=paper_size, export_children=export_children, export_attachment=export_attachment) as stream: with open(path, 'wb') as f: for chunk in stream.iter_content(chunk_size=10000): if chunk: f.write(chunk) f.flush() def delete(self): """ Delete the article """ self.client._perform_empty("DELETE", "/projects/%s/wiki/%s" % (self.project_key, dku_quote_fn(self.article_id))) def get_object_discussions(self): """ Get a handle to manage discussions on the article :returns: the handle to manage discussions :rtype: :class:`dataikuapi.dss.wiki.DSSObjectDiscussions` """ return DSSObjectDiscussions(self.client, self.project_key, "ARTICLE", self.article_id) class DSSWikiArticleData(object): """ A handle to manage an article """ def __init__(self, client, project_key, article_id, article_data): """Do not instantiate this class directly, instead use :meth:`dataikuapi.dss.wiki.DSSWikiArticle.get_data`""" self.client = client self.project_key = project_key self.article_id = article_id # don't need to check unicode here (already done in DSSWikiArticle) self.article_data = article_data def get_body(self): """ Get the markdown body as string :returns: the article body :rtype: str """ return self.article_data["payload"] if "payload" in self.article_data else None def set_body(self, content): """ Set the markdown body :param str content: the article content """ self.article_data["payload"] = content def get_metadata(self): """ Get the article metadata .. note :: Note that this is a direct reference, not a copy, so modifications to the returned object will be reflected when saving :returns: the article metadata :rtype: dict """ return self.article_data["article"] if "article" in self.article_data else None def set_metadata(self, metadata): """ Set the article metadata :param dict metadata: the article metadata """ self.article_data["article"] = metadata def get_name(self): """ Get the article name :returns: the article name :rtype: str """ return self.article_data["article"]["name"] def set_name(self, name): """ Set the article name :param str name: the article name """ self.article_data["article"]["name"] = name def save(self): """ Save the current article data to the backend. """ self.article_data = self.client._perform_json("PUT", "/projects/%s/wiki/%s" % (self.project_key, dku_quote_fn(self.article_id)), body=self.article_data) dataiku-api-client-python-11.4.0/dataikuapi/dss_plugin_mlflow/000077500000000000000000000000001440464736400244405ustar00rootroot00000000000000dataiku-api-client-python-11.4.0/dataikuapi/dss_plugin_mlflow/__init__.py000066400000000000000000000000371440464736400265510ustar00rootroot00000000000000from .utils import MLflowHandledataiku-api-client-python-11.4.0/dataikuapi/dss_plugin_mlflow/artifact_repository.py000066400000000000000000000164151440464736400311150ustar00rootroot00000000000000import os import tempfile import urllib import re import sys from dataikuapi import DSSClient if sys.version_info > (3, 0): # MLflow only work for python3 (in > 1.18.0) from pathlib import PurePosixPath, Path def parse_dss_managed_folder_uri(uri): parsed = urllib.parse.urlparse(uri) if parsed.scheme != "dss-managed-folder": raise Exception("Not a DSS Managed Folder URI: %s" % uri) pattern = re.compile("^(\w+\.)?\w{8}") if not parsed.netloc or not pattern.match(parsed.netloc): raise Exception("Could not find a managed folder id in URI: %s" % uri) return parsed class PluginDSSManagedFolderArtifactRepository: def __init__(self, artifact_uri): if os.environ.get("DSS_MLFLOW_APIKEY") is not None: self.client = DSSClient( os.environ.get("DSS_MLFLOW_HOST"), api_key=os.environ.get("DSS_MLFLOW_APIKEY") ) else: self.client = DSSClient( os.environ.get("DSS_MLFLOW_HOST"), internal_ticket=os.environ.get("DSS_MLFLOW_INTERNAL_TICKET") ) self.project = self.client.get_project(os.environ.get("DSS_MLFLOW_PROJECTKEY")) parsed_uri = parse_dss_managed_folder_uri(artifact_uri) self.managed_folder = self.__get_managed_folder(parsed_uri.netloc) self.base_artifact_path = PurePosixPath(parsed_uri.path) def __get_managed_folder(self, managed_folder_smart_id): chunks = managed_folder_smart_id.split('.') if len(chunks) == 1: return self.project.get_managed_folder(chunks[0]) elif len(chunks) == 2: project = self.client.get_project(chunks[0]) return project.get_managed_folder(chunks[1]) else: raise Exception("Invalid managed folder id: %s" % managed_folder_smart_id) def log_artifact(self, local_file, artifact_path=None): """ Log a local file as an artifact, optionally taking an ``artifact_path`` to place it in within the run's artifacts. Run artifacts can be organized into directories, so you can place the artifact in a directory this way. :param local_file: Path to artifact to log :param artifact_path: Directory within the run's artifact directory in which to log the artifact. """ path = self.base_artifact_path if artifact_path is not None: path /= artifact_path self.managed_folder.put_file(str(path / os.path.basename(local_file)), open(local_file, "rb")) def log_artifacts(self, local_dir, artifact_path=None): """ Log the files in the specified local directory as artifacts, optionally taking an ``artifact_path`` to place them in within the run's artifacts. :param local_dir: Directory of local artifacts to log :param artifact_path: Directory within the run's artifact directory in which to log the artifacts """ path = self.base_artifact_path if artifact_path is not None: path /= artifact_path self.managed_folder.upload_folder(str(path), local_dir) def list_artifacts(self, path=""): """ Return all the artifacts for this run_id directly under path. If path is a file, returns an empty list. Will error if path is neither a file nor directory. :param path: Relative source path that contains desired artifacts :return: List of artifacts as FileInfo listed directly under path. """ path = self.base_artifact_path / path files = [PurePosixPath(x["path"]) for x in self.managed_folder.list_contents().get("items", [])] return [file.relative_to(path) for file in files if path in file.parents] def download_artifacts(self, artifact_path, dst_path=None): """ Download an artifact file or directory to a local directory if applicable, and return a local path for it. The caller is responsible for managing the lifecycle of the downloaded artifacts. :param artifact_path: Relative source path to the desired artifacts. :param dst_path: Absolute path of the local filesystem destination directory to which to download the specified artifacts. This directory must already exist. If unspecified, the artifacts will either be downloaded to a new uniquely-named directory on the local filesystem or will be returned directly in the case of the LocalArtifactRepository. :return: Absolute path of the local filesystem location containing the desired artifacts. """ from mlflow.exceptions import MlflowException from mlflow.protos.databricks_pb2 import INVALID_PARAMETER_VALUE, RESOURCE_DOES_NOT_EXIST if dst_path is None: dst_path = tempfile.mkdtemp() dst_path = Path(dst_path).absolute() if not dst_path.exists(): raise MlflowException( message=("The destination path for downloaded artifacts does not" " exist! Destination path: {dst_path}".format(dst_path=dst_path)), error_code=RESOURCE_DOES_NOT_EXIST, ) elif not dst_path.is_dir(): raise MlflowException( message=("The destination path for downloaded artifacts must be a directory!" " Destination path: {dst_path}".format(dst_path=dst_path)), error_code=INVALID_PARAMETER_VALUE, ) for path in self.list_artifacts(artifact_path): local_path = dst_path / path local_path.parent.mkdir(exist_ok=True) self._download_file(artifact_path / path, local_path) return dst_path def _download_file(self, artifact_path, local_path): """ Download the file at the specified relative remote path and saves it at the specified local path. :param artifact_path: Source path to the remote file, relative to the root directory of the artifact repository. :param local_path: The path to which to save the downloaded file. """ full_path = self.base_artifact_path / artifact_path with self.managed_folder.get_file(str(full_path)) as remote_file: with open(local_path, "wb") as local_file: for line in remote_file: local_file.write(line) def delete_artifacts(self, artifact_path=None): """ Delete the artifacts at the specified location. Supports the deletion of a single file or of a directory. Deletion of a directory is recursive. :param artifact_path: Path of the artifact to delete """ path = ( self.base_artifact_path / artifact_path if artifact_path else self.base_artifact_path ) self.managed_folder.delete_file(str(path)) def verify_artifact_path(artifact_path): from mlflow.exceptions import MlflowException from mlflow.utils.validation import path_not_unique, bad_path_message if artifact_path and path_not_unique(artifact_path): raise MlflowException( "Invalid artifact path: '%s'. %s" % (artifact_path, bad_path_message(artifact_path)) ) dataiku-api-client-python-11.4.0/dataikuapi/dss_plugin_mlflow/header_provider.py000066400000000000000000000011211440464736400301470ustar00rootroot00000000000000import os class PluginDSSHeaderProvider: def in_context(self): env_variables = {"DSS_MLFLOW_HEADER", "DSS_MLFLOW_TOKEN", "DSS_MLFLOW_PROJECTKEY"} if len(env_variables.difference(set(os.environ))) == 0: return True def request_headers(self): headers = { os.environ.get("DSS_MLFLOW_HEADER"): os.environ.get("DSS_MLFLOW_TOKEN"), "x-dku-mlflow-project-key": os.environ.get("DSS_MLFLOW_PROJECTKEY"), "x-dku-mlflow-managed-folder-id": os.environ.get("DSS_MLFLOW_MANAGED_FOLDER_ID"), } return headers dataiku-api-client-python-11.4.0/dataikuapi/dss_plugin_mlflow/utils.py000066400000000000000000000116351440464736400261600ustar00rootroot00000000000000from ..utils import DataikuException from ..dss.managedfolder import DSSManagedFolder import logging import os import sys import tempfile import shutil from base64 import b64encode class MLflowHandle: def __init__(self, client, project, managed_folder, host=None): """ Add the MLflow-plugin parts of dataikuapi to MLflow local setup. This method deals with 1. importing dynamically the DSS MLflow plugin: MLflow uses entrypoints==0.3 to load entrypoints from plugins at import time. We add dss-mlflow-plugin entrypoints dynamically by adding them in sys.path at call time. 2. Setup the correct environment variables to setup the plugin to work with DSS backend as the tracking backend and enable using DSS managed folder as artifact location. """ self.project = project self.project_key = project.project_key self.client = client self.mlflow_env = {} # Load DSS as the plugin self.tempdir = tempfile.mkdtemp() plugin_dir = os.path.join(self.tempdir, "dss-plugin-mlflow.egg-info") if not os.path.isdir(plugin_dir): os.makedirs(plugin_dir) with open(os.path.join(plugin_dir, "entry_points.txt"), "w") as f: f.write( "[mlflow.request_header_provider]\n" "unused=dataikuapi.dss_plugin_mlflow.header_provider:PluginDSSHeaderProvider\n" "[mlflow.artifact_repository]\n" "dss-managed-folder=dataikuapi.dss_plugin_mlflow.artifact_repository:PluginDSSManagedFolderArtifactRepository\n" ) sys.path.insert(0, self.tempdir) # Reload the artifact_repository_registry in case MLflow was imported beforehand if sys.version_info > (3, 4): from importlib import reload import mlflow reload(mlflow.store.artifact.artifact_repository_registry) mlflow.set_tracking_uri(None) # if user has changed tracking backend manually before mlflow.end_run() # if user already created a run with another tracking backend # Setup authentication if client._session.auth is not None: self.mlflow_env.update({ "DSS_MLFLOW_HEADER": "Authorization", "DSS_MLFLOW_TOKEN": "Basic {}".format( b64encode("{}:".format(self.client._session.auth.username).encode("utf-8")).decode("utf-8")), "DSS_MLFLOW_APIKEY": self.client.api_key }) elif client.internal_ticket: self.mlflow_env.update({ "DSS_MLFLOW_HEADER": "X-DKU-APITicket", "DSS_MLFLOW_TOKEN": self.client.internal_ticket, "DSS_MLFLOW_INTERNAL_TICKET": self.client.internal_ticket }) if not client._session.verify: self.mlflow_env.update({"MLFLOW_TRACKING_INSECURE_TLS": "true"}) elif isinstance(client._session.verify, str): self.mlflow_env.update({"MLFLOW_TRACKING_SERVER_CERT_PATH": client._session.verify}) mf_full_id = None if isinstance(managed_folder, DSSManagedFolder): mf_full_id = managed_folder.project.project_key + "." + managed_folder.id elif isinstance(managed_folder, str): mf_full_id = managed_folder else: try: from dataiku import Folder if isinstance(managed_folder, Folder): mf_full_id = managed_folder.name except ImportError: pass if not mf_full_id: raise TypeError('Type of managed_folder must be "str", "DSSManagedFolder" or "dataiku.Folder".') if "." not in mf_full_id: mf_full_id = self.project_key + "." + mf_full_id mf_project = mf_full_id.split(".")[0] mf_id = mf_full_id.split(".")[1] try: client.get_project(mf_project).get_managed_folder(mf_id).get_definition() except DataikuException as e: if "NotFoundException" in str(e): logging.error('The managed folder "%s" does not exist, please create it in your project flow before running this command.' % (mf_full_id)) raise # Set host, tracking URI, project key and managed_folder_id self.mlflow_env.update({ "DSS_MLFLOW_PROJECTKEY": self.project_key, "MLFLOW_TRACKING_URI": self.client.host + "/dip/publicapi" if host is None else host, "DSS_MLFLOW_HOST": self.client.host, "DSS_MLFLOW_MANAGED_FOLDER_ID": mf_full_id }) os.environ.update(self.mlflow_env) def clear(self): shutil.rmtree(self.tempdir) for variable in self.mlflow_env: os.environ.pop(variable, None) def import_mlflow(self): import mlflow return mlflow def __enter__(self): return self.import_mlflow() def __exit__(self, exc_type, exc_value, traceback): self.clear() dataiku-api-client-python-11.4.0/dataikuapi/dssclient.py000066400000000000000000001713411440464736400232620ustar00rootroot00000000000000import json, warnings from requests import Session from requests import exceptions from requests.auth import HTTPBasicAuth from .dss.feature_store import DSSFeatureStore from .dss.notebook import DSSNotebook from .dss.future import DSSFuture from .dss.projectfolder import DSSProjectFolder from .dss.project import DSSProject from .dss.app import DSSApp, DSSAppListItem from .dss.plugin import DSSPlugin from .dss.admin import DSSGlobalApiKeyListItem, DSSPersonalApiKeyListItem, DSSUser, DSSUserActivity, DSSOwnUser, DSSGroup, DSSConnection, DSSConnectionListItem, DSSGeneralSettings, DSSCodeEnv, DSSGlobalApiKey, DSSCluster, DSSCodeStudioTemplate, DSSCodeStudioTemplateListItem, DSSGlobalUsageSummary, DSSInstanceVariables, DSSPersonalApiKey, DSSAuthorizationMatrix from .dss.meaning import DSSMeaning from .dss.sqlquery import DSSSQLQuery from .dss.discussion import DSSObjectDiscussions from .dss.apideployer import DSSAPIDeployer from .dss.projectdeployer import DSSProjectDeployer from .dss.utils import DSSInfoMessages from .dss.workspace import DSSWorkspace import os.path as osp from .utils import DataikuException, dku_basestring_type class DSSClient(object): """Entry point for the DSS API client""" def __init__(self, host, api_key=None, internal_ticket = None, extra_headers = None): """ Instantiate a new DSS API client on the given host with the given API key. API keys can be managed in DSS on the project page or in the global settings. The API key will define which operations are allowed for the client. """ self.api_key = api_key self.internal_ticket = internal_ticket self.host = host self._session = Session() if self.api_key is not None: self._session.auth = HTTPBasicAuth(self.api_key, "") elif self.internal_ticket is not None: self._session.headers.update({"X-DKU-APITicket" : self.internal_ticket}) else: raise ValueError("API Key is required") if extra_headers is not None: self._session.headers.update(extra_headers) ######################################################## # Futures ######################################################## def list_futures(self, as_objects=False, all_users=False): """ List the currently-running long tasks (a.k.a futures) :param boolean as_objects: if True, each returned item will be a :class:`dataikuapi.dss.future.DSSFuture` :param boolean all_users: if True, returns futures for all users (requires admin privileges). Else, only returns futures for the user associated with the current authentication context (if any) :return: list of futures. if as_objects is True, each future in the list is a :class:`dataikuapi.dss.future.DSSFuture`. Else, each future in the list is a dict. Each dict contains at least a 'jobId' field :rtype: list of :class:`dataikuapi.dss.future.DSSFuture` or list of dict """ list = self._perform_json("GET", "/futures/", params={"withScenarios":False, "withNotScenarios":True, 'allUsers' : all_users}) if as_objects: return [DSSFuture(self, state['jobId'], state) for state in list] else: return list def list_running_scenarios(self, all_users=False): """ List the running scenarios :param boolean all_users: if True, returns scenarios for all users (requires admin privileges). Else, only returns scenarios for the user associated with the current authentication context (if any) :return: list of running scenarios, each one as a dict containing at least a "jobId" field for the future hosting the scenario run, and a "payload" field with scenario identifiers :rtype: list of dicts """ return self._perform_json("GET", "/futures/", params={"withScenarios":True, "withNotScenarios":False, 'allUsers' : all_users}) def get_future(self, job_id): """ Get a handle to interact with a specific long task (a.k.a future). This notably allows aborting this future. :param str job_id: the identifier of the desired future (which can be returned by :py:meth:`list_futures` or :py:meth:`list_running_scenarios`) :returns: A handle to interact the future :rtype: :class:`dataikuapi.dss.future.DSSFuture` """ return DSSFuture(self, job_id) ######################################################## # Notebooks ######################################################## def list_running_notebooks(self, as_objects=True): """ List the currently-running Jupyter notebooks :param boolean as_objects: if True, each returned item will be a :class:`dataikuapi.dss.notebook.DSSNotebook` :return: list of notebooks. if as_objects is True, each entry in the list is a :class:`dataikuapi.dss.notebook.DSSNotebook`. Else, each item in the list is a dict which contains at least a "name" field. :rtype: list of :class:`dataikuapi.dss.notebook.DSSNotebook` or list of dict """ notebook_list = self._perform_json("GET", "/admin/notebooks/") if as_objects: return [DSSNotebook(self, notebook['projectKey'], notebook['name'], notebook) for notebook in notebook_list] else: return notebook_list ######################################################## # Project folders ######################################################## def get_root_project_folder(self): """ Get a handle to interact with the root project folder. :returns: A :class:`dataikuapi.dss.projectfolder.DSSProjectFolder` to interact with this project folder """ return self.get_project_folder("ROOT") def get_project_folder(self, project_folder_id): """ Get a handle to interact with a project folder. :param str project_folder_id: the project folder ID of the desired project folder :returns: A :class:`dataikuapi.dss.projectfolder.DSSProjectFolder` to interact with this project folder """ data = self._perform_json("GET", "/project-folders/%s" % project_folder_id) return DSSProjectFolder(self, data) ######################################################## # Projects ######################################################## def list_project_keys(self): """ List the project keys (=project identifiers). :returns: list of project keys identifiers, as strings :rtype: list of strings """ return [x["projectKey"] for x in self._perform_json("GET", "/projects/")] def list_projects(self): """ List the projects :returns: a list of projects, each as a dict. Each dictcontains at least a 'projectKey' field :rtype: list of dicts """ return self._perform_json("GET", "/projects/") def get_project(self, project_key): """ Get a handle to interact with a specific project. :param str project_key: the project key of the desired project :returns: A :class:`dataikuapi.dss.project.DSSProject` to interact with this project """ return DSSProject(self, project_key) def get_default_project(self): """ Get a handle to the current default project, if available (i.e. if dataiku.default_project_key() is valid) """ import dataiku return DSSProject(self, dataiku.default_project_key()) def create_project(self, project_key, name, owner, description=None, settings=None, project_folder_id=None): """ Creates a new project, and return a project handle to interact with it. Note: this call requires an API key with admin rights or the rights to create a project :param str project_key: the identifier to use for the project. Must be globally unique :param str name: the display name for the project. :param str owner: the login of the owner of the project. :param str description: a description for the project. :param dict settings: Initial settings for the project (can be modified later). The exact possible settings are not documented. :param str project_folder_id: the project folder ID in which the project will be created (root project folder if not specified) :returns: A :class:`dataikuapi.dss.project.DSSProject` project handle to interact with this project """ params = {} if project_folder_id is not None: params["projectFolderId"] = project_folder_id resp = self._perform_text( "POST", "/projects/", body={ "projectKey" : project_key, "name" : name, "owner" : owner, "settings" : settings, "description" : description }, params=params) return DSSProject(self, project_key) ######################################################## # Apps ######################################################## def list_apps(self, as_type="listitems"): """ List the apps. :param str as_type: How to return the list. Supported values are "listitems" and "objects" (defaults to **listitems**). :returns: The list of the apps. If "as_type" is "listitems", each one as a :class:`dataikuapi.dss.app.DSSAppListItem`. If "as_type" is "objects", each one as a :class:`dataikuapi.dss.app.DSSApp` :rtype: list """ items = self._perform_json("GET", "/apps/") if as_type == "listitems" or as_type == "listitem": return [DSSAppListItem(self, item) for item in items] elif as_type == "objects" or as_type == "object": return [DSSApp(self, item["appId"]) for item in items] else: raise ValueError("Unknown as_type") def get_app(self, app_id): """ Get a handle to interact with a specific app. .. note:: If a project XXXX is an app template, the identifier of the associated app is PROJECT_XXXX :param str app_id: the id of the desired app :returns: A :class:`dataikuapi.dss.app.DSSApp` to interact with this project """ return DSSApp(self, app_id) ######################################################## # Plugins ######################################################## def list_plugins(self): """ List the installed plugins :returns: list of dict. Each dict contains at least a 'id' field """ return self._perform_json("GET", "/plugins/") def download_plugin_stream(self, plugin_id): """ Download a development plugin, as a binary stream :param str plugin_id: identifier of the plugin to download :param plugin_id: :return: the binary stream """ return self._perform_raw("GET", "/plugins/%s/download" % plugin_id) def download_plugin_to_file(self, plugin_id, path): """ Download a development plugin to a file :param str plugin_id: identifier of the plugin to download :param str path: the path where to download the plugin :return: None """ stream = self.download_plugin_stream(plugin_id) with open(path, 'wb') as f: for chunk in stream.iter_content(chunk_size=10000): if chunk: f.write(chunk) f.flush() def install_plugin_from_archive(self, fp): """ Install a plugin from a plugin archive (as a file object) :param object fp: A file-like object pointing to a plugin archive zip """ files = {'file': fp } self._perform_json("POST", "/plugins/actions/installFromZip", files=files) def install_plugin_from_store(self, plugin_id): """ Install a plugin from the Dataiku plugin store :param str plugin_id: identifier of the plugin to install :return: A :class:`~dataikuapi.dss.future.DSSFuture` representing the install process """ f = self._perform_json("POST", "/plugins/actions/installFromStore", body={ "pluginId": plugin_id }) return DSSFuture.from_resp(self, f) def install_plugin_from_git(self, repository_url, checkout = "master", subpath=None): """ Install a plugin from a Git repository. DSS must be setup to allow access to the repository. :param str repository_url: URL of a Git remote :param str checkout: branch/tag/SHA1 to commit. For example "master" :param str subpath: Optional, path within the repository to use as plugin. Should contain a 'plugin.json' file :return: A :class:`~dataikuapi.dss.future.DSSFuture` representing the install process """ f = self._perform_json("POST", "/plugins/actions/installFromGit", body={ "gitRepositoryUrl": repository_url, "gitCheckout" : checkout, "gitSubpath": subpath }) return DSSFuture.from_resp(self, f) def get_plugin(self, plugin_id): """ Get a handle to interact with a specific plugin :param str plugin_id: the identifier of the desired plugin :returns: A :class:`dataikuapi.dss.project.DSSPlugin` """ return DSSPlugin(self, plugin_id) ######################################################## # SQL queries ######################################################## def sql_query(self, query, connection=None, database=None, dataset_full_name=None, pre_queries=None, post_queries=None, type='sql', extra_conf=None, script_steps=None, script_input_schema=None, script_output_schema=None, script_report_location=None, read_timestamp_without_timezone_as_string=True, read_date_as_string=False, project_key=None): """ Initiate a SQL, Hive or Impala query and get a handle to retrieve the results of the query. Internally, the query is run by DSS. The database to run the query on is specified either by passing a connection name, or by passing a database name, or by passing a dataset full name (whose connection is then used to retrieve the database) :param str query: the query to run :param str connection: the connection on which the query should be run (exclusive of database and dataset_full_name) :param str database: the database on which the query should be run (exclusive of connection and dataset_full_name) :param str dataset_full_name: the dataset on the connection of which the query should be run (exclusive of connection and database) :param list pre_queries: (optional) array of queries to run before the query :param list post_queries: (optional) array of queries to run after the query :param str type: the type of query : either 'sql', 'hive' or 'impala' (default: sql) :param str project_key: The project_key on which the query should be run (especially useful for user isolation/impersonation scenario) :return: a handle on the SQL query :rtype: :class:`dataikuapi.dss.sqlquery.DSSSQLQuery` """ if extra_conf is None: extra_conf = {} return DSSSQLQuery(self, query, connection, database, dataset_full_name, pre_queries, post_queries, type, extra_conf, script_steps, script_input_schema, script_output_schema, script_report_location, read_timestamp_without_timezone_as_string, read_date_as_string, project_key) ######################################################## # Users ######################################################## def list_users(self, as_objects=False): """ List all users setup on the DSS instance Note: this call requires an API key with admin rights :return: A list of users, as a list of :class:`dataikuapi.dss.admin.DSSUser` if as_objects is True, else as a list of dicts :rtype: list of :class:`dataikuapi.dss.admin.DSSUser` or list of dicts """ users = self._perform_json("GET", "/admin/users/") if as_objects: return [DSSUser(self, user["login"]) for user in users] else: return users def get_user(self, login): """ Get a handle to interact with a specific user :param str login: the login of the desired user :return: A :class:`dataikuapi.dss.admin.DSSUser` user handle """ return DSSUser(self, login) def create_user(self, login, password, display_name='', source_type='LOCAL', groups=None, profile='DATA_SCIENTIST'): """ Create a user, and return a handle to interact with it Note: this call requires an API key with admin rights :param str login: the login of the new user :param str password: the password of the new user :param str display_name: the displayed name for the new user :param str source_type: the type of new user. Admissible values are 'LOCAL' or 'LDAP' :param list groups: the names of the groups the new user belongs to (defaults to `[]`) :param str profile: The profile for the new user, can be one of READER, DATA_ANALYST or DATA_SCIENTIST :return: A :class:`dataikuapi.dss.admin.DSSUser` user handle """ if groups is None: groups = [] resp = self._perform_text( "POST", "/admin/users/", body={ "login" : login, "password" : password, "displayName" : display_name, "sourceType" : source_type, "groups" : groups, "userProfile" : profile }) return DSSUser(self, login) def get_own_user(self): return DSSOwnUser(self) def list_users_activity(self, enabled_users_only=False): """ List all users activity Note: this call requires an API key with admin rights :return: A list of user activity logs, as a list of :class:`dataikuapi.dss.admin.DSSUserActivity` if as_objects is True, else as a list of dict :rtype: list of :class:`dataikuapi.dss.admin.DSSUserActivity` or a list of dict """ params = { "enabledUsersOnly": enabled_users_only } all_activity = self._perform_json("GET", "/admin/users-activity", params=params) return [DSSUserActivity(self, user_activity["login"], user_activity) for user_activity in all_activity] def get_authorization_matrix(self): """ Get the authorization matrix for all enabled users and groups Note: this call requires an API key with admin rights :return: The authorization matrix :rtype: A :class:`dataikuapi.dss.admin.DSSAuthorizationMatrix` authorization matrix handle """ resp = self._perform_json("GET", "/admin/authorization-matrix") return DSSAuthorizationMatrix(resp) ######################################################## # Groups ######################################################## def list_groups(self): """ List all groups setup on the DSS instance Note: this call requires an API key with admin rights :returns: A list of groups, as an list of dicts :rtype: list of dicts """ return self._perform_json( "GET", "/admin/groups/") def get_group(self, name): """ Get a handle to interact with a specific group :param str name: the name of the desired group :returns: A :class:`dataikuapi.dss.admin.DSSGroup` group handle """ return DSSGroup(self, name) def create_group(self, name, description=None, source_type='LOCAL'): """ Create a group, and return a handle to interact with it Note: this call requires an API key with admin rights :param str name: the name of the new group :param str description: (optional) a description of the new group :param source_type: the type of the new group. Admissible values are 'LOCAL' and 'LDAP' :returns: A :class:`dataikuapi.dss.admin.DSSGroup` group handle """ resp = self._perform_text( "POST", "/admin/groups/", body={ "name" : name, "description" : description, "sourceType" : source_type }) return DSSGroup(self, name) ######################################################## # Connections ######################################################## def list_connections(self, as_type="dictitems"): """ List all connections setup on the DSS instance. .. note:: This call requires an API key with admin rights :param string as_type: how to return the connection. Possible values are "dictitems", "listitems" and "objects" :return: if **as_type** is dictitems, a dict of connection name to :class:`dataikuapi.dss.admin.DSSConnectionListItem`. if **as_type** is listitems, a list of :class:`dataikuapi.dss.admin.DSSConnectionListItem`. if **as_type** is objects, a list of :class:`dataikuapi.dss.admin.DSSConnection`. :rtype: dict or list """ items_dict = self._perform_json( "GET", "/admin/connections/") if as_type == "dictitems" or as_type == "dictitem": return {name:DSSConnectionListItem(self, items_dict[name]) for name in items_dict.keys()} if as_type == "listitems" or as_type == "listitem": return [DSSConnectionListItem(self, items_dict[name]) for name in items_dict.keys()] elif as_type == "objects" or as_type == "object": return [DSSConnection(self, name) for name in items_dict.keys()] else: raise ValueError("Unknown as_type") def get_connection(self, name): """ Get a handle to interact with a specific connection :param str name: the name of the desired connection :returns: A :class:`dataikuapi.dss.admin.DSSConnection` connection handle """ return DSSConnection(self, name) def create_connection(self, name, type, params=None, usable_by='ALL', allowed_groups=None): """ Create a connection, and return a handle to interact with it Note: this call requires an API key with admin rights :param name: the name of the new connection :param type: the type of the new connection :param dict params: the parameters of the new connection, as a JSON object (defaults to `{}`) :param usable_by: the type of access control for the connection. Either 'ALL' (=no access control) or 'ALLOWED' (=access restricted to users of a list of groups) :param list allowed_groups: when using access control (that is, setting usable_by='ALLOWED'), the list of names of the groups whose users are allowed to use the new connection (defaults to `[]`) :returns: A :class:`dataikuapi.dss.admin.DSSConnection` connection handle """ if params is None: params = {} if allowed_groups is None: allowed_groups = [] resp = self._perform_text( "POST", "/admin/connections/", body={ "name" : name, "type" : type, "params" : params, "usableBy" : usable_by, "allowedGroups" : allowed_groups }) return DSSConnection(self, name) ######################################################## # Code envs ######################################################## def list_code_envs(self, as_objects=False): """ List all code envs setup on the DSS instance Note: this call requires an API key with admin rights :param boolean as_objects: if True, each returned item will be a :class:`dataikuapi.dss.future.DSSCodeEnv` :returns: a list of code envs. Each code env is a dict containing at least "name", "type" and "language" """ list = self._perform_json( "GET", "/admin/code-envs/") if as_objects: return [DSSCodeEnv(self, e.get("envLang"), e.get("envName")) for e in list] else: return list def get_code_env(self, env_lang, env_name): """ Get a handle to interact with a specific code env :param env_lang: the language (PYTHON or R) of the new code env :param env_name: the name of the new code env :returns: A :class:`dataikuapi.dss.admin.DSSCodeEnv` code env handle """ return DSSCodeEnv(self, env_lang, env_name) def create_code_env(self, env_lang, env_name, deployment_mode, params=None): """ Create a code env, and return a handle to interact with it Note: this call requires an API key with admin rights :param env_lang: the language (PYTHON or R) of the new code env :param env_name: the name of the new code env :param deployment_mode: the type of the new code env :param params: the parameters of the new code env, as a JSON object :returns: A :class:`dataikuapi.dss.admin.DSSCodeEnv` code env handle """ params = params if params is not None else {} params['deploymentMode'] = deployment_mode resp = self._perform_json( "POST", "/admin/code-envs/%s/%s" % (env_lang, env_name), body=params) if resp is None: raise Exception('Env creation returned no data') if resp.get('messages', {}).get('error', False): raise Exception('Env creation failed : %s' % (json.dumps(resp.get('messages', {}).get('messages', {})))) return DSSCodeEnv(self, env_lang, env_name) def list_code_env_usages(self): """ List all usages of a code env in the instance :return: a list of objects where the code env is used """ return self._perform_json("GET", "/admin/code-envs/usages") ######################################################## # Clusters ######################################################## def list_clusters(self): """ List all clusters setup on the DSS instance Returns: List clusters (name, type, state) """ return self._perform_json( "GET", "/admin/clusters/") def get_cluster(self, cluster_id): """ Get a handle to interact with a specific cluster Args: name: the name of the desired cluster Returns: A :class:`dataikuapi.dss.admin.DSSCluster` cluster handle """ return DSSCluster(self, cluster_id) def create_cluster(self, cluster_name, cluster_type='manual', params=None, cluster_architecture='HADOOP'): """ Create a cluster, and return a handle to interact with it :param cluster_name: the name of the new cluster :param cluster_type: the type of the new cluster :param params: the parameters of the new cluster, as a JSON object :param cluster_architecture: the architecture of the new cluster. 'HADOOP' or 'KUBERNETES' :returns: A :class:`dataikuapi.dss.admin.DSSCluster` cluster handle """ definition = {} definition['name'] = cluster_name definition['type'] = cluster_type definition['architecture'] = cluster_architecture definition['params'] = params if params is not None else {} resp = self._perform_json( "POST", "/admin/clusters/", body=definition) if resp is None: raise Exception('Cluster creation returned no data') if resp.get('messages', {}).get('error', False): raise Exception('Cluster creation failed : %s' % (json.dumps(resp.get('messages', {}).get('messages', {})))) return DSSCluster(self, resp['id']) ######################################################## # Code studio templates ######################################################## def list_code_studio_templates(self, as_type='listitems'): """ List all code studio templates on the DSS instance :returns: List of templates (name, type) """ items = self._perform_json("GET", "/admin/code-studios/") if as_type == "listitems" or as_type == "listitem": return [DSSCodeStudioTemplateListItem(self, item) for item in items] elif as_type == "objects" or as_type == "object": return [DSSCodeStudioTemplate(self, item["id"]) for item in items] else: raise ValueError("Unknown as_type") def get_code_studio_template(self, template_id): """ Get a handle to interact with a specific code studio template :param str template_id: the template id of the desired code studio template :returns: A :class:`dataikuapi.dss.admin.DSSCodeStudioTemplate` code studio template handle """ return DSSCodeStudioTemplate(self, template_id) ######################################################## # Global API Keys ######################################################## def list_global_api_keys(self, as_type='listitems'): """ List all global API keys set up on the DSS instance .. note:: This call requires an API key with admin rights :param str as_type: How to return the global API keys. Possible values are "listitems" and "objects" :return: if as_type=listitems, each key as a :class:`dataikuapi.dss.admin.DSSGlobalApiKeyListItem`. if as_type=objects, each key is returned as a :class:`dataikuapi.dss.admin.DSSGlobalApiKey`. """ resp = self._perform_json( "GET", "/admin/globalAPIKeys/") if as_type == "listitems": return [DSSGlobalApiKeyListItem(self, item) for item in resp] elif as_type == 'objects': return [DSSGlobalApiKey(self, item["key"]) for item in resp] else: raise ValueError("Unknown as_type") def get_global_api_key(self, key): """ Get a handle to interact with a specific Global API key :param str key: the secret key of the desired API key :returns: A :class:`dataikuapi.dss.admin.DSSGlobalApiKey` API key handle """ return DSSGlobalApiKey(self, key) def create_global_api_key(self, label=None, description=None, admin=False): """ Create a Global API key, and return a handle to interact with it .. note:: This call requires an API key with admin rights :param str label: the label of the new API key :param str description: the description of the new API key :param str admin: has the new API key admin rights (True or False) :returns: A :class:`dataikuapi.dss.admin.DSSGlobalApiKey` API key handle """ resp = self._perform_json( "POST", "/admin/globalAPIKeys/", body={ "label" : label, "description" : description, "globalPermissions": { "admin": admin } }) if resp is None: raise Exception('API key creation returned no data') if resp.get('messages', {}).get('error', False): raise Exception('API key creation failed : %s' % (json.dumps(resp.get('messages', {}).get('messages', {})))) if not resp.get('id', False): raise Exception('API key creation returned no key') key = resp.get('key', '') return DSSGlobalApiKey(self, key) ######################################################## # Personal API Keys ######################################################## def list_personal_api_keys(self, as_type='listitems'): """ List all your personal API keys. :param str as_type: How to return the personal API keys. Possible values are "listitems" and "objects" :return: if as_type=listitems, each key as a :class:`dataikuapi.dss.admin.DSSPersonalApiKeyListItem`. if as_type=objects, each key is returned as a :class:`dataikuapi.dss.admin.DSSPersonalApiKey`. """ resp = self._perform_json( "GET", "/personal-api-keys/") if as_type == "listitems": return [DSSPersonalApiKeyListItem(self, item) for item in resp] elif as_type == 'objects': return [DSSPersonalApiKey(self, item['id']) for item in resp] else: raise ValueError("Unknown as_type") def get_personal_api_key(self, id): """ Get a handle to interact with a specific Personal API key. :param str id: the id of the desired API key :returns: A :class:`dataikuapi.dss.admin.DSSPersonalApiKey` API key handle """ return DSSPersonalApiKey(self, id) def create_personal_api_key(self, label="", description="", as_type='dict'): """ Create a Personal API key associated with your user. :param str label: the label of the new API key :param str description: the description of the new API key :param str as_type: How to return the personal API keys. Possible values are "dict" and "object" :return: if as_type=dict, the new personal API key is returned as a dict. if as_type=object, the new personal API key is returned as a :class:`dataikuapi.dss.admin.DSSPersonalApiKey`. """ resp = self._perform_json( "POST", "/personal-api-keys/", body={"label": label, "description": description}) if resp is None: raise Exception('API key creation returned no data') if not resp.get('id', False): raise Exception('API key creation returned no key') if as_type == 'object': return DSSPersonalApiKey(self, resp["id"]) else: return resp def list_all_personal_api_keys(self, as_type='listitems'): """ List all personal API keys. Only admin can list all the keys. :param str as_type: How to return the personal API keys. Possible values are "listitems" and "objects" :return: if as_type=listitems, each key as a :class:`dataikuapi.dss.admin.DSSPersonalApiKeyListItem`. if as_type=objects, each key is returned as a :class:`dataikuapi.dss.admin.DSSPersonalApiKey`. """ resp = self._perform_json( "GET", "/admin/personal-api-keys/") if as_type == "listitems": return [DSSPersonalApiKeyListItem(self, item) for item in resp] elif as_type == 'objects': return [DSSPersonalApiKey(self, item['id']) for item in resp] else: raise ValueError("Unknown as_type") def create_personal_api_key_for_user(self, user, label="", description="", as_type='object'): """ Create a Personal API key associated on behalf of a user. Only admin can create a key for another user. :param str label: the label of the new API key :param str description: the description of the new API key :param str user: the id of the user to impersonate :param str as_type: How to return the personal API keys. Possible values are "dict" and "object" :return: if as_type=dict, the new personal API key is returned as a dict. if as_type=object, the new personal API key is returned as a :class:`dataikuapi.dss.admin.DSSPersonalApiKey`. """ resp = self._perform_json( "POST", "/admin/personal-api-keys/", body={"user": user, "label": label, "description": description}) if resp is None: raise Exception('API key creation returned no data') if not resp.get('id', False): raise Exception('API key creation returned no key') if as_type == 'object': return DSSPersonalApiKey(self, resp["id"]) else: return resp ######################################################## # Meanings ######################################################## def list_meanings(self): """ List all user-defined meanings on the DSS instance Note: this call requires an API key with admin rights :returns: A list of meanings. Each meaning is a dict :rtype: list of dicts """ return self._perform_json( "GET", "/meanings/") def get_meaning(self, id): """ Get a handle to interact with a specific user-defined meaning Note: this call requires an API key with admin rights :param str id: the ID of the desired meaning :returns: A :class:`dataikuapi.dss.meaning.DSSMeaning` meaning handle """ return DSSMeaning(self, id) def create_meaning(self, id, label, type, description=None, values=None, mappings=None, pattern=None, normalizationMode=None, detectable=False): """ Create a meaning, and return a handle to interact with it Note: this call requires an API key with admin rights :param id: the ID of the new meaning :param type: the type of the new meaning. Admissible values are 'DECLARATIVE', 'VALUES_LIST', 'VALUES_MAPPING' and 'PATTERN' :param description (optional): the description of the new meaning :param values (optional): when type is 'VALUES_LIST', the list of values, or a list of {'value':'the value', 'color':'an optional color'} :param mappings (optional): when type is 'VALUES_MAPPING', the mapping, as a list of objects with this structure: {'from': 'value_1', 'to': 'value_a'} :param pattern (optional): when type is 'PATTERN', the pattern :param normalizationMode (optional): when type is 'VALUES_LIST', 'VALUES_MAPPING' or 'PATTERN', the normalization mode to use for value matching. One of 'EXACT', 'LOWERCASE', or 'NORMALIZED' (not available for 'PATTERN' type). Defaults to 'EXACT'. :param detectable (optional): whether DSS should consider assigning the meaning to columns set to 'Auto-detect'. Defaults to False. :returns: A :class:`dataikuapi.dss.meaning.DSSMeaning` meaning handle """ def make_entry(v): if isinstance(v, dku_basestring_type): return {'value':v} else: return v def make_mapping(v): return {'from':v.get('from', None), 'to':make_entry(v.get('to', None))} entries = None if values is not None: entries = [make_entry(v) for v in values] if mappings is not None: mappings = [make_mapping(v) for v in mappings] resp = self._perform_text( "POST", "/meanings/", body={ "id" : id, "label": label, "type": type, "description": description, "entries": entries, "mappings": mappings, "pattern": pattern, "normalizationMode": normalizationMode, "detectable": detectable }) return DSSMeaning(self, id) ######################################################## # Logs ######################################################## def list_logs(self): """ List all available log files on the DSS instance This call requires an API key with admin rights :returns: A list of log file names """ return self._perform_json( "GET", "/admin/logs/") def get_log(self, name): """ Get the contents of a specific log file This call requires an API key with admin rights :param str name: the name of the desired log file (obtained with :meth:`list_logs`) :returns: The full content of the log file, as a string """ return self._perform_json( "GET", "/admin/logs/%s" % name) def log_custom_audit(self, custom_type, custom_params=None): """ Log a custom entry to the audit trail :param str custom_type: value for customMsgType in audit trail item :param dict custom_params: value for customMsgParams in audit trail item (defaults to `{}`) """ if custom_params is None: custom_params = {} return self._perform_empty("POST", "/admin/audit/custom/%s" % custom_type, body = custom_params) ######################################################## # Monitoring ######################################################## def get_global_usage_summary(self, with_per_project=False): """ Gets a summary of the global usage of this DSS instance (number of projects, datasets, ...) :returns: a summary object """ data = self._perform_json( "GET", "/admin/monitoring/global-usage-summary", params={'withPerProject':with_per_project}) return DSSGlobalUsageSummary(data) ######################################################## # Variables ######################################################## def get_variables(self): """ Deprecated. Use :meth:`get_global_variables` """ warnings.warn("get_variables is deprecated, please use get_global_variables", DeprecationWarning) return self.get_global_variables() def get_global_variables(self): """ Get the DSS instance's variables, as a Python dictionary This call requires an API key with admin rights :returns: A :class:`dataikuapi.dss.admin.DSSInstanceVariables` handle """ variables = self._perform_json("GET", "/admin/variables/") return DSSInstanceVariables(self, variables) def set_variables(self, variables): """ Deprecated. Use :meth:`get_global_variables` and :meth:`dataikuapi.dss.admin.DSSInstanceVariables.save` Updates the DSS instance's variables This call requires an API key with admin rights It is not possible to update a single variable, you must set all of them at once. Thus, you should only use a ``variables`` parameter that has been obtained using :meth:`get_variables`. :param dict variables: the new dictionary of all variables of the instance """ warnings.warn("set_variables is deprecated, please use get_global_variables().save()", DeprecationWarning) return DSSInstanceVariables(self, variables).save() def get_resolved_variables(self, project_key=None, typed=False): """ Get a dictionary of resolved variables of the project. :param str project_key: the project key, defaults to the current project if any :param bool typed: if True, the variable values will be typed in the returned dict, defaults to False :returns: a dictionary with instance and project variables merged """ import dataiku return self._perform_json( "GET", "/projects/%s/variables-resolved" % (dataiku.default_project_key() if project_key is None else project_key), params={ "typed": "true" if typed else "false" }) ######################################################## # General settings ######################################################## def get_general_settings(self): """ Gets a handle to interact with the general settings. This call requires an API key with admin rights :returns: a :class:`dataikuapi.dss.admin.DSSGeneralSettings` handle """ return DSSGeneralSettings(self) ######################################################## # Bundles / Import (Automation node) ######################################################## def create_project_from_bundle_local_archive(self, archive_path, project_folder=None): """ Create a project from a bundle archive. Warning: this method can only be used on an automation node. :param string archive_path: Path on the local machine where the archive is :param project_folder: the project folder in which the project will be created or None for root project folder :type project_folder: A :class:`dataikuapi.dss.projectfolder.DSSProjectFolder` """ params = { "archivePath" : osp.abspath(archive_path) } if project_folder is not None: params["projectFolderId"] = project_folder.project_folder_id return self._perform_json("POST", "/projectsFromBundle/fromArchive", params=params) def create_project_from_bundle_archive(self, fp, project_folder=None): """ Create a project from a bundle archive (as a file object) Warning: this method can only be used on an automation node. :param string fp: A file-like object pointing to a bundle archive zip :param project_folder: the project folder in which the project will be created or None for root project folder :type project_folder: A :class:`dataikuapi.dss.projectfolder.DSSProjectFolder` """ params = {} if project_folder is not None: params['projectFolderId'] = project_folder.project_folder_id files = {'file': fp } return self._perform_json("POST", "/projectsFromBundle/", files=files, params=params) def prepare_project_import(self, f): """ Prepares import of a project archive. Warning: this method can only be used on a design node. :param file-like fp: the input stream, as a file-like object :returns: a :class:`TemporaryImportHandle` to interact with the prepared import """ val = self._perform_json_upload( "POST", "/projects/import/upload", "tmp-import.zip", f) return TemporaryImportHandle(self, val.json()["id"]) ######################################################## # API Deployer ######################################################## def get_apideployer(self): """Gets a handle to work with the API Deployer :rtype: :class:`~dataikuapi.dss.apideployer.DSSAPIDeployer` """ return DSSAPIDeployer(self) ######################################################## # Project Deployer ######################################################## def get_projectdeployer(self): """Gets a handle to work with the Project Deployer :rtype: :class:`~dataikuapi.dss.projectdeployer.DSSProjectDeployer` """ return DSSProjectDeployer(self) ######################################################## # Data Catalog ######################################################## def catalog_index_connections(self, connection_names=None, all_connections=False, indexing_mode="FULL"): """ Triggers an indexing of multiple connections in the data catalog :param list connection_names: list of connections to index, ignored if `all_connections=True` (defaults to `[]`) :param bool all_connections: index all connections (defaults to `False`) """ if connection_names is None: connection_names = [] return self._perform_json("POST", "/catalog/index", body={ "connectionNames": connection_names, "indexAllConnections": all_connections, "indexingMode": indexing_mode }) ######################################################## # Model export ######################################################## def get_scoring_libs_stream(self): """ Get the scoring libraries jar required for scoring with model jars that don't include libraries. You need to close the stream after download. Failure to do so will result in the DSSClient becoming unusable. :returns: a jar file, as a stream :rtype: file-like """ return self._perform_raw("GET", "/resources/scoring-lib-jar") ######################################################## # Auth ######################################################## def get_auth_info(self, with_secrets=False): """ Returns various information about the user currently authenticated using this instance of the API client. This method returns a dict that may contain the following keys (may also contain others): * authIdentifier: login for a user, id for an API key * groups: list of group names (if context is an user) * secrets: list of dicts containing user secrets (if context is an user) :param: with_secrets boolean: Return user secrets :returns: a dict :rtype: dict """ return self._perform_json("GET", "/auth/info", params={"withSecrets": with_secrets}) def get_auth_info_from_browser_headers(self, headers_dict, with_secrets=False): """ Returns various information about the DSS user authenticated by the dictionary of HTTP headers provided in headers_dict. This is generally only used in webapp backends This method returns a dict that may contain the following keys (may also contain others): * authIdentifier: login for a user, id for an API key * groups: list of group names (if context is an user) * secrets: list of dicts containing user secrets (if context is an user) :param: headers_dict dict: Dictionary of HTTP headers :param: with_secrets boolean: Return user secrets :returns: a dict :rtype: dict """ return self._perform_json("POST", "/auth/info-from-browser-headers", params={"withSecrets": with_secrets}, body=headers_dict) def get_ticket_from_browser_headers(self, headers_dict): """ Returns a ticket for the DSS user authenticated by the dictionary of HTTP headers provided in headers_dict. This is only used in webapp backends This method returns a ticket to use as a X-DKU-APITicket header :param: headers_dict dict: Dictionary of HTTP headers :returns: a string :rtype: string """ return self._perform_json("POST", "/auth/ticket-from-browser-headers", body=headers_dict) ######################################################## # Container execution ######################################################## def push_base_images(self): """ Push base images for Kubernetes container-execution and Spark-on-Kubernetes """ resp = self._perform_json("POST", "/admin/container-exec/actions/push-base-images") return DSSFuture.from_resp(self, resp) def apply_kubernetes_namespaces_policies(self): """ Apply Kubernetes namespaces policies defined in the general settings """ resp = self._perform_json("POST", "/admin/container-exec/actions/apply-kubernetes-policies") return DSSFuture.from_resp(self, resp) ######################################################## # Global Instance Info ######################################################## def get_instance_info(self): """ Get global information about the DSS instance :returns: a :class:`DSSInstanceInfo` """ resp = self._perform_json("GET", "/instance-info") return DSSInstanceInfo(resp) ######################################################## # Instance Sanity Check ######################################################## def perform_instance_sanity_check(self, exclusion_list=[], wait=True): """ Run an Instance Sanity Check. This call requires an API key with admin rights. :param exclusion_list: a string list of codes to exclude in the sanity check, as returned by :meth:`get_sanity_check_codes` :return: a :class:`dataikuapi.dss.utils.DSSInfoMessages` if `wait` is `True`, or a :class:`dataikuapi.dss.future.DSSFuture` handle otherwise :rtype: :class:`dataikuapi.dss.utils.DSSInfoMessages` or :class:`dataikuapi.dss.future.DSSFuture` """ resp = self._perform_json("POST", "/admin/sanity-check/run", body=exclusion_list) future = DSSFuture.from_resp(self, resp, result_wrapper=DSSInfoMessages) if wait: return future.wait_for_result() else: return future def get_sanity_check_codes(self): """ Return the list of codes that can be generated by the sanity check. This call requires an API key with admin rights. :rtype: list[str] """ return self._perform_json("GET", "/admin/sanity-check/codes") ######################################################## # Licensing ######################################################## def get_licensing_status(self): """ Returns a dictionary with information about licensing status of this DSS instance :rtype: dict """ return self._perform_json("GET", "/admin/licensing/status") def set_license(self, license): """ Sets a new licence for DSS :param license: license (content of license file) :return: None """ self._perform_empty( "POST", "/admin/licensing/license", body=json.loads(license)) ######################################################## # Internal Request handling ######################################################## def _perform_http(self, method, path, params=None, body=None, stream=False, files=None, raw_body=None, headers=None): if body is not None: body = json.dumps(body) if raw_body is not None: body = raw_body try: http_res = self._session.request( method, "%s/dip/publicapi%s" % (self.host, path), params=params, data=body, files=files, stream=stream, headers=headers) http_res.raise_for_status() return http_res except exceptions.HTTPError: try: ex = http_res.json() except ValueError: ex = {"message": http_res.text} raise DataikuException("%s: %s" % (ex.get("errorType", "Unknown error"), ex.get("message", "No message"))) def _perform_empty(self, method, path, params=None, body=None, files = None, raw_body=None): self._perform_http(method, path, params=params, body=body, files=files, stream=False, raw_body=raw_body) def _perform_text(self, method, path, params=None, body=None,files=None, raw_body=None): return self._perform_http(method, path, params=params, body=body, files=files, stream=False, raw_body=raw_body).text def _perform_json(self, method, path, params=None, body=None,files=None, raw_body=None): return self._perform_http(method, path, params=params, body=body, files=files, stream=False, raw_body=raw_body).json() def _perform_raw(self, method, path, params=None, body=None,files=None, raw_body=None): return self._perform_http(method, path, params=params, body=body, files=files, stream=True, raw_body=raw_body) def _perform_json_upload(self, method, path, name, f): try: http_res = self._session.request( method, "%s/dip/publicapi%s" % (self.host, path), files = {'file': (name, f, {'Expires': '0'})} ) http_res.raise_for_status() return http_res except exceptions.HTTPError: ex = http_res.json() raise DataikuException("%s: %s" % (ex.get("errorType", "Unknown error"), ex.get("message", "No message"))) ######################################################## # Discussions ######################################################## def get_object_discussions(self, project_key, object_type, object_id): """ Get a handle to manage discussions on any object :param str project_key: identifier of the project to access :param str object_type: DSS object type :param str object_id: DSS object ID :returns: the handle to manage discussions :rtype: :class:`dataikuapi.discussion.DSSObjectDiscussions` """ return DSSObjectDiscussions(self, project_key, object_type, object_id) ######################################################## # Feature Store ######################################################## def get_feature_store(self): """ Get a handle to interact with the Feature Store. :return: a handle on the feature store :rtype: :class:`dataikuapi.feature_store.DSSFeatureStore` """ return DSSFeatureStore(self) ######################################################## # Workspaces ######################################################## def list_workspaces(self, as_objects=False): """ List the workspaces :returns: The list of workspaces. """ items = self._perform_json("GET", "/workspaces/") if as_objects: return [DSSWorkspace(self, item["workspaceKey"]) for item in items] else: return items def get_workspace(self, workspace_key): """ Get a handle to interact with a specific workspace :param str workspace_key: the workspace key of the desired workspace :returns: A :class:`dataikuapi.dss.workspace.DSSWorkspace` to interact with this workspace """ return DSSWorkspace(self, workspace_key) def create_workspace(self, workspace_key, name, permissions=None, description=None, color=None): """ Create a new workspace and return a workspace handle to interact with it :param str workspace_key: the identifier to use for the workspace. Must be globally unique :param str name: the display name for the workspace. :param [`dataikuapi.dss.workspace.DSSWorkspacePermissionItem`] permissions: Initial permissions for the workspace (can be modified later). :param str description: a description for the workspace. :param str color: The color to use (#RRGGBB format). A random color will be assigned if not specified :returns: A :class:`dataikuapi.dss.workspace.DSSWorkspace` workspace handle to interact with this workspace """ resp = self._perform_text("POST", "/workspaces/", body={ "workspaceKey": workspace_key, "displayName": name, "color": color, "description": description, "permissions": permissions }) return DSSWorkspace(self, workspace_key) class TemporaryImportHandle(object): def __init__(self, client, import_id): self.client = client self.import_id = import_id def execute(self, settings=None): """ Executes the import with provided settings. :param dict settings: Dict of import settings (defaults to `{}`). The following settings are available: * targetProjectKey (string): Key to import under. Defaults to the original project key * remapping (dict): Dictionary of connection and code env remapping settings. See example of remapping dict: .. code-block:: python "remapping" : { "connections": [ { "source": "src_conn1", "target": "target_conn1" }, { "source": "src_conn2", "target": "target_conn2" } ], "codeEnvs" : [ { "source": "src_codeenv1", "target": "target_codeenv1" }, { "source": "src_codeenv2", "target": "target_codeenv2" } ] } @warning: You must check the 'success' flag """ # Empty JSON dicts can't be parsed properly if settings is None: settings = {} if settings == {}: settings["_"] = "_" return self.client._perform_json("POST", "/projects/import/%s/process" % (self.import_id), body = settings) class DSSInstanceInfo(object): """Global information about the DSS instance""" def __init__(self, data): """Do not call this directly, use :meth:`DSSClient.get_instance_info`""" self._data = data @property def raw(self): """Returns all data as a Python dictionary""" return self._data @property def node_id(self): """Returns the node id (as defined in Cloud Stacks or in install.ini)""" return self._data["nodeId"] @property def node_name(self): """Returns the node name as it appears in the navigation bar""" return self._data["nodeName"] @property def node_type(self): """ Returns the node type :return: One of DESIGN, AUTOMATION or DEPLOYER """ return self._data["nodeType"] dataiku-api-client-python-11.4.0/dataikuapi/fm/000077500000000000000000000000001440464736400213135ustar00rootroot00000000000000dataiku-api-client-python-11.4.0/dataikuapi/fm/__init__.py000066400000000000000000000000001440464736400234120ustar00rootroot00000000000000dataiku-api-client-python-11.4.0/dataikuapi/fm/future.py000066400000000000000000000061461440464736400232060ustar00rootroot00000000000000import sys, time class FMFuture(object): """ A future on the DSS instance """ def __init__( self, client, job_id, state=None, result_wrapper=lambda result: result ): self.client = client self.job_id = job_id self.state = state self.state_is_peek = True self.result_wrapper = result_wrapper @staticmethod def from_resp(client, resp, result_wrapper=lambda result: result): """ Creates a DSSFuture from a parsed JSON response :rtype: :class:`dataikuapi.fm.future.FMFuture` """ return FMFuture( client, resp.get("jobId", None), state=resp, result_wrapper=result_wrapper ) @classmethod def get_result_wait_if_needed(cls, client, ret): if "jobId" in ret: future = FMFuture(client, ret["jobId"], ret) future.wait_for_result() return future.get_result() else: return ret["result"] def abort(self): """ Abort the future """ return self.client._perform_tenant_empty("DELETE", "/futures/%s" % self.job_id) def get_state(self): """ Get the status of the future, and its result if it's ready """ self.state = self.client._perform_tenant_json( "GET", "/futures/%s" % self.job_id, params={"peek": False} ) self.state_is_peek = False return self.state def peek_state(self): """ Get the status of the future, and its result if it's ready """ self.state = self.client._perform_tenant_json( "GET", "/futures/%s" % self.job_id, params={"peek": True} ) self.state_is_peek = True return self.state def get_result(self): """ Get the future result if it's ready, raises an Exception otherwise """ if ( self.state is None or not self.state.get("hasResult", False) or self.state_is_peek ): self.get_state() if self.state.get("hasResult", False): return self.result_wrapper(self.state.get("result", None)) else: raise Exception("Result not ready") def has_result(self): """ Checks whether the future has a result ready """ if self.state is None or not self.state.get("hasResult", False): self.get_state() return self.state.get("hasResult", False) def wait_for_result(self): """ Wait and get the future result """ if self.state.get("hasResult", False): return self.result_wrapper(self.state.get("result", None)) if ( self.state is None or not self.state.get("hasResult", False) or self.state_is_peek ): self.get_state() while not self.state.get("hasResult", False): time.sleep(5) self.get_state() if self.state.get("hasResult", False): return self.result_wrapper(self.state.get("result", None)) else: raise Exception("No result") dataiku-api-client-python-11.4.0/dataikuapi/fm/instances.py000066400000000000000000000431311440464736400236560ustar00rootroot00000000000000from .future import FMFuture from ..dssclient import DSSClient from ..govern_client import GovernClient import sys if sys.version_info > (3, 4): from enum import Enum else: class Enum(object): pass class FMInstanceCreator(object): def __init__( self, client, label, instance_settings_template_id, virtual_network_id, image_id ): """ Helper to create a DSS instance. :param client: The FM client :type client: :class:`dataikuapi.fm.fmclient` :param str label: The label of the instance :param str instance_settings_template: The instance settings template id this instance should be based on :param str virtual_network: The virtual network where the instance should be spawned :param str image_id: The ID of the DSS runtime image (ex: dss-9.0.3-default) """ self.client = client self.data = {} self.data["label"] = label self.data["instanceSettingsTemplateId"] = instance_settings_template_id self.data["virtualNetworkId"] = virtual_network_id self.data["imageId"] = image_id # Set the default value for dssNodeType self.data["dssNodeType"] = "design" def with_dss_node_type(self, dss_node_type): """ Set the DSS node type of the instance to create. The default node type is `DESIGN`. :param dss_node_type: the type of the dss node to create. :type dss_node_type: :class:`dataikuapi.fm.instances.FMNodeType` :rtype: :class:`dataikuapi.fm.instances.FMInstanceCreator` """ # backward compatibility, was a string before . be sure the value falls into the enum value = dss_node_type # python2 does not support enum, keep string if sys.version_info > (3, 4): if isinstance(dss_node_type, str): value = FMNodeType[dss_node_type.upper()] self.data["dssNodeType"] = value.value else: value = FMNodeType.get_from_string(dss_node_type.upper()) self.data["dssNodeType"] = value; return self def with_cloud_instance_type(self, cloud_instance_type): """ Set the machine type for the DSS Instance :param str cloud_instance_type: the machine type to be used for the instance :rtype: :class:`dataikuapi.fm.instances.FMInstanceCreator` """ self.data["cloudInstanceType"] = cloud_instance_type return self def with_data_volume_options( self, data_volume_type=None, data_volume_size=None, data_volume_size_max=None, data_volume_IOPS=None, data_volume_encryption=None, data_volume_encryption_key=None, ): """ Set the options of the data volume to use with the DSS Instance :param str data_volume_type: Optional, data volume type :param int data_volume_size: Optional, data volume initial size :param int data_volume_size_max: Optional, data volume maximum size :param int data_volume_IOPS: Optional, data volume IOPS :param data_volume_encryption: Optional, encryption mode of the data volume :type data_volume_encryption: :class:`dataikuapi.fm.instances.FMInstanceEncryptionMode` :param str data_volume_encryption_key: Optional, the encryption key to use when data_volume_encryption_key is FMInstanceEncryptionMode.CUSTOM :rtype: :class:`dataikuapi.fm.instances.FMInstanceCreator` """ if type(data_volume_encryption) is not FMInstanceEncryptionMode: raise TypeError( "data_volume encryption needs to be of type FMInstanceEncryptionMode" ) self.data["dataVolumeType"] = data_volume_type self.data["dataVolumeSizeGB"] = data_volume_size self.data["dataVolumeSizeMaxGB"] = data_volume_size_max self.data["dataVolumeIOPS"] = data_volume_IOPS self.data["volumesEncryption"] = data_volume_encryption.value self.data["volumesEncryptionKey"] = data_volume_encryption_key return self def with_cloud_tags(self, cloud_tags): """ Set the tags to be applied to the cloud resources created for this DSS instance :param dict cloud_tags: a key value dictionary of tags to be applied on the cloud resources :rtype: :class:`dataikuapi.fm.instances.FMInstanceCreator` """ self.data["cloudTags"] = cloud_tags return self def with_fm_tags(self, fm_tags): """ A list of tags to add on the DSS Instance in Fleet Manager :param list fm_tags: Optional, list of tags to be applied on the instance in the Fleet Manager :rtype: :class:`dataikuapi.fm.instances.FMInstanceCreator` """ self.data["fmTags"] = fm_tags return self class FMAWSInstanceCreator(FMInstanceCreator): def with_aws_root_volume_options( self, aws_root_volume_size=None, aws_root_volume_type=None, aws_root_volume_IOPS=None, ): """ Set the options of the root volume of the DSS Instance :param int aws_root_volume_size: Optional, the root volume size :param str aws_root_volume_type: Optional, the root volume type :param int aws_root_volume_IOPS: Optional, the root volume IOPS :rtype: :class:`dataikuapi.fm.instances.FMAWSInstanceCreator` """ self.data["awsRootVolumeSizeGB"] = aws_root_volume_size self.data["awsRootVolumeType"] = aws_root_volume_type self.data["awsRootVolumeIOPS"] = aws_root_volume_IOPS return self def create(self): """ Create the DSS instance :return: a newly created DSS instance :rtype: :class:`dataikuapi.fm.instances.FMAWSInstance` """ instance = self.client._perform_tenant_json( "POST", "/instances", body=self.data ) return FMAWSInstance(self.client, instance) class FMAzureInstanceCreator(FMInstanceCreator): def create(self): """ Create the DSS instance :return: a newly created DSS instance :rtype: :class:`dataikuapi.fm.instances.FMAzureInstance` """ instance = self.client._perform_tenant_json( "POST", "/instances", body=self.data ) return FMAzureInstance(self.client, instance) class FMGCPInstanceCreator(FMInstanceCreator): def create(self): """ Create the DSS instance :return: a newly created DSS instance :rtype: :class:`dataikuapi.fm.instances.FMGCPInstance` """ instance = self.client._perform_tenant_json( "POST", "/instances", body=self.data ) return FMGCPInstance(self.client, instance) class FMInstance(object): """ A handle to interact with a DSS instance. Do not create this directly, use :meth:`dataikuapi.fmclient.FMClient.get_instance` or * :meth:`dataikuapi.fmclient.FMClientAWS.new_instance_creator` * :meth:`dataikuapi.fmclient.FMClientAzure.new_instance_creator` * :meth:`dataikuapi.fmclient.FMClientGCP.new_instance_creator` """ def __init__(self, client, instance_data): self.client = client self.instance_data = instance_data self.id = instance_data["id"] def get_client(self): """ Get a Python client to communicate with a DSS instance :return: a Python client to communicate with the target instance :rtype: :class:`dataikuapi.dssclient.DSSClient` """ instance_status = self.get_status() external_url = instance_status.get("publicURL") admin_api_key_resp = self.client._perform_tenant_json( "GET", "/instances/%s/admin-api-key" % self.id ) api_key = admin_api_key_resp["adminAPIKey"] if not external_url: raise ValueError("No external URL available for node %s. This node may not be provisioned yet" % self.id) if self.instance_data.get("dssNodeType") == "govern": return GovernClient(external_url, api_key) else: return DSSClient(external_url, api_key) def reprovision(self): """ Reprovision the physical DSS instance :return: the `Future` object representing the reprovision process :rtype: :class:`dataikuapi.fm.future.FMFuture` """ future = self.client._perform_tenant_json( "GET", "/instances/%s/actions/reprovision" % self.id ) return FMFuture.from_resp(self.client, future) def deprovision(self): """ Deprovision the physical DSS instance :return: the `Future` object representing the deprovision process :rtype: :class:`dataikuapi.fm.future.FMFuture` """ future = self.client._perform_tenant_json( "GET", "/instances/%s/actions/deprovision" % self.id ) return FMFuture.from_resp(self.client, future) def restart_dss(self): """ Restart the DSS running on the physical instance :return: the `Future` object representing the restart process :rtype: :class:`dataikuapi.fm.future.FMFuture` """ future = self.client._perform_tenant_json( "GET", "/instances/%s/actions/restart-dss" % self.id ) return FMFuture.from_resp(self.client, future) def save(self): """ Update the instance """ self.client._perform_tenant_empty( "PUT", "/instances/%s" % self.id, body=self.instance_data ) self.instance_data = self.client._perform_tenant_json( "GET", "/instances/%s" % self.id ) def get_status(self): """ Get the physical DSS instance's status :return: the instance status :rtype: :class:`dataikuapi.fm.instances.FMInstanceStatus` """ status = self.client._perform_tenant_json( "GET", "/instances/%s/status" % self.id ) return FMInstanceStatus(status) def delete(self): """ Delete the DSS instance :return: the `Future` object representing the deletion process :rtype: :class:`dataikuapi.fm.future.FMFuture` """ future = self.client._perform_tenant_json( "GET", "/instances/%s/actions/delete" % self.id ) return FMFuture.from_resp(self.client, future) def get_initial_password(self): """ Get the initial DSS admin password. Can only be called once :return: a password for the 'admin' user. """ return self.client._perform_tenant_json( "GET", "/instances/%s/actions/get-initial-password" % self.id ) def reset_user_password(self, username, password): """ Reset the password for a user on the DSS instance :param string username: login :param string password: new password :return: the `Future` object representing the password reset process :rtype: :class:`dataikuapi.fm.future.FMFuture` """ future = self.client._perform_tenant_json( "GET", "/instances/%s/actions/reset-user-password" % self.id, params={ 'userName':username, 'password':password } ) return FMFuture.from_resp(self.client, future) def replay_setup_actions(self): """ Replay the setup actions on the DSS instance :return: the `Future` object representing the replay process :rtype: :class:`dataikuapi.fm.future.FMFuture` """ future = self.client._perform_tenant_json( "GET", "/instances/%s/actions/replay-setup-actions" % self.id ) return FMFuture.from_resp(self.client, future) def set_automated_snapshots(self, enable, period, keep=0): """ Set the automated snapshot policy for this instance :param boolean enable: Enable the automated snapshots :param int period: The time period between 2 snapshot in hours :param int keep: Optional, the number of snapshot to keep. Use 0 to keep all snapshots. Defaults to 0. """ self.instance_data["enableAutomatedSnapshot"] = enable self.instance_data["automatedSnapshotPeriod"] = period self.instance_data["automatedSnapshotRetention"] = keep return self def set_custom_certificate(self, pem_data): """ Set the custom certificate for this instance Only needed when Virtual Network HTTPS Strategy is set to Custom Certificate :param str pem_data: The SSL certificate """ self.instance_data["sslCertificatePEM"] = pem_data return self ######################################################## # Snapshots ######################################################## def list_snapshots(self): """ List all the snapshots of this instance :return: list of snapshots :rtype: list of :class:`dataikuapi.fm.instances.FMSnapshot` """ snapshots = self.client._perform_tenant_json("GET", "/instances/%s/snapshots" % self.id) return [FMSnapshot(self.client, self.id, x["id"], x) for x in snapshots] def get_snapshot(self, snapshot_id): """ Get a snapshot of this instance :param str snapshot_id: identifier of the snapshot :return: Snapshot :rtype: :class:`dataikuapi.fm.instances.FMSnapshot` """ return FMSnapshot(self.client, self.id, snapshot_id) def snapshot(self, reason_for_snapshot=None): """ Create a snapshot of the instance :return: Snapshot :rtype: :class:`dataikuapi.fm.instances.FMSnapshot` """ snapshot = self.client._perform_tenant_json( "POST", "/instances/%s/snapshots" % self.id, params={ "reasonForSnapshot":reason_for_snapshot } ) return FMSnapshot(self.client, self.id, snapshot["id"], snapshot) class FMAWSInstance(FMInstance): def set_elastic_ip(self, enable, elasticip_allocation_id): """ Set a public elastic ip for this instance :param boolan enable: Enable the elastic ip allocation :param str elasticip_allocation_id: AWS ElasticIP allocation ID """ self.instance_data["awsAssignElasticIP"] = enable self.instance_data["awsElasticIPAllocationId"] = elasticip_allocation_id return self class FMAzureInstance(FMInstance): def set_elastic_ip(self, enable, public_ip_id): """ Set a public elastic ip for this instance :param boolan enable: Enable the elastic ip allocation :param str public_ip_id: Azure Public IP ID """ self.instance_data["azureAssignElasticIP"] = enable self.instance_data["azurePublicIPId"] = public_ip_id return self class FMGCPInstance(FMInstance): def set_public_ip(self, enable, public_ip_id): """ Set a public ip for this instance :param boolan enable: Enable the public ip allocation :param str public_ip_id: GCP Public IP ID """ self.instance_data["gcpAssignPublicIP"] = enable self.instance_data["gcpPublicIPId"] = public_ip_id return self class FMNodeType(Enum): DESIGN = "design" DEPLOYER = "deployer" AUTOMATION = "automation" GOVERN = "govern" # Python2 emulated enum. to be removed on Python2 support removal @staticmethod def get_from_string(s): if s == "DESIGN": return FMNodeType.DESIGN if s == "DEPLOYER": return FMNodeType.DEPLOYER if s == "AUTOMATION": return FMNodeType.AUTOMATION if s == "GOVERN": return FMNodeType.GOVERN raise Exception("Invalid Node Type " + s) class FMInstanceEncryptionMode(Enum): NONE = "NONE" DEFAULT_KEY = "DEFAULT_KEY" TENANT = "TENANT" CUSTOM = "CUSTOM" class FMInstanceStatus(dict): """A class holding read-only information about an Instance. This class should not be created directly. Instead, use :meth:`FMInstance.get_status` """ def __init__(self, data): """Do not call this directly, use :meth:`FMInstance.get_status`""" super(FMInstanceStatus, self).__init__(data) class FMSnapshot(object): """ A handle to interact with a snapshot of a DSS instance. Do not create this directly, use :meth:`FMInstance.snapshot` """ def __init__(self, client, instance_id, snapshot_id, snapshot_data=None): self.client = client self.instance_id = instance_id self.snapshot_id = snapshot_id self.snapshot_data = snapshot_data def get_info(self): """ Get the information about this snapshot :return: a dict """ if self.snapshot_data is None: self.snapshot_data = self.client._perform_tenant_json( "GET", "/instances/%s/snapshots/%s" % (self.instance_id, self.snapshot_id) ) return self.snapshot_data def reprovision(self): """ Reprovision the physical DSS instance from this snapshot :return: the `Future` object representing the reprovision process :rtype: :class:`dataikuapi.fm.future.FMFuture` """ future = self.client._perform_tenant_json( "POST", "/instances/%s/snapshots/%s/reprovision" % (self.instance_id, self.snapshot_id) ) return FMFuture.from_resp(self.client, future) def delete(self): """ Delete the snapshot :return: the `Future` object representing the deletion process :rtype: :class:`dataikuapi.fm.future.FMFuture` """ future = self.client._perform_tenant_json( "DELETE", "/instances/%s/snapshots/%s" % (self.instance_id, self.snapshot_id) ) return FMFuture.from_resp(self.client, future) dataiku-api-client-python-11.4.0/dataikuapi/fm/instancesettingstemplates.py000066400000000000000000000433351440464736400272010ustar00rootroot00000000000000import json from .future import FMFuture import sys if sys.version_info > (3, 4): from enum import Enum else: class Enum(object): pass class FMInstanceSettingsTemplateCreator(object): def __init__(self, client, label): """ A builder class to create an Instance Settings Template :param str label: The label of the Virtual Network """ self.data = {} self.data["label"] = label self.client = client def create(self): """ Create a new instance settings template. :return: a newly created template :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMInstanceSettingsTemplate` """ template = self.client._perform_tenant_json( "POST", "/instance-settings-templates", body=self.data ) return FMInstanceSettingsTemplate(self.client, template) def with_setup_actions(self, setup_actions): """ Add setup actions :param list setup_actions: List of setup actions to be played on an instance :type setup_actions: list of :class:`dataikuapi.fm.instancesettingstemplates.FMSetupActions` :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMInstanceSettingsTemplateCreator` """ self.data["setupActions"] = setup_actions return self def with_license(self, license_file_path=None, license_string=None): """ Override global license :param str license_file_path: Optional, load the license from a json file :param str license_string: Optional, load the license from a json string :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMInstanceSettingsTemplateCreator` """ if license_file_path is not None: with open(license_file_path) as json_file: license = json.load(json_file) elif license_string is not None: license = json.loads(license_string) else: raise ValueError( "a valid license_file_path or license_string needs to be provided" ) self.data["license"] = json.dumps(license, indent=2) return self class FMAWSInstanceSettingsTemplateCreator(FMInstanceSettingsTemplateCreator): def with_aws_keypair(self, aws_keypair_name): """ Add an AWS Keypair to the DSS instance. Needed to get SSH access to the DSS instance, using the centos user. :param str aws_keypair_name: Name of an AWS key pair to add to the instance. :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMAWSInstanceSettingsTemplateCreator` """ self.data["awsKeyPairName"] = aws_keypair_name return self def with_startup_instance_profile(self, startup_instance_profile_arn): """ Add an Instance Profile to be assigned to the DSS instance on startup :param str startup_instance_profile_arn: ARN of the Instance profile assigned to the DSS instance at startup time :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMAWSInstanceSettingsTemplateCreator` """ self.data["startupInstanceProfileArn"] = startup_instance_profile_arn return self def with_runtime_instance_profile(self, runtime_instance_profile_arn): """ Add an Instance Profile to be assigned to the DSS instance when running :param str runtime_instance_profile_arn: ARN of the Instance profile assigned to the DSS instance during runtime :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMAWSInstanceSettingsTemplateCreator` """ self.data["runtimeInstanceProfileArn"] = runtime_instance_profile_arn return self def with_restrict_aws_metadata_server_access( self, restrict_aws_metadata_server_access=True ): """ Restrict AWS metadata server access on the DSS instance. :param boolean restrict_aws_metadata_server_access: Optional, If true, restrict the access to the metadata server access. Defaults to true :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMAWSInstanceSettingsTemplateCreator` """ self.data[ "restrictAwsMetadataServerAccess" ] = restrict_aws_metadata_server_access return self def with_default_aws_api_access_mode(self): """ The DSS Instance will use the Runtime Instance Profile to access AWS API. :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMAWSInstanceSettingsTemplateCreator` """ self.data["dataikuAwsAPIAccessMode"] = "NONE" return self def with_keypair_aws_api_access_mode( self, aws_access_key_id, aws_keypair_storage_mode="NONE", aws_secret_access_key=None, aws_secret_access_key_aws_secret_name=None, aws_secrets_manager_region=None, ): """ DSS Instance will use an Access Key to authenticate against the AWS API. :param str aws_access_key_id: AWS Access Key ID. :param str aws_keypair_storage_mode: Optional, the storage mode of the AWS api key. Accepts "NONE", "INLINE_ENCRYPTED" or "AWS_SECRETS_MANAGER". Defaults to "NONE" :param str aws_secret_access_key: Optional, AWS Access Key Secret. Only needed if keypair_storage_mode is "INLINE_ENCRYPTED" :param str aws_secret_access_key_aws_secret_name: Optional, ASM secret name. Only needed if aws_keypair_storage_mode is "AWS_SECRET_MANAGER" :param str aws_secrets_manager_region: Optional, Secret Manager region to use. Only needed if aws_keypair_storage_mode is "AWS_SECRET_MANAGER" :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMAWSInstanceSettingsTemplateCreator` """ if aws_keypair_storage_mode not in [ "NONE", "INLINE_ENCRYPTED", "AWS_SECRETS_MANAGER", ]: raise ValueError( 'aws_keypair_storage_mode should be either "NONE", "INLINE_ENCRYPTED" or "AWS_SECRET_MANAGER"' ) self.data["dataikuAwsAPIAccessMode"] = "KEYPAIR" self.data["dataikuAwsKeypairStorageMode"] = aws_keypair_storage_mode if aws_keypair_storage_mode == "NONE": return self self.data["dataikuAwsAccessKeyId"] = aws_access_key_id if aws_keypair_storage_mode == "INLINE_ENCRYPTED": if aws_secret_access_key == None: raise ValueError( 'When aws_keypair_storage_mode is "INLINE_ENCRYPTED", aws_secret_access_key should be provided' ) self.data["dataikuAwsSecretAccessKey"] = aws_secret_access_key elif aws_keypair_storage_mode == "AWS_SECRETS_MANAGER": if aws_secret_access_key_aws_secret_name == None: raise ValueError( 'When aws_keypair_storage_mode is "AWS_SECRETS_MANAGER", aws_secret_access_key_aws_secret_name should be provided' ) self.data[ "dataikuAwsSecretAccessKeyAwsSecretName" ] = aws_secret_access_key_aws_secret_name self.data["awsSecretsManagerRegion"] = aws_secrets_manager_region return self class FMAzureInstanceSettingsTemplateCreator(FMInstanceSettingsTemplateCreator): def with_ssh_key(self, ssh_public_key): """ Add an SSH public key to the DSS Instance. Needed to access it through SSH, using the centos user. :param str ssh_public_key: The content of the public key to add to the instance. :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMAzureInstanceSettingsTemplateCreator` """ self.data["azureSshKey"] = ssh_public_key return self def with_startup_managed_identity(self, startup_managed_identity): """ Add a managed identity to be assign to the DSS instance on startup :param str startup_managed_identity: Managed Identity ID :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMAzureInstanceSettingsTemplateCreator` """ self.data["startupManagedIdentity"] = startup_managed_identity return self def with_runtime_managed_identity(self, runtime_managed_identity): """ Add a managed identity to be assign to the DSS instance when running :param str runtime_managed_identity: Managed Identity ID :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMAzureInstanceSettingsTemplateCreator` """ self.data["runtimeManagedIdentity"] = runtime_managed_identity return self class FMGCPInstanceSettingsTemplateCreator(FMInstanceSettingsTemplateCreator): def with_ssh_key(self, ssh_public_key): """ Add an SSH public key to the DSS Instance. Needed to access it through SSH, using the centos user. :param str ssh_public_key: The content of the public key to add to the instance. :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMGCPInstanceSettingsTemplateCreator` """ self.data["gcpSshKey"] = ssh_public_key return self def with_restrict_metadata_server_access(self, restrict_metadata_server_access=True): """ Restrict GCloud metadata server access on the DSS instance. :param boolean restrict_metadata_server_access: Optional, If true, restrict the access to the metadata server access. Defaults to true :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMGCPInstanceSettingsTemplateCreator` """ self.data["restrictGcpMetadataServerAccess"] = restrict_metadata_server_access return self def with_block_project_wide_keys(self, block_project_wide_keys=True): """ Restrict GCloud metadata server access on the DSS instance. :param boolean block_project_wide_keys: Optional, If true, block project-wide ssh keys on the instance. Defaults to true :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMGCPInstanceSettingsTemplateCreator` """ self.data["gcpBlockProjectWideKeys"] = block_project_wide_keys return self def with_runtime_service_account(self, startup_service_account): """ Add a service account to be assigned to the DSS instance on startup :param str startup_service_account: service account email :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMGCPInstanceSettingsTemplateCreator` """ self.data["startupServiceAccount"] = startup_service_account return self class FMInstanceSettingsTemplate(object): def __init__(self, client, ist_data): self.client = client self.id = ist_data["id"] self.ist_data = ist_data def save(self): """ Update this template """ self.client._perform_tenant_empty( "PUT", "/instance-settings-templates/%s" % self.id, body=self.ist_data ) self.ist_data = self.client._perform_tenant_json( "GET", "/instance-settings-templates/%s" % self.id ) def delete(self): """ Delete this template :return: the `Future` object representing the deletion process :rtype: :class:`dataikuapi.fm.future.FMFuture` """ future = self.client._perform_tenant_json( "DELETE", "/instance-settings-templates/%s" % self.id ) return FMFuture.from_resp(self.client, future) def add_setup_action(self, setup_action): """ Add a setup action :param setup_action: the action to add :type setup_action: :class:`dataikuapi.fm.instancesettingstemplates.FMSetupAction` :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMInstanceSettingsTemplate` """ self.ist_data["setupActions"].append(setup_action) return self class FMSetupAction(dict): def __init__(self, setupActionType, params=None): """ A class representing a setup action Do not create this directly, use the static methods in this class, for example: - :meth:`dataikuapi.fm.instancesettingstemplates.FMSetupAction.add_authorized_key` """ data = { "type": setupActionType.value, } if params: data["params"] = params super(FMSetupAction, self).__init__(data) @staticmethod def add_authorized_key(ssh_key): """ Return an ADD_AUTHORIZED_KEY setup action :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMSetupAction` """ return FMSetupAction(FMSetupActionType.ADD_AUTHORIZED_KEY, {"sshKey": ssh_key}) @staticmethod def run_ansible_task(stage, yaml_string): """ Return a RUN_ANSIBLE_TASK setup action :param stage: the action stage :type stage: :class:`dataikuapi.fm.instancesettingstemplates.FMSetupActionStage` :param str yaml_string: a yaml encoded string defining the ansibles tasks to run :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMSetupAction` """ return FMSetupAction( FMSetupActionType.RUN_ANSIBLE_TASKS, {"stage": stage.value, "ansibleTasks": yaml_string}, ) @staticmethod def install_system_packages(packages): """ Return an INSTALL_SYSTEM_PACKAGES setup action :param list packages: List of packages to install :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMSetupAction` """ return FMSetupAction( FMSetupActionType.INSTALL_SYSTEM_PACKAGES, {"packages": packages} ) @staticmethod def setup_advanced_security(basic_headers=True, hsts=False): """ Return a SETUP_ADVANCED_SECURITY setup action :param boolean basic_headers: Optional, Prevent browsers to render Web content served by DSS to be embedded into a frame, iframe, embed or object tag. Defaults to True :param boolean hsts: Optional, Enforce HTTP Strict Transport Security. Defaults to False :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMSetupAction` """ return FMSetupAction( FMSetupActionType.SETUP_ADVANCED_SECURITY, {"basic_headers": basic_headers, "hsts": hsts}, ) @staticmethod def install_jdbc_driver( database_type, url, paths_in_archive=None, http_headers=None, http_username=None, http_password=None, datadir_subdirectory=None, ): """ Return a INSTALL_JDBC_DRIVER setup action :param database_type: the database type :type database_type: :class:`dataikuapi.fm.instancesettingstemplates.FMSetupActionAddJDBCDriverDatabaseType` :param str url: The full address to the driver. Supports http(s)://, s3://, abs:// or file:// endpoints :param list paths_in_archive: Optional, must be used when the driver is shipped as a tarball or a ZIP file. Add here all the paths to find the JAR files in the driver archive. Paths are relative to the top of the archive. Wildcards are supported. :param dict http_headers: Optional, If you download the driver from a HTTP(S) endpoint, add here the headers you want to add to the query. This setting is ignored for any other type of download. :param str http_username: Optional, If the HTTP(S) endpoint expect a Basic Authentication, add here the username. To explicitely specify which Assigned Identity use if the machine have several, set the client_id here. To authenticate with a SAS Token on Azure Blob Storage (not recommended), use "token" as the value here. :param str http_password: Optional, If the HTTP(S) endpoint expect a Basic Authentication, add here the password. To authenticate with a SAS Token on Azure Blob Storage (not recommended), store the token in this field. :param str datadir_subdirectory: Optional, Some drivers are shipped with a high number of JAR files along with them. In that case, you might want to install them under an additional level in the DSS data directory. Set the name of this subdirectory here. Not required for most drivers. :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMSetupAction` """ return FMSetupAction( FMSetupActionType.INSTALL_JDBC_DRIVER, { "url": url, "dbType": database_type.value, "pathsInArchive": paths_in_archive, "headers": http_headers, "username": http_username, "password": http_password, "subpathInDatadir": datadir_subdirectory, }, ) @staticmethod def setup_k8s_and_spark(): """ Return a SETUP_K8S_AND_SPARK setup action :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMSetupAction` """ return FMSetupAction(FMSetupActionType.SETUP_K8S_AND_SPARK) class FMSetupActionType(Enum): RUN_ANSIBLE_TASKS = "RUN_ANSIBLE_TASKS" INSTALL_SYSTEM_PACKAGES = "INSTALL_SYSTEM_PACKAGES" INSTALL_DSS_PLUGINS_FROM_STORE = "INSTALL_DSS_PLUGINS_FROM_STORE" SETUP_K8S_AND_SPARK = "SETUP_K8S_AND_SPARK" SETUP_RUNTIME_DATABASE = "SETUP_RUNTIME_DATABASE" SETUP_MUS_AUTOCREATE = "SETUP_MUS_AUTOCREATE" SETUP_UI_CUSTOMIZATION = "SETUP_UI_CUSTOMIZATION" SETUP_MEMORY_SETTINGS = "SETUP_MEMORY_SETTINGS" SETUP_GRAPHICS_EXPORT = "SETUP_GRAPHICS_EXPORT" ADD_AUTHORIZED_KEY = "ADD_AUTHORIZED_KEY" INSTALL_JDBC_DRIVER = "INSTALL_JDBC_DRIVER" SETUP_ADVANCED_SECURITY = "SETUP_ADVANCED_SECURITY" class FMSetupActionStage(Enum): after_dss_startup = "after_dss_startup" after_install = "after_install" before_install = "before_install" class FMSetupActionAddJDBCDriverDatabaseType(Enum): mysql = "mysql" mssql = "mssql" oracle = "oracle" mariadb = "mariadb" snowflake = "snowflake" athena = "athena" bigquery = "bigquery" dataiku-api-client-python-11.4.0/dataikuapi/fm/tenant.py000066400000000000000000000121621440464736400231600ustar00rootroot00000000000000import json class FMCloudCredentials(object): """ A Tenant Cloud Credentials in the FM instance """ def __init__(self, client, cloud_credentials): self.client = client self.cloud_credentials = cloud_credentials def set_cmk_key(self, cmk_key_id): self.cloud_credentials["awsCMKId"] = cmk_key_id return self def set_static_license(self, license_file_path=None, license_string=None): """ Set a default static license for the DSS instances :param str license_file_path: Optional, load the license from a json file :param str license_string: Optional, load the license from a json string """ if license_file_path is not None: with open(license_file_path) as json_file: license = json.load(json_file) elif license_string is not None: license = json.loads(license_string) else: raise ValueError( "a valid license_file_path or license_string needs to be provided" ) self.cloud_credentials["licenseMode"] = "STATIC" self.cloud_credentials["license"] = json.dumps(license, indent=2) return self def set_automatically_updated_license(self, license_token): """ Set an automatically updated license for the DSS instances :param str license_token: License token """ if license_token is None: raise ValueError("a valid license_token needs to be provided") self.cloud_credentials["licenseMode"] = "AUTO_UPDATE" self.cloud_credentials["licenseToken"] = license_token return self def set_authentication(self, authentication): """ Set the authentication for the tenant :param authentication: the authentication to be used :type authentication: :class:`dataikuapi.fm.tenant.FMCloudAuthentication` """ self.cloud_credentials.update(authentication) return self def save(self): """ Saves back the settings to the project """ self.client._perform_tenant_empty( "PUT", "/cloud-credentials", body=self.cloud_credentials ) class FMCloudTags(object): """ A Tenant Cloud Tags in the FM instance """ def __init__(self, client, cloud_tags): self.client = client self.cloud_tags = json.loads(cloud_tags["msg"]) @property def tags(self): return self.cloud_tags def save(self): """ Saves the tags on FM """ self.client._perform_tenant_empty("PUT", "/cloud-tags", body=self.cloud_tags) class FMCloudAuthentication(dict): def __init__(self, data): """ A class holding the Cloud Authentication information Do not create this directly, use: - :meth:`dataikuapi.fm.tenant.FMCloudAuthentication.aws_same_as_fm` to use the same authentication as Fleet Manager - :meth:`dataikuapi.fm.tenant.FMCloudAuthentication.aws_iam_role` to use a custom IAM Role - :meth:`dataikuapi.fm.tenant.FMCloudAuthentication.aws_keypair` to use a AWS Access key ID and AWS Secret Access Key pair """ super(FMCloudAuthentication, self).__init__(data) @staticmethod def aws_same_as_fm(): """ AWS Only: use the same authentication as Fleet Manager """ return FMCloudAuthentication({"awsAuthenticationMode": "SAME_AS_FM"}) @staticmethod def aws_iam_role(role_arn): """ AWS Only: use an IAM Role :param str role_arn: ARN of the IAM Role """ return FMCloudAuthentication( {"awsAuthenticationMode": "IAM_ROLE", "awsIAMRoleARN": role_arn} ) @staticmethod def aws_keypair(access_key_id, secret_access_key): """ AWS Only: use an AWS Access Key :param str access_key_id: AWS Access Key ID :param str secret_access_key: AWS Secret Access Key """ return FMCloudAuthentication( { "awsAuthenticationMode": "KEYPAIR", "awsAccessKeyId": access_key_id, "awsSecretAccessKey": secret_access_key, } ) @staticmethod def azure(subscription, tenant_id, environment, client_id): """ Azure Only :param str subscription: Azure Subscription :param str tenant_id: Azure Tenant Id :param str environment: Azure Environment :param str client_id: Azure Client Id """ data = { "azureSubscription": subscription, "azureTenantId": tenant_id, "azureEnvironment": environment, "azureFMAppClientId": client_id, } return FMCloudAuthentication(data) @staticmethod def gcp(project_id, service_account_key): """ GCP Only :param str project_id: GCP project :param str service_account_key: Optional, service account key (JSON) """ data = { "gcpProjectId": project_id, "gcpServiceAccountKey": service_account_key } return FMCloudAuthentication(data) dataiku-api-client-python-11.4.0/dataikuapi/fm/virtualnetworks.py000066400000000000000000000324541440464736400251600ustar00rootroot00000000000000from .future import FMFuture class FMVirtualNetworkCreator(object): def __init__(self, client, label): """ A builder class to create virtual networks :param str label: The label of the virtual network """ self.client = client self.data = {} self.use_default_values = False self.data["label"] = label self.data["mode"] = "EXISTING_MONOTENANT" def with_internet_access_mode(self, internet_access_mode): """ Set the Internet access mode :param str internet_access_mode: The internet access mode of the instances created in this virtual network. Accepts "YES", "NO", "EGRESS_ONLY". Defaults to "YES" """ if internet_access_mode not in ["YES", "NO", "EGRESS_ONLY"]: raise ValueError( 'internet_access_mode should be either "YES", "NO", or "EGRESS_ONLY"' ) self.data["internetAccessMode"] = internet_access_mode return self def with_default_values(self): """ Set the VPC and Subnet to their default values: the vpc and subnet of the FM instance """ self.use_default_values = True return self class FMAWSVirtualNetworkCreator(FMVirtualNetworkCreator): def with_vpc(self, aws_vpc_id, aws_subnet_id): """ Setup the VPC and Subnet to be used by the virtual network :param str aws_vpc_id: ID of the VPC to use :param str aws_subnet_id: ID of the subnet to use """ self.data["awsVpcId"] = aws_vpc_id self.data["awsSubnetId"] = aws_subnet_id return self def with_auto_create_security_groups(self): """ Automatically create the AWS Security Groups when creating this virtual network """ self.data["awsAutoCreateSecurityGroups"] = True return self def with_aws_security_groups(self, *aws_security_groups): """ Use pre-created AWS Security Groups :param str aws_security_groups: Up to 5 security group ids to assign to the instances created in this virtual network """ self.data["awsAutoCreateSecurityGroups"] = False self.data["awsSecurityGroups"] = aws_security_groups return self def create(self): """ Create a new virtual network :return: a newly created network :rtype: :class:`dataikuapi.fm.virtualnetworks.FMAWSVirtualNetwork` """ vn = self.client._perform_tenant_json( "POST", "/virtual-networks", body=self.data, params={ 'useDefaultValues':self.use_default_values } ) return FMAWSVirtualNetwork(self.client, vn) class FMAzureVirtualNetworkCreator(FMVirtualNetworkCreator): def with_azure_virtual_network(self, azure_vn_id, azure_subnet_id): """ Setup the Azure Virtual Network and Subnet to be used by the virtual network :param str azure_vn_id: Resource ID of the Azure Virtual Network to use :param str azure_subnet_id: Resource ID of the subnet to use """ self.data["azureVnId"] = azure_vn_id self.data["azureSubnetId"] = azure_subnet_id return self def with_auto_update_security_groups(self, auto_update_security_groups=True): """ Auto update the security groups of the Azure Virtual Network :param boolean auto_update_security_groups: Optional, Auto update the subnet security group. Defaults to True """ self.data["azureAutoUpdateSecurityGroups"] = auto_update_security_groups return self def create(self): """ Create a new virtual network :return: a newly created network :rtype: :class:`dataikuapi.fm.virtualnetworks.FMAzureVirtualNetwork` """ vn = self.client._perform_tenant_json( "POST", "/virtual-networks", body=self.data, params={ 'useDefaultValues':self.use_default_values } ) return FMAzureVirtualNetwork(self.client, vn) class FMGCPVirtualNetworkCreator(FMVirtualNetworkCreator): def with_vpc(self, project_id, network, subnetwork): """ Setup the VPC which the virtual network will use :param str project_id: ID of the project in which the network is defined :param str network: name of the network :param str subnetwork: name of the subnetwork """ self.data["gcpProjectId"] = project_id self.data["gcpNetwork"] = network self.data["gcpSubnetwork"] = subnetwork return self def with_network_tags(self, *network_tags): """ Use network tags on the instances created in the virtual network :param str network_tags: network tags to assign to the instances created in this virtual network. """ self.data["gcpNetworkTags"] = network_tags return self def create(self): """ Create the virtual network :return: a newly created network :rtype: :class:`dataikuapi.fm.virtualnetworks.FMGCPVirtualNetwork` """ vn = self.client._perform_tenant_json( "POST", "/virtual-networks", body=self.data, params={ 'useDefaultValues':self.use_default_values } ) return FMGCPVirtualNetwork(self.client, vn) class FMVirtualNetwork(object): def __init__(self, client, vn_data): self.client = client self.vn_data = vn_data self.id = self.vn_data["id"] def save(self): """ Update this virtual network. """ self.client._perform_tenant_empty( "PUT", "/virtual-networks/%s" % self.id, body=self.vn_data ) self.vn_data = self.client._perform_tenant_json( "GET", "/virtual-networks/%s" % self.id ) def delete(self): """ Delete this virtual network. :return: the `Future` object representing the deletion process :rtype: :class:`dataikuapi.fm.future.FMFuture` """ future = self.client._perform_tenant_json( "DELETE", "/virtual-networks/%s" % self.id ) return FMFuture.from_resp(self.client, future) def set_fleet_management( self, enable, event_server=None, deployer_management="NO_MANAGED_DEPLOYER", govern_server=None ): """ When enabled, all instances in this virtual network know each other and can centrally manage deployer and logs centralization :param boolean enable: Enable or not the Fleet Management :param str event_server: Optional, Node name of the node that should act as the centralized event server for logs concentration. Audit logs of all design, deployer and automation nodes will automatically be sent there. :param str deployer_management: Optional, Accepts: - "NO_MANAGED_DEPLOYER": Do not manage the deployer. This is the default mode. - "CENTRAL_DEPLOYER": Central deployer. Recommended if you have more than one design node or may have more than one design node in the future. - "EACH_DESIGN_NODE": Deployer from design. Recommended if you have a single design node and want a simpler setup. :param str govern_server: Optional, node name of the node that should act as the centralized Govern server. """ self.vn_data["managedNodesDirectory"] = enable self.vn_data["eventServerNodeLabel"] = event_server self.vn_data["nodesDirectoryDeployerMode"] = deployer_management self.vn_data["governServerNodeLabel"] = govern_server return self def set_https_strategy(self, https_strategy): """ Set the HTTPS strategy for this virtual network :param https_strategy: the strategy to set :type https_strategy: :class:`dataikuapi.fm.virtualnetworks.FMHTTPSStrategy` """ self.vn_data.update(https_strategy) return self class FMAWSVirtualNetwork(FMVirtualNetwork): def set_dns_strategy( self, assign_domain_name, aws_private_ip_zone53_id=None, aws_public_ip_zone53_id=None, ): """ Set the DNS strategy for this virtual network :param boolean assign_domain_name: If false, don't assign domain names, use ip_only :param str aws_private_ip_zone53_id: Optional, AWS Only, the ID of the AWS Route53 Zone to use for private ip :param str aws_public_ip_zone53_id: Optional, AWS Only, the ID of the AWS Route53 Zone to use for public ip """ if assign_domain_name: self.vn_data["dnsStrategy"] = "VN_SPECIFIC_CLOUD_DNS_SERVICE" self.vn_data["awsRoute53PrivateIPZoneId"] = aws_private_ip_zone53_id self.vn_data["awsRoute53PublicIPZoneId"] = aws_public_ip_zone53_id else: self.vn_data["dnsStrategy"] = "NONE" return self class FMAzureVirtualNetwork(FMVirtualNetwork): def set_dns_strategy(self, assign_domain_name, azure_dns_zone_id=None): """ Set the DNS strategy for this virtual network :param boolean assign_domain_name: If false, don't assign domain names, use ip_only :param str azure_dns_zone_id: Optional, Azure Only, the ID of the Azure DNS zone to use """ if assign_domain_name: self.vn_data["dnsStrategy"] = "VN_SPECIFIC_CLOUD_DNS_SERVICE" self.vn_data["azureDnsZoneId"] = azure_dns_zone_id else: self.vn_data["dnsStrategy"] = "NONE" return self class FMGCPVirtualNetwork(FMVirtualNetwork): def set_assign_public_ip(self, public_ip=True): """ Sets whether the instances on this network will have a publicly accessible IP :param bool public_ip: if False, the instances will not be accessible from outside GCP """ if public_ip is not None: self.vn_data["gcpAssignPublicIP"] = public_ip return self def set_location_for_created_resources(self, project_id=None, zone=None): """ Set the location in GCP of the instances created using this virtual network :param str project_id: Optional, the project in which instances should be created. If empty string, then the project of the FM instance is used :param str zone: Optional, the zone in which instances should be created. If empty string, then the zone of the FM instance is used """ if zone == '': self.vn_data["gcpZone"] = None elif zone is not None: self.vn_data["gcpZone"] = zone if project_id == '': self.vn_data["gcpProjectIdForCreatedResources"] = None elif project_id is not None: self.vn_data["gcpProjectIdForCreatedResources"] = project_id return self def set_dns_strategy(self, assign_domain_name, private_ip_zone_id=None,public_ip_zone_id=None): """ Set the DNS strategy for this virtual network :param boolean assign_domain_name: If false, don't assign domain names, use ip_only :param str private_ip_zone_id: Optional, the ID of the Cloud DNS Zone to use for private ip :param str public_ip_zone_id: Optional, the ID of the Cloud DNS Zone to use for public ip """ if assign_domain_name: self.vn_data["dnsStrategy"] = "VN_SPECIFIC_CLOUD_DNS_SERVICE" self.vn_data["gcpCloudDnsPrivateIPZoneId"] = private_ip_zone_id self.vn_data["gcpCloudDnsPublicIPZoneId"] = public_ip_zone_id else: self.vn_data["dnsStrategy"] = "NONE" return self class FMHTTPSStrategy(dict): def __init__(self, data, https_strategy, http_redirect=False): """ A class holding HTTPS Strategy for virtual networks Do not create this directly, use: - :meth:`dataikuapi.fm.virtualnetwork.FMHTTPSStrategy.disable` to use HTTP only - :meth:`dataikuapi.fm.virtualnetwork.FMHTTPSStrategy.self_signed` to use self-signed certificates - :meth:`dataikuapi.fm.virtualnetwork.FMHTTPSStrategy.custom_cert` to use custom certificates - :meth:`dataikuapi.fm.virtualnetwork.FMHTTPSStrategy.lets_encrypt` to use Let's Encrypt """ super(FMHTTPSStrategy, self).__init__(data) self["httpsStrategy"] = https_strategy if http_redirect: self["httpStrategy"] = "REDIRECT" else: self["httpStrategy"] = "DISABLE" @staticmethod def disable(): """ Use HTTP only """ return FMHTTPSStrategy({}, "NONE", False) @staticmethod def self_signed(http_redirect): """ Use self-signed certificates :param bool http_redirect: If true, HTTP is redirected to HTTPS. If false, HTTP is disabled. Defaults to false """ return FMHTTPSStrategy({}, "SELF_SIGNED", http_redirect) @staticmethod def custom_cert(http_redirect): """ Use a custom certificate for each instance :param bool http_redirect: If true, HTTP is redirected to HTTPS. If false, HTTP is disabled. Defaults to false """ return FMHTTPSStrategy({}, "CUSTOM_CERTIFICATE", http_redirect) @staticmethod def lets_encrypt(contact_mail): """ Use Let's Encrypt to generate https certificates :param str contact_mail: The contact email provided to Let's Encrypt """ return FMHTTPSStrategy({"contactMail": contact_mail}, "LETSENCRYPT", True) dataiku-api-client-python-11.4.0/dataikuapi/fmclient.py000066400000000000000000000354701440464736400230750ustar00rootroot00000000000000import json from requests import Session from requests import exceptions from requests.auth import HTTPBasicAuth import os.path as osp from .utils import DataikuException from .fm.tenant import FMCloudCredentials, FMCloudTags from .fm.virtualnetworks import ( FMVirtualNetwork, FMAWSVirtualNetworkCreator, FMAzureVirtualNetworkCreator, FMGCPVirtualNetworkCreator, FMAWSVirtualNetwork, FMAzureVirtualNetwork, FMGCPVirtualNetwork ) from .fm.instances import ( FMInstance, FMInstanceEncryptionMode, FMAWSInstanceCreator, FMAzureInstanceCreator, FMGCPInstanceCreator, FMAWSInstance, FMAzureInstance, FMGCPInstance ) from .fm.instancesettingstemplates import ( FMInstanceSettingsTemplate, FMAWSInstanceSettingsTemplateCreator, FMAzureInstanceSettingsTemplateCreator, FMGCPInstanceSettingsTemplateCreator ) import sys if sys.version_info > (3, 4): from enum import Enum else: class Enum(object): pass class FMClient(object): def __init__( self, host, api_key_id, api_key_secret, tenant_id="main", extra_headers=None, ): """ Base class for the different FM Clients Do not create this class, instead use :class:`dataikuapi.FMClientAWS`, :class:`dataikuapi.FMClientAzure` or :class:`dataikuapi.FMClientGCP` """ if self.cloud == None: raise NotImplementedError( "Do not use FMClient directly, instead use FMClientAWS, FMClientAzure or FMClientGCP" ) self.api_key_id = api_key_id self.api_key_secret = api_key_secret self.host = host self.__tenant_id = tenant_id self._session = Session() if self.api_key_id is not None and self.api_key_secret is not None: self._session.auth = HTTPBasicAuth(self.api_key_id, self.api_key_secret) else: raise ValueError("API Key ID and API Key secret are required") if extra_headers is not None: self._session.headers.update(extra_headers) ######################################################## # Tenant ######################################################## def get_cloud_credentials(self): """ Get the cloud credentials :return: cloud credentials :rtype: :class:`dataikuapi.fm.tenant.FMCloudCredentials` """ creds = self._perform_tenant_json("GET", "/cloud-credentials") return FMCloudCredentials(self, creds) def get_cloud_tags(self): """ Get the tenant's cloud tags :return: tenant's cloud tags :rtype: :class:`dataikuapi.fm.tenant.FMCloudTags` """ tags = self._perform_tenant_json("GET", "/cloud-tags") return FMCloudTags(self, tags) ######################################################## # VirtualNetwork ######################################################## def _make_virtual_network(self, vn): if self.cloud == "AWS": return FMAWSVirtualNetwork(self, vn) elif self.cloud == "Azure": return FMAzureVirtualNetwork(self, vn) elif self.cloud == "GCP": return FMGCPVirtualNetwork(self, vn) else: raise Exception("Unknown cloud type %s" % self.cloud) def list_virtual_networks(self): """ List all virtual networks :return: list of virtual networks :rtype: list of :class:`dataikuapi.fm.virtualnetworks.FMVirtualNetwork` """ vns = self._perform_tenant_json("GET", "/virtual-networks") return [self._make_virtual_network(x) for x in vns] def get_virtual_network(self, virtual_network_id): """ Get a virtual network by its id :param str virtual_network_id: the id of the network to retrieve :return: the requested virtual network :rtype: :class:`dataikuapi.fm.virtualnetworks.FMVirtualNetwork` """ vn = self._perform_tenant_json( "GET", "/virtual-networks/%s" % virtual_network_id ) return self._make_virtual_network(vn) ######################################################## # Instance settings template ######################################################## def list_instance_templates(self): """ List all instance settings templates :return: list of instance settings template :rtype: list of :class:`dataikuapi.fm.tenant.FMInstanceSettingsTemplate` """ templates = self._perform_tenant_json("GET", "/instance-settings-templates") return [FMInstanceSettingsTemplate(self, x) for x in templates] def get_instance_template(self, template_id): """ Get an instance setting template template by its id :param str template_id: the id of the template to retrieve :return: the requested instance settings template :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMInstanceSettingsTemplate` """ template = self._perform_tenant_json( "GET", "/instance-settings-templates/%s" % template_id ) return FMInstanceSettingsTemplate(self, template) ######################################################## # Instance ######################################################## def _make_instance(self, i): if self.cloud == "AWS": return FMAWSInstance(self, i) elif self.cloud == "Azure": return FMAzureInstance(self, i) elif self.cloud == "GCP": return FMGCPInstance(self, i) else: raise Exception("Unknown cloud type %s" % self.cloud) def list_instances(self): """ List all DSS instances :return: list of instances :rtype: list of :class:`dataikuapi.fm.instances.FMInstance` """ instances = self._perform_tenant_json("GET", "/instances") return [self._make_instance(x) for x in instances] def get_instance(self, instance_id): """ Get a DSS instance by its id :param str instance_id: the id of the instance to retrieve :return: the requested instance if any :rtype: :class:`dataikuapi.fm.instances.FMInstance` """ instance = self._perform_tenant_json("GET", "/instances/%s" % instance_id) return self._make_instance(instance) def list_instance_images(self): """ List all available images to create new instances :return: list of images, as a pair of id and label """ return self._perform_tenant_json("GET", "/images") ######################################################## # Internal Request handling ######################################################## def _perform_http( self, method, path, params=None, body=None, stream=False, files=None, raw_body=None, ): if body is not None: body = json.dumps(body) if raw_body is not None: body = raw_body try: http_res = self._session.request( method, "%s/api/public%s" % (self.host, path), params=params, data=body, files=files, stream=stream, ) http_res.raise_for_status() return http_res except exceptions.HTTPError: try: ex = http_res.json() except ValueError: ex = {"message": http_res.text} raise DataikuException( "%s: %s" % ( ex.get("errorType", "Unknown error"), ex.get("message", "No message"), ) ) def _perform_empty( self, method, path, params=None, body=None, files=None, raw_body=None ): self._perform_http( method, path, params=params, body=body, files=files, stream=False, raw_body=raw_body, ) def _perform_json( self, method, path, params=None, body=None, files=None, raw_body=None ): return self._perform_http( method, path, params=params, body=body, files=files, stream=False, raw_body=raw_body, ).json() def _perform_tenant_json( self, method, path, params=None, body=None, files=None, raw_body=None ): return self._perform_json( method, "/tenants/%s%s" % (self.__tenant_id, path), params=params, body=body, files=files, raw_body=raw_body, ) def _perform_tenant_empty( self, method, path, params=None, body=None, files=None, raw_body=None ): self._perform_empty( method, "/tenants/%s%s" % (self.__tenant_id, path), params=params, body=body, files=files, raw_body=raw_body, ) class FMClientAWS(FMClient): def __init__( self, host, api_key_id, api_key_secret, tenant_id="main", extra_headers=None, ): """ AWS Only - Instantiate a new FM API client on the given host with the given API key. The API key will define which operations are allowed for the client. :param str host: Full url of the FM """ self.cloud = "AWS" super(FMClientAWS, self).__init__( host, api_key_id, api_key_secret, tenant_id, extra_headers ) def new_virtual_network_creator(self, label): """ Instantiate a new virtual network creator :param str label: The label of the network :rtype: :class:`dataikuapi.fm.virtualnetworks.FMAWSVirtualNetworkCreator` """ return FMAWSVirtualNetworkCreator(self, label) def new_instance_template_creator(self, label): """ Instantiate a new instance template creator :param str label: The label of the instance :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMAWSInstanceSettingsTemplateCreator` """ return FMAWSInstanceSettingsTemplateCreator(self, label) def new_instance_creator( self, label, instance_settings_template_id, virtual_network_id, image_id ): """ Instantiate a new instance creator :param str label: The label of the instance :param str instance_settings_template: The instance settings template id this instance should be based on :param str virtual_network: The virtual network where the instance should be spawned :param str image_id: The ID of the DSS runtime image (ex: dss-9.0.3-default) :rtype: :class:`dataikuapi.fm.instances.FMAWSInstanceCreator` """ return FMAWSInstanceCreator( self, label, instance_settings_template_id, virtual_network_id, image_id ) class FMClientAzure(FMClient): def __init__( self, host, api_key_id, api_key_secret, tenant_id="main", extra_headers=None, ): """ Azure Only - Instantiate a new FM API client on the given host with the given API key. The API key will define which operations are allowed for the client. :param str host: Full url of the FM """ self.cloud = "Azure" super(FMClientAzure, self).__init__( host, api_key_id, api_key_secret, tenant_id, extra_headers ) def new_virtual_network_creator(self, label): """ Instantiate a new virtual network creator :param str label: The label of the network :rtype: :class:`dataikuapi.fm.virtualnetworks.FMAzureVirtualNetworkCreator` """ return FMAzureVirtualNetworkCreator(self, label) def new_instance_template_creator(self, label): """ Instantiate a new instance template creator :param str label: The label of the instance :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMAzureInstanceSettingsTemplateCreator` """ return FMAzureInstanceSettingsTemplateCreator(self, label) def new_instance_creator( self, label, instance_settings_template_id, virtual_network_id, image_id ): """ Instantiate a new instance creator :param str label: The label of the instance :param str instance_settings_template: The instance settings template id this instance should be based on :param str virtual_network: The virtual network where the instance should be spawned :param str image_id: The ID of the DSS runtime image (ex: dss-9.0.3-default) :rtype: :class:`dataikuapi.fm.instances.FMAzureInstanceCreator` """ return FMAzureInstanceCreator( self, label, instance_settings_template_id, virtual_network_id, image_id ) class FMClientGCP(FMClient): def __init__( self, host, api_key_id, api_key_secret, tenant_id="main", extra_headers=None, ): """ GCP Only - Instantiate a new FM API client on the given host with the given API key. The API key will define which operations are allowed for the client. :param str host: Full url of the FM """ self.cloud = "GCP" super(FMClientGCP, self).__init__( host, api_key_id, api_key_secret, tenant_id, extra_headers ) def new_virtual_network_creator(self, label): """ Instantiate a new virtual network creator :param str label: The label of the network :rtype: :class:`dataikuapi.fm.virtualnetworks.FMGCPVirtualNetworkCreator` """ return FMGCPVirtualNetworkCreator(self, label) def new_instance_template_creator(self, label): """ Instantiate a new instance template creator :param str label: The label of the instance :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMGCPInstanceSettingsTemplateCreator` """ return FMGCPInstanceSettingsTemplateCreator(self, label) def new_instance_creator( self, label, instance_settings_template_id, virtual_network_id, image_id ): """ Instantiate a new instance creator :param str label: The label of the instance :param str instance_settings_template: The instance settings template id this instance should be based on :param str virtual_network: The virtual network where the instance should be spawned :param str image_id: The ID of the DSS runtime image (ex: dss-9.0.3-default) :rtype: :class:`dataikuapi.fm.instances.FMGCPInstanceCreator` """ return FMGCPInstanceCreator( self, label, instance_settings_template_id, virtual_network_id, image_id ) dataiku-api-client-python-11.4.0/dataikuapi/utils.py000066400000000000000000000067071440464736400224350ustar00rootroot00000000000000import csv, sys from dateutil import parser as date_iso_parser from contextlib import closing import os import zipfile import itertools if sys.version_info > (3,0): import codecs dku_basestring_type = str dku_zip_longest = itertools.zip_longest else: dku_basestring_type = basestring dku_zip_longest = itertools.izip_longest class DataikuException(Exception): """Exception launched by the Dataiku API clients when an error occurs""" class DataikuUTF8CSVReader(object): """ A CSV reader which will iterate over lines in the CSV file "f", which is encoded in UTF-8. """ def __init__(self, f, **kwds): self.reader = csv.reader(f, **kwds) def next(self): row = self.reader.next() return [unicode(s, "utf-8") for s in row] def __iter__(self): return self def none_if_throws(f): def aux(*args, **kargs): try: return f(*args, **kargs) except: return None return aux class DataikuStreamedHttpUTF8CSVReader(object): """ A CSV reader with a schema """ def __init__(self, schema, csv_stream): self.schema = schema self.csv_stream = csv_stream def iter_rows(self): def decode(x): if sys.version_info > (3,0): return x else: return unicode(x, "utf8") def parse_iso_date(s): if s == "": return None else: return date_iso_parser.parse(s) def str_to_bool(s): if s is None: return False return s.lower() == "true" CASTERS = { "tinyint" : int, "smallint" : int, "int": int, "bigint": int, "float": float, "double": float, "date": parse_iso_date, "boolean": str_to_bool, } schema = self.schema casters = [ CASTERS.get(col["type"], decode) for col in schema ] with closing(self.csv_stream) as r: if sys.version_info > (3,0): raw_generator = codecs.iterdecode(r.raw, 'utf-8') else: raw_generator = r.raw for uncasted_tuple in csv.reader(raw_generator, delimiter='\t', quotechar='"', doublequote=True): yield [none_if_throws(caster)(val) for (caster, val) in dku_zip_longest(casters, uncasted_tuple)] class CallableStr(str): def __init__(self, val): self.val = val def __call__(self): return self.val def _make_zipfile(output_filename, source_dir): relroot = os.path.abspath(os.path.join(source_dir)) with zipfile.ZipFile(output_filename, "w", zipfile.ZIP_DEFLATED) as zipfp: for root, dirs, files in os.walk(source_dir): for file in files: filename = os.path.join(root, file) if os.path.isfile(filename): arcname = os.path.join(os.path.relpath(root, relroot), file) zipfp.write(filename, arcname) return output_filename def _write_response_content_to_file(response, path): with open(path, 'wb') as f: for chunk in response.iter_content(chunk_size=10000): if chunk: f.write(chunk) f.flush() dataiku-api-client-python-11.4.0/meta/000077500000000000000000000000001440464736400175235ustar00rootroot00000000000000dataiku-api-client-python-11.4.0/meta/main.yml000066400000000000000000000003211440464736400211660ustar00rootroot00000000000000--- galaxy_info: author: Dataiku description: Dataiku API Client company: Dataiku license: Apache 2.0 min_ansible_version: 2.5 galaxy_tags: ["dataiku","dss","data-science-studio"] dependencies: [] dataiku-api-client-python-11.4.0/module_utils/000077500000000000000000000000001440464736400213025ustar00rootroot00000000000000dataiku-api-client-python-11.4.0/module_utils/dataikuapi000077700000000000000000000000001440464736400256062../dataikuapiustar00rootroot00000000000000dataiku-api-client-python-11.4.0/requirements.txt000066400000000000000000000000431440464736400220560ustar00rootroot00000000000000requests<2.22 python-dateutil nose dataiku-api-client-python-11.4.0/setup.py000066400000000000000000000021211440464736400203030ustar00rootroot00000000000000#!/usr/bin/env python from setuptools import setup VERSION = "11.1.0" long_description = (open('README').read() + '\n\n' + open('HISTORY.txt').read()) setup( name='dataiku-api-client', version=VERSION, license="Apache Software License", packages=["dataikuapi", "dataikuapi.dss", "dataikuapi.apinode_admin", "dataikuapi.fm", "dataikuapi.dss_plugin_mlflow"], description="Python API client for Dataiku APIs", long_description=long_description, author="Dataiku", author_email="support@dataiku.com", url="https://www.dataiku.com", download_url="https://github.com/dataiku/dataiku-api-client-python/tarball/" + VERSION, classifiers=[ 'Development Status :: 5 - Production/Stable', 'Intended Audience :: Developers', 'License :: OSI Approved :: Apache Software License', 'Topic :: Software Development :: Libraries', 'Programming Language :: Python', 'Operating System :: OS Independent' ], install_requires=[ "requests>=2,<2.22", "python-dateutil" ] )