pax_global_header00006660000000000000000000000064127567721640014533gustar00rootroot0000000000000052 comment=7dd383ad5863c992258ee36f02bd1480875779c8 dataiku-api-client-python-3.1.2/000077500000000000000000000000001275677216400165225ustar00rootroot00000000000000dataiku-api-client-python-3.1.2/.gitignore000066400000000000000000000000061275677216400205060ustar00rootroot00000000000000*.pyc dataiku-api-client-python-3.1.2/.travis.yml000066400000000000000000000001521275677216400206310ustar00rootroot00000000000000language: python python: - "2.7" - "3.4" install: "pip install -r requirements.txt" script: nosetests dataiku-api-client-python-3.1.2/LICENSE.txt000066400000000000000000000010501275677216400203410ustar00rootroot00000000000000Copyright 2015 Dataiku SAS Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. dataiku-api-client-python-3.1.2/README000066400000000000000000000002161275677216400174010ustar00rootroot00000000000000API client for Dataiku Data Science Studio For more information, see: https://doc.dataiku.com/dss/latest/api/public/client-python/index.html dataiku-api-client-python-3.1.2/dataikuapi/000077500000000000000000000000001275677216400206365ustar00rootroot00000000000000dataiku-api-client-python-3.1.2/dataikuapi/__init__.py000066400000000000000000000001751275677216400227520ustar00rootroot00000000000000from dssclient import DSSClient from apinode_client import APINodeClient from apinode_admin_client import APINodeAdminClientdataiku-api-client-python-3.1.2/dataikuapi/apinode_admin/000077500000000000000000000000001275677216400234255ustar00rootroot00000000000000dataiku-api-client-python-3.1.2/dataikuapi/apinode_admin/__init__.py000066400000000000000000000000001275677216400255240ustar00rootroot00000000000000dataiku-api-client-python-3.1.2/dataikuapi/apinode_admin/auth.py000066400000000000000000000013131275677216400247360ustar00rootroot00000000000000 class APINodeAuth(object): """ A handle to interact with authentication settings on API node """ def __init__(self, client): self.client = client def list_keys(self): """Lists the Admin API keys""" return self.client._perform_json("GET", "keys") def add_key(self, label=None, description=None, created_by=None): """Add an Admin API key. Returns the key details""" key = { "label" : label, "description" : description, "createdBy" : created_by } return self.client._perform_json("POST", "keys", body=key) def delete_key(self, key): self.client._perform_empty("DELETE", "keys/%s" % key)dataiku-api-client-python-3.1.2/dataikuapi/apinode_admin/service.py000066400000000000000000000041751275677216400254460ustar00rootroot00000000000000from os import path as osp class APINodeService(object): """ A handle to interact with the settings of an API node service """ def __init__(self, client, service_id): self.client = client self.service_id = service_id def delete(self): """Deletes the API node service""" self.client._perform_empty("DELETE", "services/%s" % self.service_id) ######################################################## # On-disk generations management ######################################################## def list_generations(self): return self.client._perform_json("GET", "services/%s/generations" % self.service_id)["generations"] def import_generation_from_archive(self, file_path): self.client._perform_empty("POST", "services/%s/generations/actions/importFromArchive" % (self.service_id), params = { "filePath" : osp.abspath(file_path) }) def preload_generation(self, generation): self.client._perform_empty("POST", "services/%s/generations/%s/preload" % (self.service_id, generation)) ######################################################## # Switch / mapping management ######################################################## def disable(self): """Disable the service.""" self.client._perform_empty("POST", "services/%s/actions/disable" % self.service_id) def enable(self): self.client._perform_empty("POST", "services/%s/actions/enable" % self.service_id) def set_generations_mapping(self, mapping): """Setting a generations mapping automatically enables the service""" self.client._perform_empty("POST", "services/%s/actions/setMapping" % self.service_id, body = mapping) def switch_to_newest(self): return self.client._perform_json("POST", "services/%s/actions/switchToNewest" % self.service_id) def switch_to_single_generation(self, generation): self.client._perform_empty("POST", "services/%s/actions/switchTo/%s" % (self.service_id, generation)) dataiku-api-client-python-3.1.2/dataikuapi/apinode_admin_client.py000066400000000000000000000023121275677216400253330ustar00rootroot00000000000000from apinode_admin.service import APINodeService from apinode_admin.auth import APINodeAuth from .utils import DataikuException from .base_client import DSSBaseClient class APINodeAdminClient(DSSBaseClient): """Entry point for the DSS APINode admin client""" def __init__(self, uri, api_key): """ Instantiate a new DSS API client on the given base uri with the given API key. """ DSSBaseClient.__init__(self, "%s/%s" % (uri, "admin/api"), api_key) ######################################################## # Services generations ######################################################## def create_service(self, service_id): self._perform_empty("POST", "services/", body = { "serviceId" : service_id }) def list_services(self): return self._perform_json("GET", "services") def service(self, service_id): """ Gets a handle to interact with a service """ return APINodeService(self, service_id) def auth(self): """Returns a handle to interact with authentication""" return APINodeAuth(self) def get_metrics(self): return self._perform_json("GET", "metrics") dataiku-api-client-python-3.1.2/dataikuapi/apinode_client.py000066400000000000000000000063011275677216400241650ustar00rootroot00000000000000from .utils import DataikuException from .base_client import DSSBaseClient class APINodeClient(DSSBaseClient): """Entry point for the DSS API Node client This is an API client for the user-facing API of DSS API Node server (user facing API) """ def __init__(self, uri, service_id, api_key=None): """ Instantiate a new DSS API client on the given base URI with the given API key. :param str uri: Base URI of the DSS API node server (http://host:port/ or https://host:port/) :param str service_id: Identifier of the service to query :param str api_key: Optional, API key for the service. Only required if the service has authentication """ DSSBaseClient.__init__(self, "%s/%s" % (uri, "public/api/v1/%s" % service_id), api_key) def predict_record(self, endpoint_id, features, forced_generation=None, dispatch_key=None, context=None): """ Predicts a single record on a DSS API node endpoint (standard or custom prediction) :param str endpoint_id: Identifier of the endpoint to query :param features: Python dictionary of features of the record :param forced_generation: See documentation about multi-version prediction :param dispatch_key: See documentation about multi-version prediction :param context: Optional, Python dictionary of additional context information. The context information is logged, but not directly used. :return: a Python dict of the API answer. The answer contains a "result" key (itself a dict) """ obj = { "features" :features } if context is not None: obj["context"] = context if forced_generation is not None: obj["dispatch"] = {"forcedGeneration" : forced_generation } elif dispatch_key is not None: obj["dispatch"] = {"dispatchKey" : dispatch_key } return self._perform_json("POST", "%s/predict" % endpoint_id, body = obj) def predict_records(self, endpoint_id, records, forced_generation=None, dispatch_key=None): """ Predicts a batch of records on a DSS API node endpoint (standard or custom prediction) :param str endpoint_id: Identifier of the endpoint to query :param records: Python list of records. Each record must be a Python dict. Each record must contain a "features" dict (see predict_record) and optionally a "context" dict. :param forced_generation: See documentation about multi-version prediction :param dispatch_key: See documentation about multi-version prediction :return: a Python dict of the API answer. The answer contains a "results" key (which is an array of result objects) """ for record in records: if not "features" in record: raise ValueError("Each record must contain a 'features' dict") obj = { "items" : records } if forced_generation is not None: obj["dispatch"] = {"forcedGeneration" : forced_generation } elif dispatch_key is not None: obj["dispatch"] = {"dispatchKey" : dispatch_key } return self._perform_json("POST", "%s/predict-multi" % endpoint_id, body = obj)dataiku-api-client-python-3.1.2/dataikuapi/base_client.py000066400000000000000000000032361275677216400234640ustar00rootroot00000000000000import json from requests import Session, exceptions from requests import exceptions from requests.auth import HTTPBasicAuth from .utils import DataikuException class DSSBaseClient(object): def __init__(self, base_uri, api_key): self.api_key = api_key self.base_uri = base_uri self._session = Session() ######################################################## # Internal Request handling ######################################################## def _perform_http(self, method, path, params=None, body=None, stream=False): if body: body = json.dumps(body) auth = HTTPBasicAuth(self.api_key, "") try: http_res = self._session.request( method, "%s/%s" % (self.base_uri, path), params=params, data=body, auth=auth, stream = stream) http_res.raise_for_status() return http_res except exceptions.HTTPError: ex = http_res.json() raise DataikuException("%s: %s" % (ex.get("errorType", "Unknown error"), ex.get("message", "No message"))) def _perform_empty(self, method, path, params=None, body=None): self._perform_http(method, path, params, body, False) def _perform_text(self, method, path, params=None, body=None): return self._perform_http(method, path, params, body, False).text def _perform_json(self, method, path, params=None, body=None): return self._perform_http(method, path, params, body, False).json() def _perform_raw(self, method, path, params=None, body=None): return self._perform_http(method, path, params, body, True) dataiku-api-client-python-3.1.2/dataikuapi/dss/000077500000000000000000000000001275677216400214275ustar00rootroot00000000000000dataiku-api-client-python-3.1.2/dataikuapi/dss/__init__.py000066400000000000000000000000001275677216400235260ustar00rootroot00000000000000dataiku-api-client-python-3.1.2/dataikuapi/dss/admin.py000066400000000000000000000112161275677216400230720ustar00rootroot00000000000000 class DSSConnection(object): """ A connection on the DSS instance """ def __init__(self, client, name): self.client = client self.name = name ######################################################## # User deletion ######################################################## def delete(self): """ Delete the connection Note: this call requires an API key with admin rights """ return self.client._perform_empty( "DELETE", "/admin/connections/%s" % self.name) ######################################################## # User description ######################################################## def get_definition(self): """ Get the connection's definition (type, name, params, usage restrictions) Note: this call requires an API key with admin rights Returns: the connection definition, as a JSON object """ return self.client._perform_json( "GET", "/admin/connections/%s" % self.name) def set_definition(self, description): """ Set the connection's definition. Note: this call requires an API key with admin rights Args: definition: the definition for the connection, as a JSON object. """ return self.client._perform_json( "PUT", "/admin/connections/%s" % self.name, body = description) class DSSUser(object): """ A handle for a user on the DSS instance """ def __init__(self, client, login): self.client = client self.login = login ######################################################## # User deletion ######################################################## def delete(self): """ Deletes the user Note: this call requires an API key with admin rights """ return self.client._perform_empty( "DELETE", "/admin/users/%s" % self.login) ######################################################## # User description ######################################################## def get_definition(self): """ Get the user's definition (login, type, display name, permissions, ...) Note: this call requires an API key with admin rights :return: the user's definition, as a dict """ return self.client._perform_json( "GET", "/admin/users/%s" % self.login) def set_definition(self, definition): """ Set the user's definition. Note: this call requires an API key with admin rights :param dict definition: the definition for the user, as a dict. You should obtain the definition using get_definition, not create one. The fields that can be changed are: * email * displayName * groups * userProfile * password """ return self.client._perform_json( "PUT", "/admin/users/%s" % self.login, body = definition) class DSSGroup(object): """ A group on the DSS instance """ def __init__(self, client, name): self.client = client self.name = name ######################################################## # Group deletion ######################################################## def delete(self): """ Delete the group Note: this call requires an API key with admin rights """ return self.client._perform_empty( "DELETE", "/admin/groups/%s" % self.name) ######################################################## # User description ######################################################## def get_definition(self): """ Get the group's definition (name, description, admin abilities, type, ldap name mapping) Note: this call requires an API key with admin rights Returns: the group definition, as a JSON object """ return self.client._perform_json( "GET", "/admin/groups/%s" % self.name) def set_definition(self, definition): """ Set the group's definition. Note: this call requires an API key with admin rights Args: definition: the definition for the group, as a JSON object. """ return self.client._perform_json( "PUT", "/admin/groups/%s" % self.name, body = definition) dataiku-api-client-python-3.1.2/dataikuapi/dss/apiservice.py000066400000000000000000000040631275677216400241360ustar00rootroot00000000000000 class DSSAPIService(object): """ An API Service on the DSS instance """ def __init__(self, client, project_key, service_id): self.client = client self.project_key = project_key self.service_id = service_id def list_packages(self): """ List the packages of this API services Returns: the list of API service packages, each one as a JSON object """ return self.client._perform_json( "GET", "/projects/%s/apiservices/%s/packages" % (self.project_key, self.service_id)) def create_package(self, package_id): """ Prepare a package of this API service """ return self.client._perform_empty( "POST", "/projects/%s/apiservices/%s/packages/%s" % (self.project_key, self.service_id, package_id)) def delete_package(self, package_id): """ Delete a package of this API service """ return self.client._perform_empty( "DELETE", "/projects/%s/apiservices/%s/packages/%s" % (self.project_key, self.service_id, package_id)) def download_package_stream(self, package_id): """ Download a package archive that can be deployed in a DSS API Node, as a binary stream. Warning: this stream will monopolize the DSSClient until closed. """ return self.client._perform_raw( "GET", "/projects/%s/apiservices/%s/packages/%s/archive" % (self.project_key, self.service_id, package_id)).raw def download_package_to_file(self, package_id, path): """ Download a package archive that can be deployed in a DSS API Node, into the given output file. """ package_stream = self.client._perform_raw( "GET", "/projects/%s/apiservices/%s/packages/%s/archive" % (self.project_key, self.service_id, package_id)) with open(path, 'wb') as f: for chunk in package_stream.iter_content(chunk_size=10000): if chunk: f.write(chunk) f.flush() dataiku-api-client-python-3.1.2/dataikuapi/dss/dataset.py000066400000000000000000000211501275677216400234250ustar00rootroot00000000000000from ..utils import DataikuException from ..utils import DataikuUTF8CSVReader from ..utils import DataikuStreamedHttpUTF8CSVReader import json from .metrics import ComputedMetrics class DSSDataset(object): """ A dataset on the DSS instance """ def __init__(self, client, project_key, dataset_name): self.client = client self.project_key = project_key self.dataset_name = dataset_name ######################################################## # Dataset deletion ######################################################## def delete(self): """ Delete the dataset """ return self.client._perform_empty( "DELETE", "/projects/%s/datasets/%s" % (self.project_key, self.dataset_name)) ######################################################## # Dataset definition ######################################################## def get_definition(self): """ Get the definition of the dataset Returns: the definition, as a JSON object """ return self.client._perform_json( "GET", "/projects/%s/datasets/%s/" % (self.project_key, self.dataset_name)) def set_definition(self, definition): """ Set the definition of the dataset Args: definition: the definition, as a JSON object. You should only set a definition object that has been retrieved using the get_definition call. """ return self.client._perform_json( "PUT", "/projects/%s/datasets/%s/" % (self.project_key, self.dataset_name), body=definition) ######################################################## # Dataset metadata ######################################################## def get_schema(self): """ Get the schema of the dataset Returns: a JSON object of the schema, with the list of columns """ return self.client._perform_json( "GET", "/projects/%s/datasets/%s/schema" % (self.project_key, self.dataset_name)) def set_schema(self, schema): """ Set the schema of the dataset Args: schema: the desired schema for the dataset, as a JSON object. All columns have to provide their name and type """ return self.client._perform_json( "PUT", "/projects/%s/datasets/%s/schema" % (self.project_key, self.dataset_name), body=schema) def get_metadata(self): """ Get the metadata attached to this dataset. The metadata contains label, description checklists, tags and custom metadata of the dataset Returns: a dict object. For more information on available metadata, please see https://doc.dataiku.com/dss/api/latest """ return self.client._perform_json( "GET", "/projects/%s/datasets/%s/metadata" % (self.project_key, self.dataset_name)) def set_metadata(self, metadata): """ Set the metadata on this dataset. Args: metadata: the new state of the metadata for the dataset. You should only set a metadata object that has been retrieved using the get_metadata call. """ return self.client._perform_json( "PUT", "/projects/%s/datasets/%s/metadata" % (self.project_key, self.dataset_name), body=metadata) ######################################################## # Dataset data ######################################################## def iter_rows(self, partitions=None): """ Get the dataset's data Return: an iterator over the rows, each row being a tuple of values. The order of values in the tuples is the same as the order of columns in the schema returned by get_schema """ csv_stream = self.client._perform_raw( "GET" , "/projects/%s/datasets/%s/data/" %(self.project_key, self.dataset_name), params = { "format" : "tsv-excel-noheader", "partitions" : partitions }) return DataikuStreamedHttpUTF8CSVReader(self.get_schema()["columns"], csv_stream).iter_rows() def list_partitions(self): """ Get the list of all partitions of this dataset Returns: the list of partitions, as a list of strings """ return self.client._perform_json( "GET", "/projects/%s/datasets/%s/partitions" % (self.project_key, self.dataset_name)) def clear(self, partitions=None): """ Clear all data in this dataset Args: partitions: (optional) a list of partitions to clear. When not provided, the entire dataset is cleared """ return self.client._perform_json( "DELETE", "/projects/%s/datasets/%s/data" % (self.project_key, self.dataset_name), params={"partitions" : partitions}) ######################################################## # Dataset actions ######################################################## def synchronize_hive_metastore(self): """ Synchronize this dataset with the Hive metastore """ self.client._perform_empty( "POST" , "/projects/%s/datasets/%s/actions/synchronizeHiveMetastore" %(self.project_key, self.dataset_name)) def compute_metrics(self, partition='', metric_ids=None, probes=None): """ Compute metrics on a partition of this dataset. If neither metric ids nor custom probes set are specified, the metrics setup on the dataset are used. """ url = "/projects/%s/datasets/%s/actions" % (self.project_key, self.dataset_name) if metric_ids is not None: return self.client._perform_json( "POST" , "%s/computeMetricsFromIds" % url, params={'partition':partition}, body={"metricIds" : metric_ids}) elif probes is not None: return self.client._perform_json( "POST" , "%s/computeMetrics" % url, params={'partition':partition}, body=probes) else: return self.client._perform_empty( "POST" , "%s/computeMetrics" % url, params={'partition':partition}) def run_checks(self, partition='', checks=None): """ Run checks on a partition of this dataset. If the checks are not specified, the checks setup on the dataset are used. """ if checks is None: return self.client._perform_empty( "POST" , "/projects/%s/datasets/%s/actions/runChecks" %(self.project_key, self.dataset_name, partition)) else: return self.client._perform_json( "POST" , "/projects/%s/datasets/%s/actions/runChecks" %(self.project_key, self.dataset_name, partition), params=checks) ######################################################## # Metrics ######################################################## def get_last_metric_values(self, partition=''): """ Get the last values of the metrics on this dataset Returns: a list of metric objects and their value """ return ComputedMetrics(self.client._perform_json( "GET", "/projects/%s/datasets/%s/metrics/last/%s" % (self.project_key, self.dataset_name, 'NP' if len(partition) == 0 else partition))) def get_metric_history(self, metric, partition=''): """ Get the history of the values of the metric on this dataset Returns: an object containing the values of the metric, cast to the appropriate type (double, boolean,...) """ return self.client._perform_json( "GET", "/projects/%s/datasets/%s/metrics/history/%s" % (self.project_key, self.dataset_name, 'NP' if len(partition) == 0 else partition), params={'metricLookup' : metric if isinstance(metric, str) or isinstance(metric, unicode) else json.dumps(metric)}) ######################################################## # Usages ######################################################## def get_usages(self): """ Get the recipes or analyses referencing this dataset Returns: a list of usages """ return self.client._perform_json("GET", "/projects/%s/datasets/%s/usages" % (self.project_key, self.dataset_name)) dataiku-api-client-python-3.1.2/dataikuapi/dss/job.py000066400000000000000000000022041275677216400225510ustar00rootroot00000000000000 class DSSJob(object): """ A job on the DSS instance """ def __init__(self, client, project_key, id): self.client = client self.id = id self.project_key = project_key def abort(self): """ Aborts the job """ return self.client._perform_json( "POST", "/projects/%s/jobs/%s/abort" % (self.project_key, self.id)) def get_status(self): """ Get the current status of the job Returns: the state of the job, as a JSON object """ return self.client._perform_json( "GET", "/projects/%s/jobs/%s/" % (self.project_key, self.id)) def get_log(self, activity=None): """ Get the logs of the job Args: activity: (optional) the name of the activity in the job whose log is requested Returns: the log, as a string """ return self.client._perform_text( "GET", "/projects/%s/jobs/%s/log" % (self.project_key, self.id), params={ "activity" : activity }) dataiku-api-client-python-3.1.2/dataikuapi/dss/managedfolder.py000066400000000000000000000135761275677216400246050ustar00rootroot00000000000000from ..utils import DataikuException from ..utils import DataikuUTF8CSVReader from ..utils import DataikuStreamedHttpUTF8CSVReader import json from .metrics import ComputedMetrics class DSSManagedFolder(object): """ A managed folder on the DSS instance """ def __init__(self, client, project_key, odb_id): self.client = client self.project_key = project_key self.odb_id = odb_id ######################################################## # Managed folder deletion ######################################################## def delete(self): """ Delete the managed folder """ return self.client._perform_empty( "DELETE", "/projects/%s/managedfolders/%s" % (self.project_key, self.odb_id)) ######################################################## # Managed folder definition ######################################################## def get_definition(self): """ Get the definition of the managed folder Returns: the definition, as a JSON object """ return self.client._perform_json( "GET", "/projects/%s/managedfolders/%s/" % (self.project_key, self.odb_id)) def set_definition(self, definition): """ Set the definition of the managed folder Args: definition: the definition, as a JSON object. You should only set a definition object that has been retrieved using the get_definition call. """ return self.client._perform_json( "PUT", "/projects/%s/managedfolders/%s/" % (self.project_key, self.odb_id), body=definition) ######################################################## # Managed folder contents ######################################################## def list_contents(self): """ Get the list of files in the managed folder Returns: the list of files, as a JSON object """ return self.client._perform_json( "GET", "/projects/%s/managedfolders/%s/contents" % (self.project_key, self.odb_id)) def get_file(self, path): """ Get a file from the managed folder Returns: the file's content, as a stream """ return self.client._perform_raw( "GET", "/projects/%s/managedfolders/%s/contents/%s" % (self.project_key, self.odb_id, path)) def delete_file(self, path): """ Delete a file from the managed folder """ return self.client._perform_empty( "DELETE", "/projects/%s/managedfolders/%s/contents/%s" % (self.project_key, self.odb_id, path)) def put_file(self, name, f): """ Upload the file to the managed folder Args: f: the file contents, as a stream name: the name of the file """ return self.client._perform_json_upload( "POST", "/projects/%s/managedfolders/%s/contents/" % (self.project_key, self.odb_id), name, f) ######################################################## # Managed folder actions ######################################################## def compute_metrics(self, metric_ids=None, probes=None): """ Compute metrics on this managed folder. If the metrics are not specified, the metrics setup on the managed folder are used. """ url = "/projects/%s/managedfolders/%s/actions" % (self.project_key, self.odb_id) if metric_ids is not None: return self.client._perform_json( "POST" , "%s/computeMetricsFromIds" % url, body={"metricIds" : metric_ids}) elif probes is not None: return self.client._perform_json( "POST" , "%s/computeMetrics" % url, body=probes) else: return self.client._perform_empty( "POST" , "%s/computeMetrics" % url) if metrics is None: self.client._perform_empty( "POST" , "/projects/%s/managedfolders/%s/actions/computeMetrics" %(self.project_key, self.odb_id)) else: self.client._perform_json( "POST" , "/projects/%s/managedfolders/%s/actions/computeMetrics" %(self.project_key, self.odb_id), params=metrics) ######################################################## # Metrics ######################################################## def get_last_metric_values(self): """ Get the last values of the metrics on this managed folder Returns: a list of metric objects and their value """ return ComputedMetrics(self.client._perform_json( "GET", "/projects/%s/managedfolders/%s/metrics/last" % (self.project_key, self.odb_id))) def get_metric_history(self, metric): """ Get the history of the values of the metric on this dataset Returns: an object containing the values of the metric, cast to the appropriate type (double, boolean,...) """ return self.client._perform_json( "GET", "/projects/%s/managedfolders/%s/metrics/history" % (self.project_key, self.odb_id), params={'metricLookup' : metric if isinstance(metric, str) or isinstance(metric, unicode) else json.dumps(metric)}) ######################################################## # Usages ######################################################## def get_usages(self): """ Get the recipes referencing this folder Returns: a list of usages """ return self.client._perform_json("GET", "/projects/%s/managedfolders/%s/usages" % (self.project_key, self.odb_id)) dataiku-api-client-python-3.1.2/dataikuapi/dss/meaning.py000066400000000000000000000017651275677216400234300ustar00rootroot00000000000000 class DSSMeaning(object): """ A user-defined meaning on the DSS instance """ def __init__(self, client, id): self.client = client self.id = id ######################################################## # Meaning definition ######################################################## def get_definition(self): """ Get the meaning's definition. Note: this call requires an API key with admin rights Returns: the meaning definition, as a JSON object """ return self.client._perform_json( "GET", "/meanings/%s" % self.id) def set_definition(self, definition): """ Set the meaning's definition. Note: this call requires an API key with admin rights Args: definition: the definition for the meaning, as a JSON object """ return self.client._perform_json( "PUT", "/meanings/%s" % self.id, body = definition) dataiku-api-client-python-3.1.2/dataikuapi/dss/metrics.py000066400000000000000000000036321275677216400234530ustar00rootroot00000000000000 class ComputedMetrics(object): def __init__(self, raw): self.raw = raw def get_metric_by_id(self, id): all_ids = [] for metric in self.raw["metrics"]: all_ids.append(metric["metric"]["id"]) if metric["metric"]["id"] == id: return metric raise Exception("Metric %s not found among: %s" % (id, all_ids)) def get_global_data(self, metric_id): for partition_data in self.get_metric_by_id(metric_id)["lastValues"]: if partition_data["partition"]== "NP" or partition_data["partition"] == "ALL": return partition_data raise Exception("No data found for global partition for metric %s" % metric_id) def get_global_value(self, metric_id): return ComputedMetrics.get_value_from_data(self.get_global_data(metric_id)) def get_partition_data(self, metric_id, partition): for partition_data in self.get_metric_by_id(metric_id)["lastValues"]: if partition_data["partition"] == partition: return partition_data def get_partition_value(self, metric_id, partition): return ComputedMetrics.get_value_from_data(self.get_partition_data(metric_id, partition)) def get_first_partition_data(self, metric_id): for partition_data in self.get_metric_by_id(metric_id)["lastValues"]: return partition_data raise Exception("No partition data for metric %s" % metric_id) def get_all_ids(self): all_ids = [] for metric in self.raw["metrics"]: all_ids.append(metric["metric"]["id"]) return all_ids @staticmethod def get_value_from_data(data): dtype = data.get("dataType", "STRING") if dtype in ["BIGINT", "INT"]: return int(data["value"]) elif dtype in ["FLOAT", "DOUBLE"]: return float(data["value"]) else: return data["value"]dataiku-api-client-python-3.1.2/dataikuapi/dss/project.py000066400000000000000000000416721275677216400234610ustar00rootroot00000000000000from dataset import DSSDataset from recipe import DSSRecipe from managedfolder import DSSManagedFolder from savedmodel import DSSSavedModel from job import DSSJob from scenario import DSSScenario from apiservice import DSSAPIService import sys class DSSProject(object): """ A project on the DSS instance """ def __init__(self, client, project_key): self.client = client self.project_key = project_key ######################################################## # Project deletion ######################################################## def delete(self): """ Delete the project Note: this call requires an API key with admin rights """ return self.client._perform_empty( "DELETE", "/projects/%s" % self.project_key) ######################################################## # Project export ######################################################## def get_export_stream(self, options = {}): """ Return a stream of the exported project Warning: this stream will monopolize the DSSClient until closed """ return self.client._perform_raw( "POST", "/projects/%s/export" % self.project_key, body=options).raw def export_to_file(self, path, options={}): """ Export the project to a file Args: path: the path of the file in which the exported project should be saved """ with open(path, 'w') as f: export_stream = self.client._perform_raw( "POST", "/projects/%s/export" % self.project_key, body=options) for chunk in export_stream.iter_content(chunk_size=32768): if chunk: f.write(chunk) f.flush() ######################################################## # Project infos ######################################################## def get_metadata(self): """ Get the metadata attached to this project. The metadata contains label, description checklists, tags and custom metadata of the project Returns: a dict object. For more information on available metadata, please see https://doc.dataiku.com/dss/api/latest """ return self.client._perform_json( "GET", "/projects/%s/metadata" % self.project_key) def set_metadata(self, metadata): """ Set the metadata on this project. Args: metadata: the new state of the metadata for the project. You should only set a metadata object that has been retrieved using the get_metadata call. """ return self.client._perform_empty( "PUT", "/projects/%s/metadata" % self.project_key, body = metadata) def get_permissions(self): """ Get the permissions attached to this project Returns: a JSON object, containing the owner and the permissions, as a list of pairs of group name and permission type """ return self.client._perform_json( "GET", "/projects/%s/permissions" % self.project_key) def set_permissions(self, permissions): """ Set the permissions on this project Args: permissions: a JSON object of the same structure as the one returned by get_permissions call """ return self.client._perform_empty( "PUT", "/projects/%s/permissions" % self.project_key, body = permissions) ######################################################## # Datasets ######################################################## def list_datasets(self): """ List the datasets in this project Returns: the list of the datasets, each one as a JSON object """ return self.client._perform_json( "GET", "/projects/%s/datasets/" % self.project_key) def get_dataset(self, dataset_name): """ Get a handle to interact with a specific dataset Args: dataset_name: the name of the desired dataset Returns: A :class:`dataikuapi.dss.dataset.DSSDataset` dataset handle """ return DSSDataset(self.client, self.project_key, dataset_name) def create_dataset(self, dataset_name, type, params={}, formatType=None, formatParams={}): """ Create a new dataset in the project, and return a handle to interact with it Args: dataset_name: the name for the new dataset type: the type of the dataset params: the parameters for the type, as a JSON object formatType: an optional format to create the dataset with formatParams: the parameters to the format, as a JSON object Returns: A :class:`dataikuapi.dss.dataset.DSSDataset` dataset handle """ obj = { "name" : dataset_name, "projectKey" : self.project_key, "type" : type, "params" : params, "formatType" : formatType, "formatParams" : formatParams } self.client._perform_json("POST", "/projects/%s/datasets/" % self.project_key, body = obj) return DSSDataset(self.client, self.project_key, dataset_name) ######################################################## # Saved models ######################################################## def list_saved_models(self): """ List the saved models in this project Returns: the list of the saved models, each one as a JSON object """ return self.client._perform_json( "GET", "/projects/%s/savedmodels/" % self.project_key) def get_saved_model(self, sm_id): """ Get a handle to interact with a specific saved model Args: sm_id: the identifier of the desired saved model Returns: A :class:`dataikuapi.dss.savedmodel.DSSSavedModel` saved model handle """ return DSSSavedModel(self.client, self.project_key, sm_id) ######################################################## # Managed folders ######################################################## def list_managed_folders(self): """ List the managed folders in this project Returns: the list of the managed folders, each one as a JSON object """ return self.client._perform_json( "GET", "/projects/%s/managedfolders/" % self.project_key) def get_managed_folder(self, odb_id): """ Get a handle to interact with a specific managed folder Args: odb_id: the identifier of the desired managed folder Returns: A :class:`dataikuapi.dss.managedfolder.DSSManagedFolder` managed folder handle """ return DSSManagedFolder(self.client, self.project_key, odb_id) def create_managed_folder(self, name): """ Create a new managed folder in the project, and return a handle to interact with it Args: name: the name of the managed folder Returns: A :class:`dataikuapi.dss.managedfolder.DSSManagedFolder` managed folder handle """ obj = { "name" : name, "projectKey" : self.project_key } res = self.client._perform_json("POST", "/projects/%s/managedfolders/" % self.project_key, body = obj) odb_id = res['id'] return DSSManagedFolder(self.client, self.project_key, odb_id) ######################################################## # Jobs ######################################################## def list_jobs(self): """ List the jobs in this project Returns: a list of the jobs, each one as a JSON object, containing both the definition and the state """ return self.client._perform_json( "GET", "/projects/%s/jobs/" % self.project_key) def get_job(self, id): """ Get a handler to interact with a specific job Returns: A :class:`dataikuapi.dss.job.DSSJob` job handle """ return DSSJob(self.client, self.project_key, id) def start_job(self, definition): """ Create a new job, and return a handle to interact with it Args: definition: the definition for the job to create. The definition must contain the type of job (RECURSIVE_BUILD, NON_RECURSIVE_FORCED_BUILD, RECURSIVE_FORCED_BUILD, RECURSIVE_MISSING_ONLY_BUILD) and a list of outputs to build. Optionally, a refreshHiveMetastore field can specify whether to re-synchronize the Hive metastore for recomputed HDFS datasets. Returns: A :class:`dataikuapi.dss.job.DSSJob` job handle """ job_def = self.client._perform_json("POST", "/projects/%s/jobs/" % self.project_key, body = definition) return DSSJob(self.client, self.project_key, job_def['id']) ######################################################## # Variables ######################################################## def get_variables(self): """ Gets the variables of this project. Returns: a dictionary containing two dictionaries : "standard" and "local". "standard" are regular variables, exported with bundles. "local" variables are not part of the bundles for this project """ return self.client._perform_json( "GET", "/projects/%s/variables/" % self.project_key) def set_variables(self, obj): """ Sets the variables of this project. @param obj: must be a modified version of the object returned by get_variables """ if not "standard" in obj: raise ValueError("Missing 'standard' key in argument") if not "local" in obj: raise ValueError("Missing 'local' key in argument") self.client._perform_empty( "PUT", "/projects/%s/variables/" % self.project_key, body=obj) ######################################################## # API Services ######################################################## def list_api_services(self): """ List the API services in this project Returns: the list of API services, each one as a JSON object """ return self.client._perform_json( "GET", "/projects/%s/apiservices/" % self.project_key) def get_api_service(self, service_id): """ Get a handle to interact with a specific API service Args: service_id: the ID of the desired API service Returns: A :class:`dataikuapi.dss.dataset.DSSAPIService` API Service handle """ return DSSAPIService(self.client, self.project_key, service_id) ######################################################## # Bundles / Export (Design node) ######################################################## def list_exported_bundles(self): return self.client._perform_json("GET", "/projects/%s/bundles/exported" % self.project_key) def export_bundle(self, bundle_id): return self.client._perform_json("PUT", "/projects/%s/bundles/exported/%s" % (self.project_key, bundle_id)) def get_exported_bundle_archive_stream(self, bundle_id): """ Download a bundle archive that can be deployed in a DSS automation Node, as a binary stream. Warning: this stream will monopolize the DSSClient until closed. """ return self.client._perform_raw("GET", "/projects/%s/bundles/exported/%s/archive" % (self.project_key, bundle_id)) def download_exported_bundle_archive_to_file(self, bundle_id, path): """ Download a bundle archive that can be deployed in a DSS automation Node into the given output file. @param path if "-", will write to /dev/stdout """ if path == "-": path= "/dev/stdout" stream = self.get_exported_bundle_archive_stream(bundle_id) with open(path, 'wb') as f: for chunk in stream.iter_content(chunk_size=10000): if chunk: f.write(chunk) f.flush() stream.close() ######################################################## # Bundles / Import (Automation node) ######################################################## def list_imported_bundles(self): return self.client._perform_json("GET", "/projects/%s/bundles/imported" % self.project_key) def import_bundle_from_archive(self, archive_path): return self.client._perform_json("POST", "/projects/%s/bundles/imported/actions/importFromArchive" % (self.project_key), params = { "archivePath" : osp.abspath(archive_path) }) def activate_bundle(self, bundle_id): return self.client._perform_json("POST", "/projects/%s/bundles/imported/%s/actions/activate" % (self.project_key, bundle_id)) ######################################################## # Scenarios ######################################################## def list_scenarios(self): """ List the scenarios in this project Returns: the list of scenarios, each one as a JSON object. """ return self.client._perform_json( "GET", "/projects/%s/scenarios/" % self.project_key) def get_scenario(self, scenario_id): """ Get a handle to interact with a specific scenario Args: scenario_id: the ID of the desired scenario Returns: A :class:`dataikuapi.dss.scenario.DSSScenario` scenario handle """ return DSSScenario(self.client, self.project_key, scenario_id) def create_scenario(self, scenario_name, type, definition={}): """ Create a new scenario in the project, and return a handle to interact with it Args: scenario_name: the name for the new scenario type: the type of the scenario ('step_based' or 'custom_python') definition: the definition of the scenario, as a JSON object Returns: A :class:`dataikuapi.dss.scenario.DSSScenario` scenario handle """ definition['type'] = type definition['name'] = scenario_name scenario_id = self.client._perform_json("POST", "/projects/%s/scenarios/" % self.project_key, body = definition)['id'] return DSSScenario(self.client, self.project_key, scenario_id) ######################################################## # Recipes ######################################################## def list_recipes(self): """ List the recipes in this project Returns: the list of the recipes, each one as a JSON object """ return self.client._perform_json( "GET", "/projects/%s/recipes/" % self.project_key) def get_recipe(self, recipe_name): """ Get a handle to interact with a specific recipe Args: recipe_name: the name of the desired recipe Returns: A :class:`dataikuapi.dss.recipe.DSSRecipe` recipe handle """ return DSSRecipe(self.client, self.project_key, recipe_name) def create_recipe(self, recipe_name, type, recipe_inputs=[], recipe_outputs=[], recipe_params={}): """ Create a new recipe in the project, and return a handle to interact with it. Some recipe types take additional parameters in recipe_params: * 'grouping' : a 'groupKey' column name * 'python', 'sql_query', 'hive', 'impala' : the code of the recipe as a 'payload' string Args: recipe_name: the name for the new recipe type: the type of the scenario ('sync', 'grouping', 'join', 'vstack', 'python', 'sql_query', 'hive', 'impala') recipe_inputs: an array of recipes inputs, as objects {'ref':'...', 'deps':[...]} recipe_outputs: an array of recipes outputs, as objects {'ref':'...', 'appendMode':True/False} recipe_params: additional parameters for the recipe creation Returns: A :class:`dataikuapi.dss.recipe.DSSRecipe` recipe handle """ definition = {'type' : type, 'name' : recipe_name, 'inputs' : {'items':recipe_inputs}, 'outputs' : {'items':recipe_outputs}} definition['params'] = recipe_params recipe_name = self.client._perform_json("POST", "/projects/%s/recipes/" % self.project_key, body = definition)['name'] return DSSRecipe(self.client, self.project_key, recipe_name) dataiku-api-client-python-3.1.2/dataikuapi/dss/recipe.py000066400000000000000000000073441275677216400232600ustar00rootroot00000000000000from ..utils import DataikuException import json class DSSRecipe(object): """ A recipe on the DSS instance """ def __init__(self, client, project_key, recipe_name): self.client = client self.project_key = project_key self.recipe_name = recipe_name ######################################################## # Dataset deletion ######################################################## def delete(self): """ Delete the recipe """ return self.client._perform_empty( "DELETE", "/projects/%s/recipes/%s" % (self.project_key, self.recipe_name)) ######################################################## # Recipe definition ######################################################## def get_definition_and_payload(self): """ Get the definition of the recipe Returns: the definition, as a DSSRecipeDefinitionAndPayload object, containing the recipe definition itself and its payload """ data = self.client._perform_json( "GET", "/projects/%s/recipes/%s/" % (self.project_key, self.recipe_name)) return DSSRecipeDefinitionAndPayload(data) def set_definition_and_payload(self, definition): """ Set the definition of the recipe Args: definition: the definition, as a DSSRecipeDefinitionAndPayload object. You should only set a definition object that has been retrieved using the get_definition call. """ return self.client._perform_json( "PUT", "/projects/%s/recipes/%s/" % (self.project_key, self.recipe_name), body=definition.data) ######################################################## # Recipe metadata ######################################################## def get_metadata(self): """ Get the metadata attached to this recipe. The metadata contains label, description checklists, tags and custom metadata of the recipe Returns: a dict object. For more information on available metadata, please see https://doc.dataiku.com/dss/api/latest """ return self.client._perform_json( "GET", "/projects/%s/recipes/%s/metadata" % (self.project_key, self.recipe_name)) def set_metadata(self, metadata): """ Set the metadata on this recipe. Args: metadata: the new state of the metadata for the recipe. You should only set a metadata object that has been retrieved using the get_metadata call. """ return self.client._perform_json( "PUT", "/projects/%s/recipes/%s/metadata" % (self.project_key, self.recipe_name), body=metadata) class DSSRecipeDefinitionAndPayload(object): """ Definition for a recipe, that is, the recipe definition itself and its payload """ def __init__(self, data): self.data = data def get_recipe_raw_definition(self): return self.data.get('recipe', None) def get_recipe_inputs(self): return self.data.get('recipe').get('inputs') def get_recipe_outputs(self): return self.data.get('recipe').get('outputs') def get_recipe_params(self): return self.data.get('recipe').get('params') def get_payload(self): return self.data.get('payload', None) def get_json_payload(self): return json.loads(self.data.get('payload', None)) def set_payload(self, payload): self.data['payload'] = payload def set_json_payload(self, payload): self.data['payload'] = json.dumps(payload) dataiku-api-client-python-3.1.2/dataikuapi/dss/savedmodel.py000066400000000000000000000034271275677216400241320ustar00rootroot00000000000000from ..utils import DataikuException from ..utils import DataikuUTF8CSVReader from ..utils import DataikuStreamedHttpUTF8CSVReader import json from .metrics import ComputedMetrics class DSSSavedModel(object): """ A saved model on the DSS instance """ def __init__(self, client, project_key, sm_id): self.client = client self.project_key = project_key self.sm_id = sm_id ######################################################## # Versions ######################################################## def list_versions(self): """ Get the versions of this saved model Returns: an list of the versions """ return self.client._perform_json( "GET", "/projects/%s/savedmodels/%s/versions" % (self.project_key, self.sm_id)) ######################################################## # Metrics ######################################################## def get_metric_values(self, version_id): """ Get the values of the metrics on the version of this saved model Returns: a list of metric objects and their value """ return ComputedMetrics(self.client._perform_json( "GET", "/projects/%s/savedmodels/%s/metrics/%s" % (self.project_key, self.sm_id, version_id))) ######################################################## # Usages ######################################################## def get_usages(self): """ Get the recipes referencing this model Returns: a list of usages """ return self.client._perform_json("GET", "/projects/%s/savedmodels/%s/usages" % (self.project_key, self.sm_id)) dataiku-api-client-python-3.1.2/dataikuapi/dss/scenario.py000066400000000000000000000154251275677216400236130ustar00rootroot00000000000000class DSSScenario(object): """ A handle to interact with a scenario on the DSS instance """ def __init__(self, client, project_key, id): self.client = client self.id = id self.project_key = project_key def abort(self): """ Aborts the scenario if it currently running """ return self.client._perform_json( "POST", "/projects/%s/scenarios/%s/abort" % (self.project_key, self.id)) def run(self, params={}): """ Requests a run of the scenario, which will start after a few seconds. :params dict params: additional parameters that will be passed to the scenario through trigger params """ trigger_fire = self.client._perform_json( "POST", "/projects/%s/scenarios/%s/run" % (self.project_key, self.id), body=params) return DSSTriggerFire(self, trigger_fire) def get_last_runs(self, limit=10, only_finished_runs=False): """ Get the list of the last runs of the scenario. :return: A list of :class:`dataikuapi.dss.scenario.DSSScenarioRun` """ runs = self.client._perform_json( "GET", "/projects/%s/scenarios/%s/get-last-runs" % (self.project_key, self.id), params={ 'limit' : limit, 'onlyFinishedRuns' : only_finished_runs }) return [DSSScenarioRun(self.client, run) for run in runs] def get_current_run(self): """ Get the current run of the scenario, or None if it is not running at the moment :return: A :class:`dataikuapi.dss.scenario.DSSScenarioRun` """ last_run = self.get_last_runs(1) if len(last_run) == 0: return None last_run = last_run[0] if 'result' in last_run.run: return None # has a result means it's done else: return last_run def get_run(self, run_id): """ Get a handle to a run of the scenario :return: A :class:`dataikuapi.dss.scenario.DSSScenarioRun` """ return DSSScenarioRun(self.client, {'scenario' : {'projectKey':self.project_key, 'id':self.id}, 'runId' : run_id}) def get_definition(self, with_status=True): """ Returns the definition of the scenario Args: with_status: if True, the definition contains the run status of the scenario but not its actions' definition. If False, the definition doesn't contain the run status but has the scenario's actions definition """ suffix = '/no-params' if with_status else '' return self.client._perform_json( "GET", "/projects/%s/scenarios/%s%s" % (self.project_key, self.id, suffix)) def set_definition(self, definition, with_status=True): """ Updates the definition of this scenario Args: with_status: should be the same as the value passed to get_definition(). If True, the params, triggers and reporters fields of the scenario are ignored, """ suffix = '/no-params' if with_status else '' return self.client._perform_json( "PUT", "/projects/%s/scenarios/%s%s" % (self.project_key, self.id, suffix), body = definition) def get_payload(self, extension='py'): """ Returns the payload of the scenario :param str extension: the type of script. Default is 'py' for python """ return self.client._perform_json( "GET", "/projects/%s/scenarios/%s/payload" % (self.project_key, self.id)).get('script', '') def set_payload(self, script, with_status=True): """ Updates the payload of this scenario :param str extension: the type of script. Default is 'py' for python """ return self.client._perform_json( "PUT", "/projects/%s/scenarios/%s/payload" % (self.project_key, self.id), body = {'script' : script}) def get_average_duration(self, limit=3): """ Get the average duration (in fractional seconds) of the last runs of this scenario that finished, where finished means it ended with SUCCESS or WARNING. If there are not enough runs to perform the average, returns None Args: limit: number of last runs to average on """ last_runs = self.get_last_runs(limit=limit, only_finished_runs=True) if len(last_runs) < limit: return None return sum([run.get_duration() for run in last_runs]) / len(last_runs) class DSSScenarioRun(object): """ A handle containing basic info about a past run of a scenario. This handle can also be used to fetch additional information about the urn """ def __init__(self, client, run): self.client = client self.run = run def get_info(self): """ Get the basic information of the scenario run """ return self.run def get_details(self): """ Get the full details of the scenario run, including its step runs. Note: this perform another API call """ return self.client._perform_json( "GET", "/projects/%s/scenarios/%s/%s/" % (self.run['scenario']['projectKey'], self.run['scenario']['id'], self.run['runId'])) def get_start_time(self): """ Get the start time of the scenario run """ return datetime.fromtimestamp(self.run['start'] / 1000) def get_duration(self): """ Get the duration of this run (in fractional seconds). If the run is still running, get the duration since it started """ end_time = datetime.now() if self.run['end'] > 0: end_time = datetime.fromtimestamp(self.run['end'] / 1000) duration = (end_time - self.get_start_time()).total_seconds() return duration class DSSTriggerFire(object): """ The activation of a trigger on the DSS instance """ def __init__(self, scenario, trigger_fire): self.client = scenario.client self.project_key = scenario.project_key self.scenario_id = scenario.id self.trigger_id = trigger_fire['trigger']['id'] self.run_id = trigger_fire['runId'] self.trigger_fire = trigger_fire def get_scenario_run(self): """ Get the run of the scenario that this trigger activation launched """ run = self.client._perform_json( "GET", "/projects/%s/scenarios/%s/get-run-for-trigger" % (self.project_key, self.scenario_id), params= { 'triggerId' : self.trigger_id, 'triggerRunId' : self.run_id }) if 'scenarioRun' not in run: return None else: return DSSScenarioRun(self.client, run['scenarioRun']) dataiku-api-client-python-3.1.2/dataikuapi/dss/sqlquery.py000066400000000000000000000051511275677216400236700ustar00rootroot00000000000000from ..utils import DataikuException from ..utils import DataikuUTF8CSVReader from ..utils import DataikuStreamedHttpUTF8CSVReader class DSSSQLQuery(object): """ A connection to a database or database-like on which queries can be run through DSS """ def __init__(self, client, query, connection, database, dataset_full_name, pre_queries, post_queries, type, extra_conf): self.client = client self.query = query self.connection = connection self.database = database self.dataset_full_name = dataset_full_name self.pre_queries = pre_queries self.post_queries = post_queries self.extra_conf = extra_conf self.type = type self.streaming_session = self.client._perform_json( "POST", "/sql/queries/", body = { "query" : self.query, "preQueries" : self.pre_queries, "postQueries" : self.post_queries, "connection" : self.connection, "database" : self.database, "datasetFullName" : self.dataset_full_name, "type" : self.type, "extraConf" : extra_conf }) self.queryId = self.streaming_session['queryId'] def get_schema(self): """ Get the query's result set's schema Returns: the schema as a JSON array of columns """ return self.streaming_session['schema'] def iter_rows(self): """ Get the query's results Returns: an iterator over the rows, each row being a tuple of values. The order of values in the tuples is the same as the order of columns in the schema returned by get_schema """ csv_stream = self.client._perform_raw( "GET", "/sql/queries/%s/stream" % (self.queryId), params = { "format" : "tsv-excel-noheader" }) return DataikuStreamedHttpUTF8CSVReader(self.get_schema(), csv_stream).iter_rows() def verify(self): """ Verify that the result set streaming completed successfully and was not truncated Raises: if the query failed at some point while streaming the results, an exception will be raised. If the call completes without exception, then the query was successfully streamed """ resp = self.client._perform_empty( "GET", "/sql/queries/%s/finish-streaming" % (self.queryId)) # exception raising is done in _perform_empty()dataiku-api-client-python-3.1.2/dataikuapi/dssclient.py000066400000000000000000000431461275677216400232100ustar00rootroot00000000000000import json from requests import Session from requests import exceptions from requests.auth import HTTPBasicAuth from dss.project import DSSProject from dss.admin import DSSUser, DSSGroup, DSSConnection from dss.meaning import DSSMeaning from dss.sqlquery import DSSSQLQuery import os.path as osp from .utils import DataikuException class DSSClient(object): """Entry point for the DSS API client""" def __init__(self, host, api_key): """ Instantiate a new DSS API client on the given host with the given API key. API keys can be managed in DSS on the project page or in the global settings. The API key will define which operations are allowed for the client. """ self.api_key = api_key self.host = host self._session = Session() ######################################################## # Projects ######################################################## def list_project_keys(self): """ List the project keys (=project identifiers). Returns: list of identifiers (=strings) """ return [x["projectKey"] for x in self._perform_json("GET", "/projects/")] def list_projects(self): """ List the projects Returns: list of objects. Each object contains at least a 'projectKey' field """ return self._perform_json("GET", "/projects/") def get_project(self, project_key): """ Get a handle to interact with a specific project. Args: project_key: the project key of the desired project Returns: A :class:`dataikuapi.dss.project.DSSProject` """ return DSSProject(self, project_key) def create_project(self, project_key, name, owner, description=None, settings=None): """ Create a project, and return a project handle to interact with it. Note: this call requires an API key with admin rights Args: project_key: the identifier to use for the project. name: the name for the project. owner: the owner of the project. description: a short description for the project. Returns: A :class:`dataikuapi.dss.project.DSSProject` project handle """ resp = self._perform_text( "POST", "/projects/", body={ "projectKey" : project_key, "name" : name, "owner" : owner, "settings" : settings, "description" : description }) return DSSProject(self, project_key) ######################################################## # SQL queries ######################################################## def sql_query(self, query, connection=None, database=None, dataset_full_name=None, pre_queries=None, post_queries=None, type='sql', extra_conf={}): """ Initiate a SQL, Hive or Impala query and get a handle to retrieve the results of the query. Internally, the query is run by DSS. The database to run the query on is specified either by passing a connection name, or by passing a database name, or by passing a dataset full name (whose connection is then used to retrieve the database) Args: query: the query to run connection: the connection on which the query should be run (exclusive of database and dataset_full_name) database: the database on which the query should be run (exclusive of connection and dataset_full_name) dataset_full_name: the dataset on the connection of which the query should be run (exclusive of connection and database) pre_queries: (optional) array of queries to run before the query post_queries: (optional) array of queries to run after the query type: the type of query : either 'sql', 'hive' or 'impala' Returns: A :class:`dataikuapi.dss.sqlquery.DSSSQLQuery` query handle """ return DSSSQLQuery(self, query, connection, database, dataset_full_name, pre_queries, post_queries, type, extra_conf) ######################################################## # Users ######################################################## def list_users(self): """ List all users setup on the DSS instance Note: this call requires an API key with admin rights :return: A list of users, as an array of dicts. """ return self._perform_json( "GET", "/admin/users/") def get_user(self, login): """ Get a handle to interact with a specific user :param str login: the login of the desired user :return: A :class:`dataikuapi.dss.admin.DSSUser` user handle """ return DSSUser(self, login) def create_user(self, login, password, display_name='', source_type='LOCAL', groups=[], profile='DATA_SCIENTIST'): """ Create a user, and return a handle to interact with it Note: this call requires an API key with admin rights :param str login: the login of the new user :param str password: the password of the new user :param str display_name: the displayed name for the new user :param str source_type: the type of new user. Admissible values are 'LOCAL' or 'LDAP' :param list groups: the names of the groups the new user belongs to :param str profile: The profile for the new user, can be one of READER, DATA_ANALYST or DATA_SCIENTIST :return: A :class:`dataikuapi.dss.admin.DSSUser` user handle """ resp = self._perform_text( "POST", "/admin/users/", body={ "login" : login, "password" : password, "displayName" : display_name, "sourceType" : source_type, "groups" : groups, "userProfile" : userProfile }) return DSSUser(self, login) ######################################################## # Groups ######################################################## def list_groups(self): """ List all groups setup on the DSS instance Note: this call requires an API key with admin rights Returns: A list of groups, as an array of JSON objects """ return self._perform_json( "GET", "/admin/groups/") def get_group(self, name): """ Get a handle to interact with a specific group Args: name: the name of the desired group Returns: A :class:`dataikuapi.dss.admin.DSSGroup` group handle """ return DSSGroup(self, name) def create_group(self, name, description=None, source_type='LOCAL'): """ Create a group, and return a handle to interact with it Note: this call requires an API key with admin rights Args: name: the name of the new group description: a description of the new group source_type: the type of the new group. Admissible values are 'LOCAL', 'LDAP', 'SAAS' Returns: A :class:`dataikuapi.dss.admin.DSSGroup` group handle """ resp = self._perform_text( "POST", "/admin/groups/", body={ "name" : name, "description" : description, "sourceType" : source_type }) return DSSGroup(self, name) ######################################################## # Connections ######################################################## def list_connections(self): """ List all connections setup on the DSS instance Note: this call requires an API key with admin rights Returns: All connections, as a map of connection name to connection definition """ return self._perform_json( "GET", "/admin/connections/") def get_connection(self, name): """ Get a handle to interact with a specific connection Args: name: the name of the desired connection Returns: A :class:`dataikuapi.dss.admin.DSSConnection` connection handle """ return DSSConnection(self, name) def create_connection(self, name, type=None, params=None, usable_by='ALL', allowed_groups=None): """ Create a connection, and return a handle to interact with it Note: this call requires an API key with admin rights Args: name: the name of the new connection type: the type of the new connection params: the parameters of the new connection, as a JSON object usable_by: the type of access control for the connection. Either 'ALL' (=no access control) or 'ALLOWED' (=access restricted to users of a list of groups) allowed_groups: when using access control (that is, setting usable_by='ALLOWED'), the list of names of the groups whose users are allowed to use the new connection Returns: A :class:`dataikuapi.dss.admin.DSSConnection` connection handle """ resp = self._perform_text( "POST", "/admin/connections/", body={ "name" : name, "type" : type, "params" : params, "usableBy" : usable_by, "allowedGroups" : allowed_groups }) return DSSConnection(self, name) ######################################################## # Meanings ######################################################## def list_meanings(self): """ List all user-defined meanings on the DSS instance Note: this call requires an API key with admin rights Returns: A list of meanings, as an array of JSON objects """ return self._perform_json( "GET", "/meanings/") def get_meaning(self, id): """ Get a handle to interact with a specific user-defined meaning Note: this call requires an API key with admin rights Args: id: the ID of the desired meaning Returns: A :class:`dataikuapi.dss.meaning.DSSMeaning` meaning handle """ return DSSMeaning(self, id) def create_meaning(self, id, label, type, description=None, values=None, mappings=None, pattern=None, normalizationMode=None, detectable=False): """ Create a meaning, and return a handle to interact with it Note: this call requires an API key with admin rights Args: id: the ID of the new meaning type: the type of the new meaning. Admissible values are 'DECLARATIVE', 'VALUES_LIST', 'VALUES_MAPPING' and 'PATTERN' description (optional): the description of the new meaning values (optional): when type is 'VALUES_LIST', the list of values mappings (optional): when type is 'VALUES_MAPPING', the mapping, as a list of objects with this structure: {'from': 'value_1', 'to': 'value_a'} pattern (optional): when type is 'PATTERN', the pattern normalizationMode (optional): when type is 'VALUES_LIST', 'VALUES_MAPPING' or 'PATTERN', the normalization mode to use for value matching. One of 'EXACT', 'LOWERCASE', or 'NORMALIZED' (not available for 'PATTERN' type). Defaults to 'EXACT'. detectable (optional): whether DSS should consider assigning the meaning to columns set to 'Auto-detect'. Defaults to False. Returns: A :class:`dataikuapi.dss.meaning.DSSMeaning` meaning handle """ resp = self._perform_text( "POST", "/meanings/", body={ "id" : id, "label": label, "type": type, "description": description, "values": values, "mappings": mappings, "pattern": pattern, "normalizationMode": normalizationMode, "detectable": detectable }) return DSSMeaning(self, id) ######################################################## # Logs ######################################################## def list_logs(self): """ List all logs on the DSS instance Note: this call requires an API key with admin rights Returns: A list of log names """ return self._perform_json( "GET", "/admin/logs/") def get_log(self, name): """ Get a specific log Note: this call requires an API key with admin rights Args: name: the name of the desired log Returns: The full log, as a string """ return self._perform_json( "GET", "/admin/logs/%s" % name) ######################################################## # Variables ######################################################## def get_variables(self): """ Get the DSS instance's variables Note: this call requires an API key with admin rights Returns: A JSON object """ return self._perform_json( "GET", "/admin/variables/") def set_variables(self, variables): """ Set the DSS instance's variables Note: this call requires an API key with admin rights Args: variables: the new state of all variables of the instance, as a JSON object """ return self._perform_empty( "PUT", "/admin/variables/", body=variables) ######################################################## # Bundles / Import (Automation node) ######################################################## def create_project_from_bundle_local_archive(self, archive_path): return self._perform_json("POST", "/projectsFromBundle/fromArchive", params = { "archivePath" : osp.abspath(archive_path) }) def create_project_from_bundle_archive(self, fp): files = {'file': fp } return self._perform_json("POST", "/projectsFromBundle/", files=files) def prepare_project_import(self, f): """ Prepares import of a project archive @param: fp: the input stream, as a file-like object Returns a handle for the prepared import """ val = self._perform_json_upload( "POST", "/projects/import/upload", "tmp-import.zip", f) return TemporaryImportHandle(self, val.json()["id"]) ######################################################## # Internal Request handling ######################################################## def _perform_http(self, method, path, params=None, body=None, stream=False, files=None): if body: body = json.dumps(body) auth = HTTPBasicAuth(self.api_key, "") try: http_res = self._session.request( method, "%s/dip/publicapi%s" % (self.host, path), params=params, data=body, files = files, auth=auth, stream = stream) http_res.raise_for_status() return http_res except exceptions.HTTPError: try: ex = http_res.json() except ValueError: ex = {"message": http_res.text} raise DataikuException("%s: %s" % (ex.get("errorType", "Unknown error"), ex.get("message", "No message"))) def _perform_empty(self, method, path, params=None, body=None, files = None): self._perform_http(method, path, params=params, body=body, files=files, stream=False) def _perform_text(self, method, path, params=None, body=None,files=None): return self._perform_http(method, path, params=params, body=body, files=files, stream=False).text def _perform_json(self, method, path, params=None, body=None,files=None): return self._perform_http(method, path, params=params, body=body, files=files, stream=False).json() def _perform_raw(self, method, path, params=None, body=None,files=None): return self._perform_http(method, path, params=params, body=body, files=files, stream=True) def _perform_json_upload(self, method, path, name, f): auth = HTTPBasicAuth(self.api_key, "") try: http_res = self._session.request( method, "%s/dip/publicapi%s" % (self.host, path), auth=auth, files = {'file': (name, f, {'Expires': '0'})} ) http_res.raise_for_status() return http_res except exceptions.HTTPError: ex = http_res.json() raise DataikuException("%s: %s" % (ex.get("errorType", "Unknown error"), ex.get("message", "No message"))) class TemporaryImportHandle(object): def __init__(self, client, import_id): self.client = client self.import_id = import_id def execute(self, settings = {}): """ Executes the import with provided settings. @warning: You must check the 'success' flag """ # Empty JSON dicts can't be parsed properly if settings == {}: settings["_"] = "_" return self.client._perform_json("POST", "/projects/import/%s/process" % (self.import_id), body = settings)dataiku-api-client-python-3.1.2/dataikuapi/utils.py000066400000000000000000000041431275677216400223520ustar00rootroot00000000000000import csv from dateutil import parser as date_iso_parser from itertools import izip_longest from contextlib import closing class DataikuException(Exception): """Exception launched by the Dataiku API clients when an error occurs""" class DataikuUTF8CSVReader(object): """ A CSV reader which will iterate over lines in the CSV file "f", which is encoded in UTF-8. """ def __init__(self, f, **kwds): self.reader = csv.reader(f, **kwds) def next(self): row = self.reader.next() return [unicode(s, "utf-8") for s in row] def __iter__(self): return self def none_if_throws(f): def aux(*args, **kargs): try: return f(*args, **kargs) except: return None return aux class DataikuStreamedHttpUTF8CSVReader(object): """ A CSV reader with a schema """ def __init__(self, schema, csv_stream): self.schema = schema self.csv_stream = csv_stream def iter_rows(self): def decode(x): return unicode(x, "utf8") def parse_iso_date(s): if s == "": return None else: return date_iso_parser.parse(s) def str_to_bool(s): if s is None: return False return s.lower() == "true" CASTERS = { "tinyint" : int, "smallint" : int, "int": int, "bigint": int, "float": float, "double": float, "date": parse_iso_date, "boolean": str_to_bool, } schema = self.schema casters = [ CASTERS.get(col["type"], decode) for col in schema ] with closing(self.csv_stream) as r: for uncasted_tuple in csv.reader(r.raw, delimiter='\t', quotechar='"', doublequote=True): yield [none_if_throws(caster)(val) for (caster, val) in izip_longest(casters, uncasted_tuple)] dataiku-api-client-python-3.1.2/requirements.txt000066400000000000000000000000361275677216400220050ustar00rootroot00000000000000requests python-dateutil nose dataiku-api-client-python-3.1.2/setup.py000066400000000000000000000010231275677216400202300ustar00rootroot00000000000000#!/usr/bin/env python from distutils.core import setup VERSION = "3.1.0" setup( name='dataiku-api-client', version=VERSION, license="Apache Software License", packages=["dataikuapi", "dataikuapi.dss", "dataikuapi.apinode_admin"], description="Python API client for Dataiku APIs", author="Dataiku", author_email="support@dataiku.com", url="https://www.dataiku.com", download_url = "https://github.com/dataiku/dataiku-api-client-python/tarball/3.1.0" ) dataiku-api-client-python-3.1.2/tests/000077500000000000000000000000001275677216400176645ustar00rootroot00000000000000dataiku-api-client-python-3.1.2/tests/__init__.py000066400000000000000000000003651275677216400220010ustar00rootroot00000000000000host="http://localhost:8082" apiKey="ZZYqWxPnc2nWMJMUXwykn6wzA7jokbp5" testProjectKey="IO_HARMO" testDataset="train_set_pg" testHiveDataset="train_set_hived" testPartitionedProjectKey="PARTITIONED" testPartitionedDataset="random_data_prepared" dataiku-api-client-python-3.1.2/tests/connection_tests.py000066400000000000000000000016621275677216400236240ustar00rootroot00000000000000from dataikuapi.dssclient import DSSClient from nose.tools import ok_ from nose.tools import eq_ host="http://localhost:8082" apiKey="ZZYqWxPnc2nWMJMUXwykn6wzA7jokbp5" def list_connections_test(): client = DSSClient(host, apiKey) connections = client.list_connections() ok_(len(connections) > 0) def create_delete_connection_test(): client = DSSClient(host, apiKey) count = len(client.list_connections()) connection = client.create_connection("toto", "HDFS") eq_(count + 1, len(client.list_connections())) connection.delete() eq_(count, len(client.list_connections())) def get_set_connection_test(): client = DSSClient(host, apiKey) connection = client.create_connection("toto", "HDFS") desc = connection.get_definition() desc['usableBy'] = 'ALLOWED' desc['allowedGroups'] = ['a','b'] connection.set_definition(desc) desc2 = connection.get_definition() eq_('ALLOWED', desc2['usableBy']) connection.delete() dataiku-api-client-python-3.1.2/tests/dataset_tests.py000066400000000000000000000043441275677216400231120ustar00rootroot00000000000000from dataikuapi.dssclient import DSSClient import json from nose.tools import ok_ from nose.tools import eq_ host="http://localhost:8082" apiKey="ZZYqWxPnc2nWMJMUXwykn6wzA7jokbp5" testProjectKey="IMPALA" testDataset="tube" testHiveDataset="cat_train_hdfs" testPartitionedProjectKey="PARTITIONED" testPartitionedDataset="rundec2" testDropPartitionedDataset="rundec2" testDropPartition="access" testClearDataset = 'train_set_pg2' def list_datasets_test(): client = DSSClient(host, apiKey) datasets = client.get_project(testProjectKey).list_datasets() ok_(len(datasets) > 0) def dataset_data_test(): client = DSSClient(host, apiKey) p = client.get_project(testProjectKey) d = p.get_dataset(testDataset) counter = 0 for r in d.iter_rows(): counter = counter + 1 if counter > 5: break eq_(6, counter) # note : backend will get a pipe broken def sync_metastore_test(): client = DSSClient(host, apiKey) dataset = client.get_project(testProjectKey).get_dataset(testHiveDataset) dataset.synchronize_hive_metastore() # didn't die def clear_test(): client = DSSClient(host, apiKey) dataset = client.get_project(testProjectKey).get_dataset(testClearDataset) dataset.clear() def list_partitions_test(): client = DSSClient(host, apiKey) dataset = client.get_project(testPartitionedProjectKey).get_dataset(testPartitionedDataset) ok_(len(dataset.list_partitions()) > 0) def clear_partitions_test(): client = DSSClient(host, apiKey) dataset = client.get_project(testPartitionedProjectKey).get_dataset(testDropPartitionedDataset) dataset.clear(json.dumps([testDropPartition])) def create_delete_test(): client = DSSClient(host, apiKey) project = client.get_project(testProjectKey) count = len(project.list_datasets()) dataset = project.create_dataset("titi", "Filesystem") eq_(count + 1, len(project.list_datasets())) dataset.delete() eq_(count, len(project.list_datasets())) def get_set_test(): client = DSSClient(host, apiKey) project = client.get_project(testProjectKey) dataset = project.create_dataset("titi", "Filesystem") definition = dataset.get_definition() definition['managed'] = True dataset.set_definition(definition) definition2 = dataset.get_definition() eq_(True, definition2['managed']) dataset.delete() dataiku-api-client-python-3.1.2/tests/group_tests.py000066400000000000000000000014411275677216400226140ustar00rootroot00000000000000from dataikuapi.dssclient import DSSClient from nose.tools import ok_ from nose.tools import eq_ host="http://localhost:8082" apiKey="ZZYqWxPnc2nWMJMUXwykn6wzA7jokbp5" def list_groups_test(): client = DSSClient(host, apiKey) groups = client.list_groups() ok_(len(groups) > 0) def create_delete_group_test(): client = DSSClient(host, apiKey) count = len(client.list_groups()) group = client.create_group("toto") eq_(count + 1, len(client.list_groups())) group.delete() eq_(count, len(client.list_groups())) def get_set_group_test(): client = DSSClient(host, apiKey) group = client.create_group("toto") desc = group.get_definition() desc['description'] = 'here' group.set_definition(desc) desc2 = group.get_definition() eq_('here', desc2['description']) group.delete() dataiku-api-client-python-3.1.2/tests/job_tests.py000066400000000000000000000025121275677216400222320ustar00rootroot00000000000000from dataikuapi.dssclient import DSSClient from nose.tools import ok_ from nose.tools import eq_ host="http://localhost:8082" apiKey="ZZYqWxPnc2nWMJMUXwykn6wzA7jokbp5" testProjectKey="IMPALA" testDataset="cat_train_hdfs" def list_jobs_test(): client = DSSClient(host, apiKey) jobs = client.get_project(testProjectKey).list_jobs() ok_(len(jobs) > 0) def job_status_test(): client = DSSClient(host, apiKey) project = client.get_project(testProjectKey) jobs = project.list_jobs() for job in jobs: status = project.get_job(job['def']['id']).get_status() ok_(status is not None) def job_log_test(): client = DSSClient(host, apiKey) project = client.get_project(testProjectKey) jobs = project.list_jobs() for job in jobs: log = project.get_job(job['def']['id']).get_log() ok_(log is not None) def job_start_abort_test(): client = DSSClient(host, apiKey) project = client.get_project(testProjectKey) count = len(client.get_project(testProjectKey).list_jobs()) job = project.start_job({ "initiator" : "me", "name" : "some job", "triggeredFrom" : "API", "type" : "NON_RECURSIVE_FORCED_BUILD", "outputs" : [{ "type" : "DATASET", "projectKey" : testProjectKey, "id" : testDataset, "partition" : "NP" }] }) job.abort() eq_(count + 1, len(client.get_project(testProjectKey).list_jobs())) dataiku-api-client-python-3.1.2/tests/log_tests.py000066400000000000000000000005661275677216400222500ustar00rootroot00000000000000from dataikuapi.dssclient import DSSClient from nose.tools import ok_ from nose.tools import eq_ host="http://localhost:8082" apiKey="ZZYqWxPnc2nWMJMUXwykn6wzA7jokbp5" def list_logs_test(): client = DSSClient(host, apiKey) ok_(len(client.list_logs()) > 0) def get_log_test(): client = DSSClient(host, apiKey) log = client.get_log('error.log') ok_(log is not None) dataiku-api-client-python-3.1.2/tests/managedfolder_tests.py000066400000000000000000000120121275677216400242440ustar00rootroot00000000000000from dataikuapi.dssclient import DSSClient from dataikuapi.utils import DataikuException import json from nose.tools import ok_ from nose.tools import eq_ from nose.tools import raises import tempfile import filecmp import os.path as osp import sys, traceback from contextlib import closing host="http://localhost:8082" apiKey="5YZ4lHpXexhlNk29FqRi8AcO2EVGddtA" testProjectKey="BOX3" def list_managedfolders_test(): client = DSSClient(host, apiKey) managedfolders = client.get_project(testProjectKey).list_managed_folders() ok_(len(managedfolders) > 0) def create_delete_test(): client = DSSClient(host, apiKey) project = client.get_project(testProjectKey) count = len(project.list_managed_folders()) managedfolder = project.create_managed_folder("titi") eq_(count + 1, len(project.list_managed_folders())) managedfolder.delete() eq_(count, len(project.list_managed_folders())) def get_set_test(): client = DSSClient(host, apiKey) project = client.get_project(testProjectKey) managedfolder = project.create_managed_folder("titi") definition = managedfolder.get_definition() definition['description'] = 'describe me!' managedfolder.set_definition(definition) definition2 = managedfolder.get_definition() eq_('describe me!', definition2['description']) managedfolder.delete() def updload_download_test(): temp_folder = tempfile.mkdtemp() stuff = osp.join(temp_folder, "test.txt") with open(stuff, "w") as f: f.write('some contents\n on several\nlines') client = DSSClient(host, apiKey) project = client.get_project(testProjectKey) managedfolder = project.create_managed_folder("titi") count = len(managedfolder.list_contents()['items']) with open(stuff, "r") as f: managedfolder.put_file('stuff', f) eq_(count + 1, len(managedfolder.list_contents()['items'])) stuff2 = osp.join(temp_folder, "test2.txt") with open(stuff2, "w") as f: with closing(managedfolder.get_file('stuff')) as s: f.write(s.raw.read()) eq_(True, filecmp.cmp(stuff, stuff2)) managedfolder.delete() def updload_replace_delete_test(): temp_folder = tempfile.mkdtemp() stuff = osp.join(temp_folder, "test.txt") stuffb = osp.join(temp_folder, "testb.txt") with open(stuff, "w") as f: f.write('some contents\n on several\nlines') client = DSSClient(host, apiKey) project = client.get_project(testProjectKey) managedfolder = project.create_managed_folder("titi") count = len(managedfolder.list_contents()['items']) with open(stuff, "r") as f: managedfolder.put_file('stuff', f) eq_(count + 1, len(managedfolder.list_contents()['items'])) with open(stuffb, "w") as f: with closing(managedfolder.get_file('stuff')) as s: f.write(s.raw.read()) eq_(True, filecmp.cmp(stuff, stuffb)) managedfolder.delete_file('stuff') eq_(count, len(managedfolder.list_contents()['items'])) managedfolder.delete() @raises(DataikuException) def access_contents_outside_test(): temp_folder = tempfile.mkdtemp() file_content = 'some contents\n on several\nlines' stuff = osp.join(temp_folder, "test.txt") with open(stuff, "w") as f: f.write(file_content) client = DSSClient(host, apiKey) project = client.get_project(testProjectKey) managedfolder = project.create_managed_folder("titi") count = len(managedfolder.list_contents()['items']) with open(stuff, "r") as f: managedfolder.put_file('stuff', f) eq_(count + 1, len(managedfolder.list_contents()['items'])) with closing(managedfolder.get_file('stuff')) as s: c = s.raw.read() eq_(file_content, c) try: # climb the file hierarchy up to the dip_home with closing(managedfolder.get_file('../../../shared-secret.txt')) as s: c = s.raw.read() raise AssertionError('Access outside folder should fail') finally: managedfolder.delete() @raises(DataikuException) def delete_contents_outside_test(): temp_folder = tempfile.mkdtemp() file_content = 'some contents\n on several\nlines' stuff = osp.join(temp_folder, "test.txt") with open(stuff, "w") as f: f.write(file_content) client = DSSClient(host, apiKey) project = client.get_project(testProjectKey) managedfolder = project.create_managed_folder("titi") count = len(managedfolder.list_contents()['items']) with open(stuff, "r") as f: managedfolder.put_file('stuff', f) eq_(count + 1, len(managedfolder.list_contents()['items'])) with closing(managedfolder.get_file('stuff')) as s: c = s.raw.read() eq_(file_content, c) try: # climb the file hierarchy up to the dip_home managedfolder.delete_file('../../../shared-secret.txt') raise AssertionError('Access outside folder should fail') finally: managedfolder.delete() dataiku-api-client-python-3.1.2/tests/project_tests.py000066400000000000000000000031201275677216400231220ustar00rootroot00000000000000from dataikuapi.dssclient import DSSClient from nose.tools import ok_ from nose.tools import eq_ host="http://localhost:8082" apiKey="ZZYqWxPnc2nWMJMUXwykn6wzA7jokbp5" testProjectKey="MYSQL" def list_projects_test(): client = DSSClient(host, apiKey) projects = client.list_project_keys() ok_(len(projects) > 0) def project_metadata_test(): client = DSSClient(host, apiKey) project = client.get_project(testProjectKey) meta = project.get_metadata() ok_(meta is not None) project.set_metadata(meta) def project_permissions_test(): client = DSSClient(host, apiKey) project = client.get_project(testProjectKey) perms = project.get_permissions() ok_(perms is not None) project.set_permissions(perms) def project_create_delete_test(): client = DSSClient(host, apiKey) count = len(client.list_project_keys()) p = client.create_project("toto","name of toto", "me") eq_(count + 1, len(client.list_project_keys())) p.delete() eq_(count, len(client.list_project_keys())) """ def sql_test(): client = DSSClient(host, apiKey) projects = client.list_project_keys() projectKey = projects[0]['projectKey'] p = client.get_project(projectKey) q = p.sql_query('select * from train_set_pg limit 5', 'local_postgress') for r in q.iter_rows(): print(r) q.verify() def project_export_stream_test(): client = DSSClient(host, apiKey) with client.get_project('PARTITIONED').get_export_stream() as e: with open('export.zip', 'w') as f: f.write(e.read()) def project_export_file_test(): client = DSSClient(host, apiKey) client.get_project('PARTITIONED').export_to_file("ex2.zip") """ dataiku-api-client-python-3.1.2/tests/user_tests.py000066400000000000000000000015741275677216400224450ustar00rootroot00000000000000from dataikuapi.dssclient import DSSClient from nose.tools import ok_ from nose.tools import eq_ host="http://localhost:8082" apiKey="ZZYqWxPnc2nWMJMUXwykn6wzA7jokbp5" def list_users_test(): client = DSSClient(host, apiKey) users = client.list_users() ok_(len(users) > 0) def create_delete_user_test(): client = DSSClient(host, apiKey) count = len(client.list_users()) user = client.create_user("toto", "password", "display name of toto", groups=['a','b']) eq_(count + 1, len(client.list_users())) user.delete() eq_(count, len(client.list_users())) def get_set_user_test(): client = DSSClient(host, apiKey) user = client.create_user("toto", "password", "display name of toto", groups=['a','b']) desc = user.get_definition() desc['displayName'] = 'tata' user.set_definition(desc) desc2 = user.get_definition() eq_('tata', desc2['displayName']) user.delete() dataiku-api-client-python-3.1.2/tests/variables_tests.py000066400000000000000000000005701275677216400234320ustar00rootroot00000000000000from dataikuapi.dssclient import DSSClient host="http://localhost:8082" apiKey="ZZYqWxPnc2nWMJMUXwykn6wzA7jokbp5" def get_variables_test(): client = DSSClient(host, apiKey) print(client.get_variables()) def set_variables_test(): client = DSSClient(host, apiKey) vars = client.get_variables() vars['a'] = 'b' client.set_variables(vars) print(client.get_variables())