sciserver · mtaghiza · Jun 25, 2024 · Dec 8, 2021 · Dec 17, 2021 · Feb 24, 2022
diff --git a/Examples/Sciquery_Demo(long).ipynb b/Examples/Sciquery_Demo(long).ipynb
diff --git a/README.md b/README.md
@@ -12,6 +12,8 @@ Some SciServer tools you can access with this package:
 
  * [CasJobs](http://skyserver.sdss.org/CasJobs): Database storage and querying.
 
+* SciQuery: Advanced Database storage and querying.
+
  * [SciDrive](http://www.scidrive.org/): Drag-and-drop file storage and sharing.
 
  * [SkyServer](http://skyserver.sdss.org/): Access to the SDSS astronomical survey.
@@ -37,6 +39,8 @@ Authors: Gerard Lemson, Manuchehr Taghizadeh-Popp.
 
 ## 3) Installation:
 
+First, add required packages with `pip install -r requirements.txt`
+
 There are 2 possibilities: automatic or manual installation.
 
 ### a) Automatic Installation and Update:

diff --git a/docs_sphinx/SciServer.rst b/docs_sphinx/SciServer.rst
@@ -14,6 +14,14 @@ SciServer.CasJobs module
     :undoc-members:
     :show-inheritance:
 
+SciServer.SciQuery module
+-------------------------
+
+.. automodule:: SciServer.SciQuery
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
 SciServer.Config module
 -----------------------
 

diff --git a/docs_sphinx/conf.py b/docs_sphinx/conf.py
@@ -48,16 +48,16 @@
 
 # General information about the project.
 project = 'SciScript-Python'
-copyright = '2018, SciServer'
+copyright = '2024, SciServer'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 #
 # The short X.Y version.
-version = '2.1.0'
+version = '2.2.0'
 # The full version, including alpha/beta/rc tags.
-release = '2.1.0'
+release = '2.2.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
@@ -182,6 +182,7 @@
 # Output file base name for HTML help builder.
 htmlhelp_basename = 'sphinxdoc'
 
+autoclass_content = 'both'
 
 # -- Options for LaTeX output ---------------------------------------------
 
@@ -268,7 +269,7 @@
 epub_title = '.'
 epub_author = 'Gerard Lemson, Manuchehr Taghizadeh-Popp'
 epub_publisher = 'Gerard Lemson, Manuchehr Taghizadeh-Popp'
-epub_copyright = '2018, SciServer'
+epub_copyright = '2024, SciServer'
 
 # The basename for the epub file. It defaults to the project name.
 #epub_basename = '.'

diff --git a/py2/SciServer/Config.py b/py2/SciServer/Config.py
@@ -31,7 +31,7 @@
 RacmApiURL = "https://apps.sciserver.org/racm"
 DataRelease = "DR14"
 KeystoneTokenPath =  "/home/idies/keystone.token" #the path to the file containing the user's keystone token is hardcoded in the sciserver-compute environment
-version = "sciserver-v2.1.0" #sciserver release version
+version = "sciserver-v2.2.0" #sciserver release version
 ComputeJobDirectoryFile = "/home/idies/jobs.path" #the path to the file in the "Docker job container" that shows the directory path where the asynchronous compute job is being executed.
 
 def _load_config(filename):

diff --git a/py2/SciServer/__init__.py b/py2/SciServer/__init__.py
@@ -41,7 +41,7 @@
 
 **Authors**: Gerard Lemson <[email protected]>, Manuchehr Taghizadeh-Popp <[email protected]>
 
-**Version**: sciserver-v2.1.0
+**Version**: sciserver-v2.2.0
 
 
 """

diff --git a/py2/setup.py b/py2/setup.py
@@ -1,6 +1,6 @@
 from setuptools import setup, find_packages
 setup(
     name = "SciServer",
-    version = "2.1.0",
+    version = "2.2.0",
     packages = find_packages(),
 )
diff --git a/py3/SciServer/Authentication.py b/py3/SciServer/Authentication.py
@@ -66,6 +66,7 @@ def getKeystoneUserWithToken(token):
         ksu = KeystoneUser()
         ksu.userName = responseJson["token"]["user"]["name"]
         ksu.id = responseJson["token"]["user"]["id"]
+        ksu.token = token
         keystoneUser.token = token;
         keystoneUser.userName = ksu.userName
         keystoneUser.id = ksu.id

diff --git a/py3/SciServer/Config.py b/py3/SciServer/Config.py
@@ -22,6 +22,8 @@
 
 - **Config.ComputeUrl**: defines the base URL of the Compute webapp (string). E.g., "https://apps.sciserver.org/compute".
 
+- **Config.SciqueryURL**: defines the base URL of the SciQuery web API (string). E.g., "https://apps.sciserver.org/sciquery-api".
+
 - **Config.version**: defines the SciServer release version tag (string), to which this package belongs. E.g., "sciserver-v1.9.3"
 """
 # URLs for accessing SciServer web services (API endpoints)
@@ -33,17 +35,20 @@
 RacmApiURL = "https://apps.sciserver.org/racm"
 DataRelease = "DR15"
 KeystoneTokenPath =  "/home/idies/keystone.token" #the path to the file containing the user's keystone token is hardcoded in the sciserver-compute environment
-version = "sciserver-v2.1.0" #sciserver release version
+version = "sciserver-v2.2.0" #sciserver release version
 ComputeJobDirectoryFile = "/home/idies/jobs.path" #the path to the file in the "Docker job container" that shows the directory path where the asynchronous compute job is being executed.
 ComputeUrl = "https://apps.sciserver.org/compute"
+SciqueryURL = "https://apps.sciserver.org/sciquery-api"
+ComputeWorkDir = "/home/idies/workspace/"
+
 
 def _load_config(filename):
     if os.path.exists(filename):
         with open(filename) as f:
             _config_data = json.load(f)
             global CasJobsRESTUri, AuthenticationURL, SciDriveHost, SkyQueryUrl, SkyServerWSurl
             global RacmApiURL, DataRelease, KeystoneTokenPath, version, ComputeJobDirectoryFile
-            global ComputeUrl
+            global ComputeUrl, SciqueryURL, ComputeWorkDir
             CasJobsRESTUri = _config_data.get('CasJobsRESTUri', CasJobsRESTUri)
             AuthenticationURL = _config_data.get('AuthenticationURL', AuthenticationURL)
             SciDriveHost = _config_data.get('SciDriveHost', SciDriveHost)
@@ -55,6 +60,8 @@ def _load_config(filename):
             version = _config_data.get('version', version)
             ComputeJobDirectoryFile = _config_data.get('ComputeJobDirectoryFile', ComputeJobDirectoryFile)
             ComputeUrl = _config_data.get('ComputeUrl', ComputeUrl)
+            SciqueryURL = _config_data.get('SciqueryURL', SciqueryURL)
+            ComputeWorkDir = _config_data.get('ComputeWorkDir', ComputeWorkDir)
 
 _CONFIG_DIR = os.environ.get('XDG_CONFIG_HOME', os.path.join(os.path.expanduser('~'), '.config'))
 _SCISERVER_SYSTEM_CONFIG_DIR = '/etc/' # will not likely exist on non *nix systems

diff --git a/py3/SciServer/Jobs.py b/py3/SciServer/Jobs.py
@@ -18,7 +18,7 @@ def getDockerComputeDomains():
     :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the JOBM API returns an error.
     :example: dockerComputeDomains = Jobs.getDockerComputeDomains();
 
-    .. seealso:: Jobs.submitShellCommandJob, Jobs.getJobStatus, Jobs.getRDBComputeDomains, Jobs.cancelJob
+    .. seealso:: Jobs.submitShellCommandJob, Jobs.getJobStatus, Jobs.cancelJob
     """
     token = Authentication.getToken()
     if token is not None and token != "":
@@ -72,7 +72,7 @@ def getDockerComputeDomainFromName(dockerComputeDomainName, dockerComputeDomains
     :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the JOBM API returns an error.
     :example: dockerComputeDomain = Jobs.getDockerComputeDomainFromName('dockerComputeDomainAtJHU');
 
-    .. seealso:: Jobs.getDockerComputeDomains, Jobs.getRDBComputeDomains, Jobs.getRDBComputeDomainFromName
+    .. seealso:: Jobs.getDockerComputeDomains
     """
     if dockerComputeDomainName is None:
         raise Exception("dockerComputeDomainName is not defined.")
@@ -90,83 +90,6 @@ def getDockerComputeDomainFromName(dockerComputeDomainName, dockerComputeDomains
         raise Exception("DockerComputeDomain of name '" + dockerComputeDomainName + "' is not available or does not exist.");
 
 
-def getRDBComputeDomains():
-    """
-    Gets a list of all registered Relational Database (RDB) compute domains that the user has access to.
-
-    :return: a list of dictionaries, each one containing the definition of an RDB compute domain.
-    :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the JOBM API returns an error.
-    :example: rdbComputeDomains = Jobs.getRDBComputeDomains();
-
-    .. seealso:: Jobs.submitShellCommandJob, Jobs.getJobStatus, Jobs.getDockerComputeDomains, Jobs.cancelJob
-    """
-    token = Authentication.getToken()
-    if token is not None and token != "":
-
-        if Config.isSciServerComputeEnvironment():
-            taskName = "Compute.SciScript-Python.Jobs.getRDBComputeDomains"
-        else:
-            taskName = "SciScript-Python.Jobs.getRDBComputeDomains"
-
-        url = Config.RacmApiURL + "/jobm/rest/computedomains/rdb?TaskName=" + taskName
-        headers = {'X-Auth-Token': token, "Content-Type": "application/json"}
-        res = requests.get(url, headers=headers, stream=True)
-        if res.status_code != 200:
-            raise Exception("Error when getting RDB Compute Domains from JOBM API.\nHttp Response from JOBM API returned status code " + str(res.status_code) + ":\n" + res.content.decode());
-        else:
-            return json.loads(res.content.decode())
-    else:
-        raise Exception("User token is not defined. First log into SciServer.")
-
-
-def getRDBComputeDomainsNames(rdbComputeDomains=None):
-    """
-    Returns the names of the RDB compute domains available to the user.
-
-    :param rdbComputeDomains: a list of rdbComputeDomain objects (dictionaries), as returned by Jobs.getRDBComputeDomains(). If not set, then an extra internal call to Jobs.getRDBComputeDomains() is made.
-    :return: an array of strings, each being the name of a rdb compute domain available to the user.
-    :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the RACM API returns an error.
-    :example: dockerComputeDomainsNames = Files.getDockerComputeDomainsNames();
-
-    .. seealso:: Files.getRDBComputeDomains
-    """
-    if rdbComputeDomains is None:
-        rdbComputeDomains = getRDBComputeDomains();
-
-    rdbComputeDomainsNames = [];
-    for rdbComputeDomain in rdbComputeDomains:
-        rdbComputeDomainsNames.append(rdbComputeDomain.get('name'))
-
-    return rdbComputeDomainsNames;
-
-
-def getRDBComputeDomainFromName(rdbComputeDomainName, rdbComputeDomains = None):
-    """
-    Returns an RDBComputeDomain object, given its registered name.
-
-    :param rdbComputeDomainName: name of the RDBComputeDomainName, as shown within the results of Jobs.getRDBComputeDomains()
-    :param rdbComputeDomains: a list of rdbComputeDomain objects (dictionaries), as returned by Jobs.getRDBComputeDomains(). If not set, then an extra internal call to Jobs.getRDBComputeDomains() is made.
-    :return: an RDBComputeDomain object (dictionary) that defines an RDB compute domain. A list of these kind of objects available to the user is returned by the function Jobs.getRDBComputeDomains().
-    :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the JOBM API returns an error.
-    :example: rdbComputeDomain = Jobs.getRDBComputeDomainFromName('rdbComputeDomainAtJHU');
-
-    .. seealso:: Jobs.getDockerComputeDomains, Jobs.getRDBComputeDomains, Jobs.getDockerComputeDomainFromName
-    """
-    if rdbComputeDomainName is None:
-        raise Exception("rdbComputeDomainName is not defined.")
-    else:
-        if rdbComputeDomains is None:
-            rdbComputeDomains = getRDBComputeDomains();
-
-        if rdbComputeDomains.__len__() > 0:
-            for rdbComputeDomain in rdbComputeDomains:
-                if rdbComputeDomainName == rdbComputeDomain.get('name'):
-                    return rdbComputeDomain;
-        else:
-            raise Exception("There are no RDBComputeDomains available for the user.");
-
-        raise Exception("RDBComputeDomain of name '" + rdbComputeDomainName + "' is not available or does not exist.");
-
 
 def getJobsList(top=10, open=None, start=None, end=None, type='all'):
     """
@@ -511,86 +434,6 @@ def submitShellCommandJob(shellCommand, dockerComputeDomain = None, dockerImageN
     else:
         raise Exception("User token is not defined. First log into SciServer.")
 
-def submitRDBQueryJob(sqlQuery, rdbComputeDomain=None, databaseContextName = None, resultsName='queryResults', resultsFolderPath="", jobAlias = ""):
-    """
-    Submits a sql query for execution (as an asynchronous job) inside a relational database (RDB) compute domain.
-
-    :param sqlQuery: sql query (string)
-    :param rdbComputeDomain: object (dictionary) that defines a relational database (RDB) compute domain. A list of these kind of objects available to the user is returned by the function Jobs.getRDBComputeDomains().
-    :param databaseContextName: database context name (string) on which the sql query is executed.
-    :param resultsName: name (string) of the table or file (without file type ending) that contains the query result. In case the sql query has multiple statements, should be set to a list of names (e.g., ['result1','result2']).
-    :param resultsFolderPath: full path to results folder (string) where query output tables are written into. E.g.: /home/idies/workspace/rootVOlume/username/userVolume/jobsFolder . If not set, then a default folder will be set automatically.
-    :param jobAlias: alias (string) of job, defined by the user.
-    :return: a dictionary containing the definition of the submitted job.
-    :raises: Throws an exception if the HTTP request to the Authentication URL returns an error. Throws an exception if the HTTP request to the JOBM API returns an error, or if the volumes defined by the user are not available in the Docker compute domain.
-    :example: job = Jobs.submitRDBQueryJob('select 1';,None, None, 'myQueryResults', 'myNewJob')
-
-    .. seealso:: Jobs.submitNotebookJob, Jobs.submitShellCommandJob, Jobs.getJobStatus, Jobs.getDockerComputeDomains, Jobs.cancelJob
-    """
-
-    token = Authentication.getToken()
-    if token is not None and token != "":
-
-        if Config.isSciServerComputeEnvironment():
-            taskName = "Compute.SciScript-Python.Jobs.submitRDBQueryJob"
-        else:
-            taskName = "SciScript-Python.Jobs.submitRDBQueryJob"
-
-        if rdbComputeDomain is None:
-            rdbComputeDomains = getRDBComputeDomains();
-            if rdbComputeDomains .__len__() > 0:
-                rdbComputeDomain = rdbComputeDomains[0];
-            else:
-                raise Exception("There are no rdbComputeDomains available for the user.");
-
-        if databaseContextName is None:
-            databaseContexts = rdbComputeDomain.get('databaseContexts');
-            if databaseContexts.__len__() > 0:
-                databaseContextName = databaseContexts[0].get('name')
-            else:
-                raise Exception("rbdComputeDomain has no database contexts available for the user.");
-
-        targets = [];
-        if type(resultsName) == str:
-            targets.append({'location': resultsName, 'type': 'FILE_CSV', 'resultNumber': 1});
-        elif type(resultsName) == list:
-            if len(set(resultsName)) != len(resultsName):
-                raise Exception("Elements of parameter 'resultsName' must be unique");
-
-            for i in range(len(resultsName)):
-                if type(resultsName[i]) == str:
-                    targets.append({'location': resultsName[i], 'type': 'FILE_CSV', 'resultNumber': i+1});
-                else:
-                    raise Exception("Elements of array 'resultsName' are not strings");
-
-        else:
-            raise Exception("Type of parameter 'resultsName' is not supported");
-
-
-        rdbDomainId = rdbComputeDomain.get('id');
-
-        dockerJobModel = {
-            "inputSql": sqlQuery,
-            "submitterDID": jobAlias,
-            "databaseContextName": databaseContextName,
-            "rdbDomainId": rdbDomainId,
-            "targets": targets,
-            "resultsFolderURI":resultsFolderPath
-        }
-
-        data = json.dumps(dockerJobModel).encode()
-        url = Config.RacmApiURL + "/jobm/rest/jobs/rdb?TaskName="+taskName;
-        headers = {'X-Auth-Token': token, "Content-Type": "application/json"}
-        res = requests.post(url, data=data, headers=headers, stream=True)
-
-        if res.status_code != 200:
-            raise Exception("Error when submitting a job to the JOBM API.\nHttp Response from JOBM API returned status code " + str(res.status_code) + ":\n" + res.content.decode());
-        else:
-            return (json.loads(res.content.decode())).get('id')
-    else:
-        raise Exception("User token is not defined. First log into SciServer.")
-
-
 def cancelJob(jobId):
     """
     Cancels the execution of a job.
-Original file line number
+Diff line change
@@ Expand Up / @@ -41,7 +41,7 @@ @@
     **Authors**: Gerard Lemson <[email protected]>, Manuchehr Taghizadeh-Popp <[email protected]>
-    **Version**: sciserver-v2.1.0
+    **Version**: sciserver-v2.2.0
     """
@@ Expand Down @@