diff --git a/Makefile b/Makefile index 1f7df67..86a53dc 100755 --- a/Makefile +++ b/Makefile @@ -9,7 +9,8 @@ CPPFLAGS = -fPIC -std=c++0x -g -O2 INCLUDES = -Iinclude -Isrc INCLUDES_PYTHON = $(INCLUDES) \ - -I/usr/include/python3.6 + -I/usr/include/python3.6 \ + -I$(HOME)/.pyenv/versions/3.6.4/lib/python3.6/site-packages/numpy/core/include PROGRAM = _griddb_python.so EXTRA = griddb_python.py griddb_python.pyc @@ -24,6 +25,7 @@ SOURCES = src/TimeSeriesProperties.cpp \ src/Query.cpp \ src/QueryAnalysisEntry.cpp \ src/RowKeyPredicate.cpp \ + src/RowList.cpp \ src/RowSet.cpp \ src/TimestampUtils.cpp \ src/Field.cpp \ diff --git a/README.md b/README.md index ebb404d..a78d06e 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,11 @@ Set CPATH and LIBRARY_PATH. export LIBRARY_PATH=$LIBRARY_PATH: +Install Pandas and Numpy as below: + + $ python -m pip install numpy + $ python -m pip install pandas + ### Build and Run 1. Execute the command on project directory. diff --git a/sample/FetchRowsWithDataFrame.py b/sample/FetchRowsWithDataFrame.py new file mode 100755 index 0000000..94ef59f --- /dev/null +++ b/sample/FetchRowsWithDataFrame.py @@ -0,0 +1,47 @@ +#!/usr/bin/python + +import griddb_python as griddb +import sys +import pandas + +factory = griddb.StoreFactory.get_instance() + +argv = sys.argv + +blob = bytearray([65, 66, 67, 68, 69, 70, 71, 72, 73, 74]) +containerName = "SamplePython_FetchRows" +update = False + +try: + # Get GridStore object + gridstore = factory.get_store(host=argv[1], port=int(argv[2]), cluster_name=argv[3], username=argv[4], password=argv[5]) + + # Create Collection + conInfo = griddb.ContainerInfo(containerName, + [["name", griddb.Type.STRING], + ["status", griddb.Type.BOOL], + ["count", griddb.Type.LONG], + ["lob", griddb.Type.BLOB]], + griddb.ContainerType.COLLECTION, True) + col = gridstore.put_container(conInfo) + print("Create Collection name=", containerName) + + # Put rows + rows = pandas.DataFrame([["name01", True, 1, blob], ["name02", False, 2, blob]]) + col.put_rows(rows) + print("Put rows with DataFrame") + + # Fetch rows + query = col.query("select *") + rs = query.fetch(update) + print("Fetch rows with DataFrame") + result = rs.fetch_rows() + print(result) + print("Success!") + +except griddb.GSException as e: + for i in range(e.get_error_stack_size()): + print("[", i, "]") + print(e.get_error_code(i)) + print(e.get_location(i)) + print(e.get_message(i)) diff --git a/sample/PutRowsWithDataFrame.py b/sample/PutRowsWithDataFrame.py new file mode 100755 index 0000000..caab037 --- /dev/null +++ b/sample/PutRowsWithDataFrame.py @@ -0,0 +1,40 @@ +#!/usr/bin/python + +import griddb_python as griddb +import sys +import pandas + +factory = griddb.StoreFactory.get_instance() + +argv = sys.argv + +blob = bytearray([65, 66, 67, 68, 69, 70, 71, 72, 73, 74]) +update = False +containerName = "SamplePython_PutRows" + +try: + # Get GridStore object + gridstore = factory.get_store(host=argv[1], port=int(argv[2]), cluster_name=argv[3], username=argv[4], password=argv[5]) + + # Create Collection + conInfo = griddb.ContainerInfo(containerName, + [["name", griddb.Type.STRING], + ["status", griddb.Type.BOOL], + ["count", griddb.Type.LONG], + ["lob", griddb.Type.BLOB]], + griddb.ContainerType.COLLECTION, True) + col = gridstore.put_container(conInfo) + print("Create Collection name=", containerName) + + # Put rows + rows = pandas.DataFrame([["name01", False, 1, blob], ["name02", False, 1, blob]]) + col.put_rows(rows) + print("Put rows with DataFrame") + print("Success!") + +except griddb.GSException as e: + for i in range(e.get_error_stack_size()): + print("[", i, "]") + print(e.get_error_code(i)) + print(e.get_location(i)) + print(e.get_message(i)) diff --git a/src/Container.cpp b/src/Container.cpp index 8bdc1c0..183f975 100755 --- a/src/Container.cpp +++ b/src/Container.cpp @@ -61,7 +61,7 @@ namespace griddb { freeMemoryContainer(); throw GSException(mContainer, "Memory allocation error"); } - + mContainerInfo->timeSeriesProperties = NULL; mContainerInfo->triggerInfoList = NULL; mContainerInfo->dataAffinity = NULL; @@ -74,8 +74,6 @@ namespace griddb { } Container::~Container() { - - // allRelated = FALSE, since all row object is managed by Row class close(GS_FALSE); } @@ -409,4 +407,11 @@ namespace griddb { int Container::getColumnCount(){ return mContainerInfo->columnCount; } + + /** + * @brief Put rows with input is numpy data + */ + void Container::put_rows(GSRow** listRow, int rowCount) { + this->multi_put(listRow, rowCount); + } } diff --git a/src/Container.h b/src/Container.h index 74f7f47..6d9f86a 100755 --- a/src/Container.h +++ b/src/Container.h @@ -58,6 +58,7 @@ class Container { GSType* getGSTypeList(); int getColumnCount(); GSRow* getGSRowPtr(); + void put_rows(GSRow** listRow, int rowCount); private: Container(GSContainer *container, GSContainerInfo* containerInfo); diff --git a/src/Query.cpp b/src/Query.cpp old mode 100644 new mode 100755 diff --git a/src/Query.h b/src/Query.h old mode 100644 new mode 100755 diff --git a/src/RowList.cpp b/src/RowList.cpp new file mode 100644 index 0000000..7101d1d --- /dev/null +++ b/src/RowList.cpp @@ -0,0 +1,69 @@ +/* + Copyright (c) 2017 TOSHIBA Digital Solutions Corporation. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "RowList.h" + +namespace griddb { + + RowList::RowList(GSRow *gsRow, GSRowSet *gsRowSet, GSType *typelist, + int columnCount, bool timetampFloat) : + mRowSet(gsRowSet), mRow(gsRow), mTypelist(typelist), + mColumnCount(columnCount), mTimetampFloat(timetampFloat) { + } + + /** + * Support iterator object. + */ + RowList* RowList::__iter__() { + return this; + } + + /** + * Support iterator object: get next row + */ + void RowList::__next__(bool* hasRow) { + *hasRow = gsHasNextRow(mRowSet); + if (*hasRow) { + gsGetNextRow(mRowSet, mRow); + } + } + + /** + * Refer GSRow pointer from RowSet + */ + GSRow* RowList::get_gsrow_ptr() { + return this->mRow; + } + + /** + * Refer GSType pointer from RowSet + */ + GSType* RowList::get_gstype_list() { + return mTypelist; + } + + /** + * Refer number column from RowSet + */ + int RowList::get_column_count() { + return mColumnCount; + } + + /** + * Refer number column from RowSet + */ + bool RowList::get_timestamp_to_float() { + return mTimetampFloat; + } + +} // namespace griddb diff --git a/src/RowList.h b/src/RowList.h new file mode 100644 index 0000000..993508f --- /dev/null +++ b/src/RowList.h @@ -0,0 +1,39 @@ +/* + Copyright (c) 2017 TOSHIBA Digital Solutions Corporation. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef _ROWLIST_H_ +#define _ROWLIST_H_ + +#include "gridstore.h" + +namespace griddb { +class RowList { + private: + GSRowSet *mRowSet; + GSRow *mRow; + GSType* mTypelist; + int mColumnCount; + bool mTimetampFloat; + public: + RowList(GSRow *gsRow, GSRowSet *gsRowSet, GSType* typelist, int columnCount, + bool timetampFloat); + void __next__(bool* hasRow); + RowList* __iter__(); + GSRow* get_gsrow_ptr(); + GSType* get_gstype_list(); + int get_column_count(); + bool get_timestamp_to_float(); +}; +} // namespace griddb + +#endif // _ROWLIST_H_ diff --git a/src/RowSet.cpp b/src/RowSet.cpp index b0ce22e..35bd331 100755 --- a/src/RowSet.cpp +++ b/src/RowSet.cpp @@ -239,7 +239,7 @@ namespace griddb { } catch (bad_alloc& ba) { throw GSException(mRowSet, "Memory allocation error"); } - + for (int i = 0; i < mContainerInfo->columnCount; i++){ typeList[i] = mContainerInfo->columnInfoList[i].type; } @@ -263,4 +263,19 @@ namespace griddb { return mRow; } + /** + * @brief Get row list data + * @param **row List row data + * @param **rowSet Rowset data + * @return A pointer store row list data in RowList object + */ + griddb::RowList* RowSet::fetch_rows(GSRow **row, GSRowSet **rowSet) { + try { + return new RowList(this->getGSRowPtr(), this->mRowSet, + this->getGSTypeList(), this->getColumnCount(), + this->timestamp_output_with_float); + } catch (bad_alloc &ba) { + throw GSException(mRowSet, "Memory allocation error"); + } + } } diff --git a/src/RowSet.h b/src/RowSet.h index 9dad3d8..f33389c 100755 --- a/src/RowSet.h +++ b/src/RowSet.h @@ -29,6 +29,7 @@ #include "QueryAnalysisEntry.h" #include "GSException.h" #include "Util.h" +#include "RowList.h" using namespace std; @@ -67,6 +68,7 @@ class RowSet { int getColumnCount(); GSRow* getGSRowPtr(); + griddb::RowList* fetch_rows(GSRow** row, GSRowSet** rowSet); private: RowSet(GSRowSet *rowSet, GSContainerInfo *containerInfo, GSRow *mRow); diff --git a/src/griddb.i b/src/griddb.i index 20a11fc..c5eaf91 100755 --- a/src/griddb.i +++ b/src/griddb.i @@ -42,6 +42,7 @@ %feature("new") griddb::Query::get_row_set; %feature("new") griddb::RowSet::get_next_query_analysis; %feature("new") griddb::RowSet::get_next_aggregation; +%feature("new") griddb::RowSet::fetch_rows; %feature("new") griddb::Store::put_container; %feature("new") griddb::Store::get_container; %feature("new") griddb::Store::get_container_info; @@ -83,6 +84,7 @@ #include "RowKeyPredicate.h" #include "Store.h" #include "StoreFactory.h" +#include "RowList.h" %} #if !defined(SWIGJAVASCRIPT) %{ @@ -109,6 +111,7 @@ %shared_ptr(griddb::RowKeyPredicate) %shared_ptr(griddb::Store) %shared_ptr(griddb::PartitionController) +%shared_ptr(griddb::RowList) #endif %include "GSException.h" @@ -125,6 +128,7 @@ %include "RowKeyPredicate.h" %include "Store.h" %include "StoreFactory.h" +%include "RowList.h" #if !defined(SWIGJAVASCRIPT) %include "TimestampUtils.h" #endif diff --git a/src/gstype_python.i b/src/gstype_python.i index 50483b5..79a857a 100755 --- a/src/gstype_python.i +++ b/src/gstype_python.i @@ -18,6 +18,9 @@ #include #include #include +#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION +#include +#include %} %ignore griddb::AggregationResult::setOutputTimestamp; %ignore griddb::Container::setOutputTimestamp; @@ -31,6 +34,7 @@ %pythonbegin %{ from enum import IntEnum +import pandas %} %pythoncode { @@ -933,11 +937,11 @@ static bool convertToFieldWithType(GSRow *row, int column, PyObject* value, GSTy /** * Typemaps for new ContainerInfo() */ -%typemap(in, fragment = "SWIG_AsCharPtrAndSize", fragment = "cleanString") +%typemap(in, fragment = "SWIG_AsCharPtrAndSize", fragment = "cleanString") (const GSColumnInfo* props, int propsCount) (PyObject* list, int i, size_t size = 0, int* alloc = 0, int res, char* v = 0) { //Convert Python list of tuple into GSColumnInfo properties - if (!PyList_Check($input)) { + if (!PyList_Check($input)) { $2 = 0; $1 = NULL; PyErr_SetString(PyExc_ValueError, "Expected a List"); @@ -1113,7 +1117,7 @@ static bool convertToFieldWithType(GSRow *row, int column, PyObject* value, GSTy /** * Typemaps for fetch_all() function */ -%typemap(in) (GSQuery* const* queryList, size_t queryCount) +%typemap(in) (GSQuery* const* queryList, size_t queryCount) (PyObject* pyQuery, std::shared_ptr query, void *vquery, int i, int res = 0) { if ($input == Py_None) { $1 = NULL; @@ -1168,8 +1172,8 @@ static bool convertToFieldWithType(GSRow *row, int column, PyObject* value, GSTy %typemap(in, fragment = "SWIG_AsCharPtrAndSize") (const GSRowKeyPredicateEntry *const * predicateList, size_t predicateCount , GSContainerRowEntry **entryList, size_t* containerCount, int **colNumList, GSType*** typeList, int **orderFromInput) - (PyObject* key, PyObject* val, std::shared_ptr pPredicate, GSRowKeyPredicateEntry* pList = NULL, - void *vpredicate, Py_ssize_t si, int i, int res = 0, size_t size = 0, int* alloc = 0, char* v = 0, + (PyObject* key, PyObject* val, std::shared_ptr pPredicate, GSRowKeyPredicateEntry* pList = NULL, + void *vpredicate, Py_ssize_t si, int i, int res = 0, size_t size = 0, int* alloc = 0, char* v = 0, GSContainerRowEntry *tmpEntryList, size_t tmpContainerCount, int *tmpcolNumList, GSType** tmpTypeList, int *tmpOrderFromInput) { if (!PyDict_Check($input)) { PyErr_SetString(PyExc_ValueError, "Expected a Dict"); @@ -1217,11 +1221,11 @@ static bool convertToFieldWithType(GSRow *row, int column, PyObject* value, GSTy } i++; } - + } } -%typemap(argout, numinputs = 1, fragment = "convertStrToObj", fragment = "getRowFields") +%typemap(argout, numinputs = 1, fragment = "convertStrToObj", fragment = "getRowFields") (const GSRowKeyPredicateEntry *const * predicateList, size_t predicateCount , GSContainerRowEntry **entryList, size_t* containerCount, int **colNumList, GSType*** typeList, int **orderFromInput) () { PyObject* dict = PyDict_New(); @@ -1441,207 +1445,466 @@ static bool convertToFieldWithType(GSRow *row, int column, PyObject* value, GSTy } /** - * Support convert data from GSRow* row to Python list + * Support convert data from GSRow* row to Python list */ -%fragment("getRowFields", "header", +%fragment("getRowFields", "header", fragment = "checkNullField", fragment = "convertStrToObj", fragment = "convertTimestampToObject") { -static bool getRowFields(GSRow* row, int columnCount, GSType* typeList, bool timestampOutput, int* columnError, +static bool getRowFields(GSRow* row, int columnCount, GSType* typeList, bool timestampOutput, int* columnError, GSType* fieldTypeError, PyObject* outList) { GSResult ret; bool retVal = true; for (int i = 0; i < columnCount; i++) { - //Check NULL value - GSBool nullValue; - ret = gsGetRowFieldNull(row, (int32_t) i, &nullValue); - if (ret != GS_RESULT_OK) { - *columnError = i; - retVal = false; - *fieldTypeError = GS_TYPE_NULL; - return retVal; - } - if (nullValue) { - Py_INCREF(Py_None); - PyList_SetItem(outList, i, Py_None); - continue; - } - switch(typeList[i]) { + // Column and type index in error case + *columnError = i; + *fieldTypeError = typeList[i]; + + switch (typeList[i]) { case GS_TYPE_LONG: { int64_t longValue; ret = gsGetRowFieldAsLong(row, (int32_t) i, &longValue); - PyList_SetItem(outList, i, SWIG_From_dec(long long)(longValue)); + if (ret != GS_RESULT_OK) { + return false; + } + if (longValue) { + PyList_SetItem(outList, i, PyLong_FromLong(longValue)); + } else if (checkNullField(row, i)) { + // NULL value + Py_INCREF(Py_None); + PyList_SetItem(outList, i, Py_None); + } else { + PyList_SetItem(outList, i, PyLong_FromLong(longValue)); + } break; } case GS_TYPE_STRING: { GSChar* stringValue; - ret = gsGetRowFieldAsString(row, (int32_t) i, (const GSChar **)&stringValue); - PyList_SetItem(outList, i, convertStrToObj(stringValue)); + ret = gsGetRowFieldAsString(row, (int32_t) i, + (const GSChar **)&stringValue); + if (ret != GS_RESULT_OK) { + return false; + } + if ((stringValue != NULL) && (stringValue[0] == '\0')) { + // Empty string + if (checkNullField(row, i)) { + // NULL value + Py_INCREF(Py_None); + PyList_SetItem(outList, i, Py_None); + } else { + PyList_SetItem(outList, i, + convertStrToObj(stringValue)); + } + } else { + PyList_SetItem(outList, i, convertStrToObj(stringValue)); + } + break; } case GS_TYPE_BLOB: { GSBlob blobValue = {0}; ret = gsGetRowFieldAsBlob(row, (int32_t) i, &blobValue); - PyList_SetItem(outList, i, PyByteArray_FromStringAndSize((const char*)blobValue.data, blobValue.size)); + if (ret != GS_RESULT_OK) { + return false; + } + if (blobValue.size) { + PyList_SetItem(outList, i, + PyByteArray_FromStringAndSize( + (const char*)blobValue.data, blobValue.size)); + } else if (checkNullField(row, i)) { + // NULL value + Py_INCREF(Py_None); + PyList_SetItem(outList, i, Py_None); + } else { + PyList_SetItem(outList, i, + PyByteArray_FromStringAndSize( + (const char*)blobValue.data, blobValue.size)); + } break; } case GS_TYPE_BOOL: { GSBool boolValue; ret = gsGetRowFieldAsBool(row, (int32_t) i, &boolValue); - PyList_SetItem(outList, i, PyBool_FromLong(boolValue)); + if (ret != GS_RESULT_OK) { + return false; + } + if (boolValue) { + PyList_SetItem(outList, i, PyBool_FromLong(boolValue)); + } else if (checkNullField(row, i)) { + // NULL value + Py_INCREF(Py_None); + PyList_SetItem(outList, i, Py_None); + } else { + PyList_SetItem(outList, i, PyBool_FromLong(boolValue)); + } break; } case GS_TYPE_INTEGER: { int32_t intValue; ret = gsGetRowFieldAsInteger(row, (int32_t) i, &intValue); - PyList_SetItem(outList, i, PyInt_FromLong(intValue)); + if (ret != GS_RESULT_OK) { + return false; + } + if (intValue) { + PyList_SetItem(outList, i, PyInt_FromLong(intValue)); + } else if (checkNullField(row, i)) { + // NULL value + Py_INCREF(Py_None); + PyList_SetItem(outList, i, Py_None); + } else { + PyList_SetItem(outList, i, PyInt_FromLong(intValue)); + } break; } case GS_TYPE_FLOAT: { float floatValue; ret = gsGetRowFieldAsFloat(row, (int32_t) i, &floatValue); - PyList_SetItem(outList, i, PyFloat_FromDouble(floatValue)); + if (ret != GS_RESULT_OK) { + return false; + } + if (floatValue) { + PyList_SetItem(outList, i, PyFloat_FromDouble(floatValue)); + } else if (checkNullField(row, i)) { + // NULL value + Py_INCREF(Py_None); + PyList_SetItem(outList, i, Py_None); + } else { + PyList_SetItem(outList, i, PyFloat_FromDouble(floatValue)); + } break; } case GS_TYPE_DOUBLE: { double doubleValue; ret = gsGetRowFieldAsDouble(row, (int32_t) i, &doubleValue); - PyList_SetItem(outList, i, PyFloat_FromDouble(doubleValue)); + if (ret != GS_RESULT_OK) { + return false; + } + if (doubleValue) { + PyList_SetItem(outList, i, PyFloat_FromDouble(doubleValue)); + } else if (checkNullField(row, i)) { + // NULL value + Py_INCREF(Py_None); + PyList_SetItem(outList, i, Py_None); + } else { + PyList_SetItem(outList, i, PyFloat_FromDouble(doubleValue)); + } break; } case GS_TYPE_TIMESTAMP: { GSTimestamp timestampValue; - ret = gsGetRowFieldAsTimestamp(row, (int32_t) i, ×tampValue); - PyList_SetItem(outList, i, convertTimestampToObject(×tampValue, timestampOutput)); + ret = gsGetRowFieldAsTimestamp(row, + (int32_t) i, ×tampValue); + if (ret != GS_RESULT_OK) { + return false; + } + if (timestampValue) { + PyList_SetItem(outList, i, + convertTimestampToObject(×tampValue, timestampOutput)); + } else if (checkNullField(row, i)) { + // NULL value + Py_INCREF(Py_None); + PyList_SetItem(outList, i, Py_None); + } else { + PyList_SetItem(outList, i, + convertTimestampToObject(×tampValue, timestampOutput)); + } break; } case GS_TYPE_BYTE: { int8_t byteValue; ret = gsGetRowFieldAsByte(row, (int32_t) i, &byteValue); - PyList_SetItem(outList, i, PyInt_FromLong(byteValue)); + if (ret != GS_RESULT_OK) { + return false; + } + if (byteValue) { + PyList_SetItem(outList, i, PyInt_FromLong(byteValue)); + } else if (checkNullField(row, i)) { + // NULL value + Py_INCREF(Py_None); + PyList_SetItem(outList, i, Py_None); + } else { + PyList_SetItem(outList, i, PyInt_FromLong(byteValue)); + } break; } case GS_TYPE_SHORT: { int16_t shortValue; ret = gsGetRowFieldAsShort(row, (int32_t) i, &shortValue); - PyList_SetItem(outList, i, PyInt_FromLong(shortValue)); + if (ret != GS_RESULT_OK) { + return false; + } + if (shortValue) { + PyList_SetItem(outList, i, PyInt_FromLong(shortValue)); + } else if (checkNullField(row, i)) { + // NULL value + Py_INCREF(Py_None); + PyList_SetItem(outList, i, Py_None); + } else { + PyList_SetItem(outList, i, PyInt_FromLong(shortValue)); + } break; } case GS_TYPE_GEOMETRY: { GSChar* geoValue; ret = gsGetRowFieldAsGeometry(row, (int32_t) i, (const GSChar **)&geoValue); - PyList_SetItem(outList, i, convertStrToObj(geoValue)); + if (ret != GS_RESULT_OK) { + return false; + } + if ((geoValue != NULL) && (geoValue[0] == '\0')) { + // Empty string + if (checkNullField(row, i)) { + // NULL value + Py_INCREF(Py_None); + PyList_SetItem(outList, i, Py_None); + } else { + PyList_SetItem(outList, i, convertStrToObj(geoValue)); + } + } else { + PyList_SetItem(outList, i, convertStrToObj(geoValue)); + } break; } case GS_TYPE_INTEGER_ARRAY: { int32_t* intArr; size_t size; - ret = gsGetRowFieldAsIntegerArray (row, (int32_t) i, (const int32_t **)&intArr, &size); + ret = gsGetRowFieldAsIntegerArray(row, (int32_t) i, + (const int32_t **)&intArr, &size); + if (ret != GS_RESULT_OK) { + return false; + } PyObject* list = PyList_New(size); + if (!list) { + return false; + } for (int j = 0; j < size; j++) { PyList_SetItem(list, j, PyInt_FromLong(intArr[j])); } - PyList_SetItem(outList, i, list); + + if (size) { + PyList_SetItem(outList, i, list); + } else if (checkNullField(row, i)) { + // NULL value + Py_INCREF(Py_None); + PyList_SetItem(outList, i, Py_None); + } else { + PyList_SetItem(outList, i, list); + } break; } case GS_TYPE_STRING_ARRAY: { GSChar** stringArrVal; size_t size; - ret = gsGetRowFieldAsStringArray (row, (int32_t) i, ( const GSChar *const **)&stringArrVal, &size); + ret = gsGetRowFieldAsStringArray(row, (int32_t) i, + (const GSChar *const **)&stringArrVal, &size); + if (ret != GS_RESULT_OK) { + return false; + } PyObject* list = PyList_New(size); + if (!list) { + return false; + } for (int j = 0; j < size; j++) { PyList_SetItem(list, j, convertStrToObj(stringArrVal[j])); } - PyList_SetItem(outList, i, list); + if (size) { + PyList_SetItem(outList, i, list); + } else if (checkNullField(row, i)) { + // NULL value + Py_INCREF(Py_None); + PyList_SetItem(outList, i, Py_None); + } else { + PyList_SetItem(outList, i, list); + } break; } case GS_TYPE_BOOL_ARRAY: { GSBool* boolArr; size_t size; - ret = gsGetRowFieldAsBoolArray(row, (int32_t) i, (const GSBool **)&boolArr, &size); + ret = gsGetRowFieldAsBoolArray(row, (int32_t) i, + (const GSBool **)&boolArr, &size); + if (ret != GS_RESULT_OK) { + return false; + } PyObject* list = PyList_New(size); + if (!list) { + return false; + } for (int j = 0; j < size; j++) { PyList_SetItem(list, j, PyBool_FromLong(boolArr[j])); } - PyList_SetItem(outList, i, list); + if (size) { + PyList_SetItem(outList, i, list); + } else if (checkNullField(row, i)) { + // NULL value + Py_INCREF(Py_None); + PyList_SetItem(outList, i, Py_None); + } else { + PyList_SetItem(outList, i, list); + } break; } case GS_TYPE_BYTE_ARRAY: { int8_t* byteArr; size_t size; - ret = gsGetRowFieldAsByteArray(row, (int32_t) i, (const int8_t **)&byteArr, &size); + ret = gsGetRowFieldAsByteArray(row, (int32_t) i, + (const int8_t **)&byteArr, &size); + if (ret != GS_RESULT_OK) { + return false; + } PyObject* list = PyList_New(size); + if (!list) { + return false; + } for (int j = 0; j < size; j++) { PyList_SetItem(list, j, PyInt_FromLong(byteArr[j])); } - PyList_SetItem(outList, i, list); + if (size) { + PyList_SetItem(outList, i, list); + } else if (checkNullField(row, i)) { + // NULL value + Py_INCREF(Py_None); + PyList_SetItem(outList, i, Py_None); + } else { + PyList_SetItem(outList, i, list); + } break; } case GS_TYPE_SHORT_ARRAY: { int16_t* shortArr; size_t size; - ret = gsGetRowFieldAsShortArray(row, (int32_t) i, (const int16_t **)&shortArr, &size); + ret = gsGetRowFieldAsShortArray(row, (int32_t) i, + (const int16_t **)&shortArr, &size); + if (ret != GS_RESULT_OK) { + return false; + } PyObject* list = PyList_New(size); + if (!list) { + return false; + } for (int j = 0; j < size; j++) { PyList_SetItem(list, j, PyInt_FromLong(shortArr[j])); } - PyList_SetItem(outList, i, list); + if (size) { + PyList_SetItem(outList, i, list); + } else if (checkNullField(row, i)) { + // NULL value + Py_INCREF(Py_None); + PyList_SetItem(outList, i, Py_None); + } else { + PyList_SetItem(outList, i, list); + } break; } case GS_TYPE_LONG_ARRAY: { int64_t* longArr; size_t size; - ret = gsGetRowFieldAsLongArray(row, (int32_t) i, (const int64_t **)&longArr, &size); + ret = gsGetRowFieldAsLongArray(row, (int32_t) i, + (const int64_t **)&longArr, &size); + if (ret != GS_RESULT_OK) { + return false; + } PyObject* list = PyList_New(size); + if (!list) { + return false; + } for (int j = 0; j < size; j++) { - PyList_SetItem(list, j, SWIG_From_dec(long long)(longArr[j])); + PyList_SetItem(list, j, PyLong_FromLong(longArr[j])); + } + if (size) { + PyList_SetItem(outList, i, list); + } else if (checkNullField(row, i)) { + // NULL value + Py_INCREF(Py_None); + PyList_SetItem(outList, i, Py_None); + } else { + PyList_SetItem(outList, i, list); } - PyList_SetItem(outList, i, list); break; } case GS_TYPE_FLOAT_ARRAY: { float* floatArr; size_t size; - ret = gsGetRowFieldAsFloatArray(row, (int32_t) i, (const float **)&floatArr, &size); + ret = gsGetRowFieldAsFloatArray(row, (int32_t) i, + (const float **)&floatArr, &size); + if (ret != GS_RESULT_OK) { + return false; + } PyObject* list = PyList_New(size); + if (!list) { + return false; + } for (int j = 0; j < size; j++) { - PyList_SetItem(list, j, PyFloat_FromDouble(static_cast(floatArr[j]))); + PyList_SetItem(list, j, + PyFloat_FromDouble(static_cast(floatArr[j]))); + } + if (size) { + PyList_SetItem(outList, i, list); + } else if (checkNullField(row, i)) { + // NULL value + Py_INCREF(Py_None); + PyList_SetItem(outList, i, Py_None); + } else { + PyList_SetItem(outList, i, list); } - PyList_SetItem(outList, i, list); break; } case GS_TYPE_DOUBLE_ARRAY: { double* doubleArr; size_t size; - ret = gsGetRowFieldAsDoubleArray(row, (int32_t) i, (const double **)&doubleArr, &size); + ret = gsGetRowFieldAsDoubleArray(row, (int32_t) i, + (const double **)&doubleArr, &size); + if (ret != GS_RESULT_OK) { + return false; + } PyObject* list = PyList_New(size); + if (!list) { + return false; + } for (int j = 0; j < size; j++) { PyList_SetItem(list, j, PyFloat_FromDouble(doubleArr[j])); } - PyList_SetItem(outList, i, list); + if (size) { + PyList_SetItem(outList, i, list); + } else if (checkNullField(row, i)) { + // NULL value + Py_INCREF(Py_None); + PyList_SetItem(outList, i, Py_None); + } else { + PyList_SetItem(outList, i, list); + } break; } case GS_TYPE_TIMESTAMP_ARRAY: { GSTimestamp* timestampArr; size_t size; - ret = gsGetRowFieldAsTimestampArray(row, (int32_t) i, (const GSTimestamp **)×tampArr, &size); + ret = gsGetRowFieldAsTimestampArray(row, (int32_t) i, + (const GSTimestamp **)×tampArr, &size); + if (ret != GS_RESULT_OK) { + return false; + } PyObject* list = PyList_New(size); + if (!list) { + return false; + } for (int j = 0; j < size; j++) { - PyList_SetItem(list, j, convertTimestampToObject(×tampArr[j], timestampOutput)); + PyList_SetItem(list, j, + convertTimestampToObject(×tampArr[j], timestampOutput)); + } + if (size) { + PyList_SetItem(outList, i, list); + } else if (checkNullField(row, i)) { + // NULL value + Py_INCREF(Py_None); + PyList_SetItem(outList, i, Py_None); + } else { + PyList_SetItem(outList, i, list); } - PyList_SetItem(outList, i, list); break; } default: { // NOT OK - ret = -1; + retVal = false; break; } } - if (ret != GS_RESULT_OK) { - *columnError = i; - *fieldTypeError = typeList[i]; - retVal = false; - return retVal; - } } return retVal; } @@ -2266,7 +2529,7 @@ static bool getRowFields(GSRow* row, int columnCount, GSType* typeList, bool tim * Support close method : Store.close() */ %typemap(in) GSBool allRelated{ - + bool tmpBool; int checkConvert = 0; @@ -2279,3 +2542,179 @@ static bool getRowFields(GSRow* row, int columnCount, GSType* typeList, bool tim $1 = ((tmpBool == true) ? GS_TRUE : GS_FALSE); } + +%fragment("checkNullField", "header") { +static bool checkNullField(GSRow* row, int32_t rowField) { + GSBool nullValue; + GSResult ret; + + ret = gsGetRowFieldNull(row, (int32_t) rowField, &nullValue); + if (ret != GS_RESULT_OK) { + return false; + } + if (nullValue == GS_TRUE) { + return true; + } + return false; +} +} + +// Typemap for Container.put_rows() +%typemap(in, numinputs = 0) (GSRow** row, GSRowSet** rowSet) { + GSRow* rowPtr; + GSRowSet* rowSetPtr; + $1 = &rowPtr; + $2 = &rowSetPtr; +} + +%feature("pythonappend") griddb::RowSet::fetch_rows(GSRow** row, + GSRowSet** rowSet) %{ + #convert data from numpy.ndarray to pandas.DataFrame + #"val" is output + columnsList = self.get_column_names() + val = pandas.DataFrame(val, columns = columnsList) +%} + +%typemap(in, numinputs = 0) (bool* hasRow) { + bool hasNextRowTmp = true; + $1 = &hasNextRowTmp; +} + +%typemap(argout, fragment = "getRowFields") (bool* hasRow) { + bool retVal; + int errorColumn; + if (*$1 == false) { + PyErr_SetNone(PyExc_StopIteration); + return NULL; + } + GSRow* row = arg1->get_gsrow_ptr(); + PyObject *outList = PyList_New(arg1->get_column_count()); + if (outList == NULL) { + PyErr_SetString(PyExc_ValueError, "Memory allocation for row is error"); + SWIG_fail; + } + GSType errorType; + retVal = getRowFields(row, arg1->get_column_count(), arg1->get_gstype_list(), + arg1->get_timestamp_to_float(), &errorColumn, &errorType, outList); + if (retVal == false) { + const int SIZE = 60; + char errorMsg[SIZE]; + sprintf(errorMsg, "Can't get data for field %d with type %d", errorColumn, + errorType); + PyErr_SetString(PyExc_ValueError, errorMsg); + SWIG_fail; + } + $result = outList; +} + +// Typemap for Container.put_rows() +%feature("pythonprepend") + griddb::Container::put_rows(GSRow** listRow, int rowCount) %{ + # listRow is input + if isinstance(listRow, pandas.DataFrame) != True: + raise Exception('Input should be DataFrame') + # Convert to numpy ndarray + listRow = listRow.to_numpy() +%} + +%typemap(in, numinputs = 1, fragment ="cleanString", + fragment = "convertToFieldWithType") (GSRow** listRow, int rowCount) { + $1 = NULL; + if (PyArray_API == NULL) { + import_array(); + } + if (!PyArray_Check($input)) { + PyErr_SetString(PyExc_ValueError, "Input should be numpy.ndarray"); + SWIG_fail; + } + PyArrayObject* array = (PyArrayObject*) PyArray_EnsureArray($input); + + npy_intp* dim = PyArray_DIMS(array); + $2 = dim[0]; // number of rows + + if ($2) { + GSType* fieldTypes = arg1->getGSTypeList(); + try { + $1 = new GSRow*[$2](); + } catch (bad_alloc& ba) { + PyErr_SetString(PyExc_ValueError, "Memory allocation error"); + SWIG_fail; + } + int length = dim[1]; + GSType type; + int columnNumber = arg1->getColumnCount(); + if (length != columnNumber) { + PyErr_SetString(PyExc_ValueError, + "num row is different with container info"); + SWIG_fail; + } + + GSResult ret; + GSContainer *mContainer = arg1->getGSContainerPtr(); + + PyArrayIterObject* arrayInter = + (PyArrayIterObject*)PyArray_IterNew((PyObject*)array); + int rowCountVal = 0; // Avoid confuse with input "int rowCount" + int columnCount = 0; + int count = 0; + int alloc = 0; + size_t size = 0; + int res; + PyObject* rowFieldPyObject; + GSChar* stringVal; + + for (int i = 0; i < $2; i++) { + ret = gsCreateRowByContainer(mContainer, &$1[i]); + if (!GS_SUCCEEDED(ret)) { + PyErr_SetString(PyExc_ValueError, "Can't create row"); + SWIG_fail; + } + } + + GSType* typeList = arg1->getGSTypeList(); + bool checkConvert; + while (PyArray_ITER_NOTDONE(arrayInter)) { + // Get data from numpy C array + rowFieldPyObject = PyArray_GETITEM((const PyArrayObject*)array, + (char*)PyArray_ITER_DATA(arrayInter)); + // Convert to C type + checkConvert = convertToFieldWithType($1[rowCountVal], + columnCount, rowFieldPyObject, typeList[columnCount]); + if (!checkConvert) { + const int SIZE = 60; + char errorMsg[SIZE]; + sprintf(errorMsg, "Can't set data for field %d with type %d", + columnCount, typeList[columnCount]); + PyErr_SetString(PyExc_ValueError, errorMsg); + Py_DECREF(arrayInter); + SWIG_fail; + } + + // Update rowCountVal, columnCount and count++ + count++; + if (count % length == 0 && rowCountVal < $2) { + // New row data for GSRow + rowCountVal++; + columnCount = 0; + } else if (columnCount < length && rowCountVal < $2) { + // Still old row + columnCount++; + } + // Get next data + PyArray_ITER_NEXT(arrayInter); + } + + Py_DECREF(arrayInter); + } +} + +%typemap(freearg) (GSRow** listRow, int rowCount) { + if ($1) { + for (int i = 0; i < $2; i++) { + if ($1[i]) { + gsCloseRow(&$1[i]); + } + } + delete [] $1; + } +}