From ded8adb4365add1fd308d4473f24592df0ccc1da Mon Sep 17 00:00:00 2001 From: Dan Fellin Date: Thu, 28 Mar 2024 12:28:44 -0700 Subject: [PATCH 01/36] add new dev version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 543ce52..7096801 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.2.1" +version = "0.2.3.dev0" description = "The PolyAPI Python Client" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ From 9ee4b83ef8d042bc32cf884bbe9394d9f07629c6 Mon Sep 17 00:00:00 2001 From: Dan Fellin Date: Mon, 8 Apr 2024 08:16:06 -0700 Subject: [PATCH 02/36] add error_handler --- polyapi/error_handler.py | 45 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 polyapi/error_handler.py diff --git a/polyapi/error_handler.py b/polyapi/error_handler.py new file mode 100644 index 0000000..7f7a797 --- /dev/null +++ b/polyapi/error_handler.py @@ -0,0 +1,45 @@ +import copy +import socketio # type: ignore +from typing import Any, Callable, Dict, Optional + +from polyapi.config import get_api_key_and_url + + +local_error_handlers: Dict[str, Any] = {} + + +async def on(path: str, callback: Callable, options: Optional[Dict[str, Any]]) -> Callable: + assert not local_error_handlers + socket = socketio.AsyncClient() + api_key, base_url = get_api_key_and_url() + await socket.connect(base_url, transports=["websocket"], namespaces=["/events"]) + + handler_id = None + data = copy.deepcopy(options or {}) + data["path"] = path + data["apiKey"] = api_key + + def registerCallback(id: int): + nonlocal handler_id, socket + handler_id = id + socket.on(f"handleError:{handler_id}", callback, namespace="/events") + + socket.emit("registerErrorHandler", data, registerCallback) + if local_error_handlers.get(path): + local_error_handlers[path].append(callback) + else: + local_error_handlers[path] = [callback] + + async def unregister(): + nonlocal handler_id, socket + if handler_id and socket: + await socket.emit( + "unregisterErrorHandler", + {"id": handler_id, "path": path, "apiKey": api_key}, + namespace="/events", + ) + + if local_error_handlers.get(path): + local_error_handlers[path].remove(callback) + + return unregister \ No newline at end of file From 1dd6e8e7448fb15eda2ced51a309319be3afa846 Mon Sep 17 00:00:00 2001 From: Dan Fellin Date: Mon, 8 Apr 2024 09:23:28 -0700 Subject: [PATCH 03/36] add error_handler --- polyapi/error_handler.py | 65 ++++++++++++++++++++++------------------ pyproject.toml | 2 +- 2 files changed, 37 insertions(+), 30 deletions(-) diff --git a/polyapi/error_handler.py b/polyapi/error_handler.py index 7f7a797..e8869cf 100644 --- a/polyapi/error_handler.py +++ b/polyapi/error_handler.py @@ -1,3 +1,4 @@ +import asyncio import copy import socketio # type: ignore from typing import Any, Callable, Dict, Optional @@ -8,38 +9,44 @@ local_error_handlers: Dict[str, Any] = {} -async def on(path: str, callback: Callable, options: Optional[Dict[str, Any]]) -> Callable: +def on(path: str, callback: Callable, options: Optional[Dict[str, Any]] = None) -> Callable: assert not local_error_handlers socket = socketio.AsyncClient() api_key, base_url = get_api_key_and_url() - await socket.connect(base_url, transports=["websocket"], namespaces=["/events"]) - - handler_id = None - data = copy.deepcopy(options or {}) - data["path"] = path - data["apiKey"] = api_key - - def registerCallback(id: int): - nonlocal handler_id, socket - handler_id = id - socket.on(f"handleError:{handler_id}", callback, namespace="/events") - - socket.emit("registerErrorHandler", data, registerCallback) - if local_error_handlers.get(path): - local_error_handlers[path].append(callback) - else: - local_error_handlers[path] = [callback] - - async def unregister(): - nonlocal handler_id, socket - if handler_id and socket: - await socket.emit( - "unregisterErrorHandler", - {"id": handler_id, "path": path, "apiKey": api_key}, - namespace="/events", - ) + async def _inner(): + await socket.connect(base_url, transports=["websocket"], namespaces=["/events"]) + + handler_id = None + data = copy.deepcopy(options or {}) + data["path"] = path + data["apiKey"] = api_key + + def registerCallback(id: int): + nonlocal handler_id, socket + handler_id = id + socket.on(f"handleError:{handler_id}", callback, namespace="/events") + + await socket.emit("registerErrorHandler", data, "/events", registerCallback) if local_error_handlers.get(path): - local_error_handlers[path].remove(callback) + local_error_handlers[path].append(callback) + else: + local_error_handlers[path] = [callback] + + async def unregister(): + nonlocal handler_id, socket + if handler_id and socket: + await socket.emit( + "unregisterErrorHandler", + {"id": handler_id, "path": path, "apiKey": api_key}, + namespace="/events", + ) + + if local_error_handlers.get(path): + local_error_handlers[path].remove(callback) + + await socket.wait() + + return unregister - return unregister \ No newline at end of file + return asyncio.run(_inner()) \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 7096801..1742bff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.2.3.dev0" +version = "0.2.3.dev1" description = "The PolyAPI Python Client" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ From 6ce28d4e3acd018acef435ffe2da7dc6128a8c34 Mon Sep 17 00:00:00 2001 From: Dan Fellin Date: Tue, 9 Apr 2024 08:04:28 -0700 Subject: [PATCH 04/36] catch if import name is different than pip name --- polyapi/function_cli.py | 10 +++++++++- pyproject.toml | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/polyapi/function_cli.py b/polyapi/function_cli.py index 91aaf0e..395b284 100644 --- a/polyapi/function_cli.py +++ b/polyapi/function_cli.py @@ -8,6 +8,7 @@ import requests from stdlib_list import stdlib_list from pydantic import TypeAdapter +from importlib.metadata import packages_distributions from polyapi.generate import get_functions_and_parse, generate_functions from polyapi.config import get_api_key_and_url from polyapi.constants import PYTHON_TO_JSONSCHEMA_TYPE_MAP @@ -112,6 +113,12 @@ def _parse_code(code: str, function_name: str): schemas = _get_schemas(code) parsed_code = ast.parse(code) + + # the pip name and the import name might be different + # e.g. kube_hunter is the import name, but the pip name is kube-hunter + # see https://stackoverflow.com/a/75144378 + pip_name_lookup = packages_distributions() + for node in ast.iter_child_nodes(parsed_code): if isinstance(node, ast.Import): for name in node.names: @@ -119,7 +126,8 @@ def _parse_code(code: str, function_name: str): requirements.append(name.name) elif isinstance(node, ast.ImportFrom): if node.module and node.module not in BASE_REQUIREMENTS: - requirements.append(node.module) + req = pip_name_lookup[node.module][0] + requirements.append(req) elif isinstance(node, ast.FunctionDef) and node.name == function_name: function_args = [arg for arg in node.args.args] diff --git a/pyproject.toml b/pyproject.toml index 1742bff..3ff7953 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.2.3.dev1" +version = "0.2.3.dev2" description = "The PolyAPI Python Client" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ From def20b592f59e057844ce90dd8c94a28bbd0d8df Mon Sep 17 00:00:00 2001 From: Dan Fellin Date: Tue, 9 Apr 2024 08:44:17 -0700 Subject: [PATCH 05/36] lets hardcode pydantic 2.5.3 and see if that fixes --- pyproject.toml | 4 ++-- requirements.txt | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 3ff7953..6d1d37b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,14 +3,14 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.2.3.dev2" +version = "0.2.3.dev3" description = "The PolyAPI Python Client" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ "requests==2.31.0", "typing_extensions==4.10.0", "jsonschema-gentypes==2.4.0", - "pydantic==2.6.4", + "pydantic==2.5.3", "stdlib_list==0.10.0", "colorama==0.4.4", "python-socketio[asyncio_client]==5.11.1", diff --git a/requirements.txt b/requirements.txt index c5060d1..10c70d3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ requests==2.31.0 typing_extensions==4.10.0 jsonschema-gentypes==2.4.0 -pydantic==2.6.4 +pydantic==2.5.3 stdlib_list==0.10.0 colorama==0.4.4 python-socketio[asyncio_client]==5.11.1 From cdd9c63c31de61b55bb77105358498b22e06d1b8 Mon Sep 17 00:00:00 2001 From: Dan Fellin Date: Tue, 9 Apr 2024 08:46:32 -0700 Subject: [PATCH 06/36] Revert "lets hardcode pydantic 2.5.3 and see if that fixes" This reverts commit def20b592f59e057844ce90dd8c94a28bbd0d8df. --- pyproject.toml | 4 ++-- requirements.txt | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6d1d37b..3ff7953 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,14 +3,14 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.2.3.dev3" +version = "0.2.3.dev2" description = "The PolyAPI Python Client" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ "requests==2.31.0", "typing_extensions==4.10.0", "jsonschema-gentypes==2.4.0", - "pydantic==2.5.3", + "pydantic==2.6.4", "stdlib_list==0.10.0", "colorama==0.4.4", "python-socketio[asyncio_client]==5.11.1", diff --git a/requirements.txt b/requirements.txt index 10c70d3..c5060d1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ requests==2.31.0 typing_extensions==4.10.0 jsonschema-gentypes==2.4.0 -pydantic==2.5.3 +pydantic==2.6.4 stdlib_list==0.10.0 colorama==0.4.4 python-socketio[asyncio_client]==5.11.1 From 395946c037e3d01a42f17b4b1a0f4b93ed0d59b5 Mon Sep 17 00:00:00 2001 From: Dan Fellin Date: Tue, 9 Apr 2024 08:46:59 -0700 Subject: [PATCH 07/36] 0.2.3.dev4, actually its problem with kube_hunter, nvm --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 3ff7953..d607eaf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.2.3.dev2" +version = "0.2.3.dev4" description = "The PolyAPI Python Client" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ From 5626e581a5ebd7f4f3605701cdd1424cafe69082 Mon Sep 17 00:00:00 2001 From: Dan Fellin Date: Tue, 9 Apr 2024 12:28:50 -0700 Subject: [PATCH 08/36] maybe we need the new requirements --- polyapi/function_cli.py | 30 +++++++++++++++++++++++------- pyproject.toml | 2 +- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/polyapi/function_cli.py b/polyapi/function_cli.py index 395b284..48ffa59 100644 --- a/polyapi/function_cli.py +++ b/polyapi/function_cli.py @@ -3,7 +3,7 @@ import json import types import sys -from typing import Dict, List, Tuple +from typing import Dict, List, Mapping, Optional, Tuple from typing_extensions import _TypedDictMeta # type: ignore import requests from stdlib_list import stdlib_list @@ -18,7 +18,7 @@ # these libraries are already installed in the base docker image # and shouldnt be included in additional requirements -BASE_REQUIREMENTS = {"polyapi", "requests", "typing_extensions", "jsonschema-gentypes", "pydantic"} +BASE_REQUIREMENTS = {"polyapi", "requests", "typing_extensions", "jsonschema-gentypes", "pydantic", "cloudevents"} all_stdlib_symbols = stdlib_list('.'.join([str(v) for v in sys.version_info[0:2]])) BASE_REQUIREMENTS.update(all_stdlib_symbols) # dont need to pip install stuff in the python standard library @@ -104,6 +104,19 @@ def _get_type(expr: ast.expr | None, schemas: List[Dict]) -> Tuple[str, Dict | N return json_type, _get_type_schema(json_type, python_type, schemas) +def _get_req_name_if_not_in_base(n: Optional[str], pip_name_lookup: Mapping[str, List[str]]) -> Optional[str]: + if not n: + return None + + if "." in n: + n = n.split(".")[0] + + if n in BASE_REQUIREMENTS: + return None + else: + return pip_name_lookup[n][0] + + def _parse_code(code: str, function_name: str): parsed_args = [] return_type = None @@ -121,13 +134,16 @@ def _parse_code(code: str, function_name: str): for node in ast.iter_child_nodes(parsed_code): if isinstance(node, ast.Import): + # TODO maybe handle `import foo.bar` case? for name in node.names: - if name.name not in BASE_REQUIREMENTS: - requirements.append(name.name) + req = _get_req_name_if_not_in_base(name.name, pip_name_lookup) + if req: + requirements.append(req) elif isinstance(node, ast.ImportFrom): - if node.module and node.module not in BASE_REQUIREMENTS: - req = pip_name_lookup[node.module][0] - requirements.append(req) + if node.module: + req = _get_req_name_if_not_in_base(node.module, pip_name_lookup) + if req: + requirements.append(req) elif isinstance(node, ast.FunctionDef) and node.name == function_name: function_args = [arg for arg in node.args.args] diff --git a/pyproject.toml b/pyproject.toml index d607eaf..4d4a710 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.2.3.dev4" +version = "0.2.3.dev5" description = "The PolyAPI Python Client" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ From bcfe77ffdff03e95f08ecbbcf762c5b11874f522 Mon Sep 17 00:00:00 2001 From: Dan Fellin Date: Thu, 11 Apr 2024 08:25:21 -0700 Subject: [PATCH 09/36] woot client functions are done? --- polyapi/client.py | 25 +++++++++ polyapi/function_cli.py | 14 ++--- polyapi/generate.py | 118 +++++++++++++++++----------------------- polyapi/typedefs.py | 6 +- pyproject.toml | 2 +- 5 files changed, 86 insertions(+), 79 deletions(-) create mode 100644 polyapi/client.py diff --git a/polyapi/client.py b/polyapi/client.py new file mode 100644 index 0000000..c7d84b8 --- /dev/null +++ b/polyapi/client.py @@ -0,0 +1,25 @@ +from typing import Any, Dict, List, Tuple + +from polyapi.typedefs import PropertySpecification +from polyapi.utils import camelCase, add_type_import_path, parse_arguments, get_type_and_def + +DEFS_TEMPLATE = """ +from typing import List, Dict, Any, TypedDict +{args_def} +{return_type_def} +""" + + +def render_client_function( + function_name: str, + code: str, + arguments: List[PropertySpecification], + return_type: Dict[str, Any], +) -> Tuple[str, str]: + args, args_def = parse_arguments(function_name, arguments) + return_type_name, return_type_def = get_type_and_def(return_type) # type: ignore + func_type_defs = DEFS_TEMPLATE.format( + args_def=args_def, + return_type_def=return_type_def, + ) + return code, func_type_defs \ No newline at end of file diff --git a/polyapi/function_cli.py b/polyapi/function_cli.py index 48ffa59..3c01b98 100644 --- a/polyapi/function_cli.py +++ b/polyapi/function_cli.py @@ -184,7 +184,8 @@ def function_add_or_update( args = parser.parse_args(subcommands) verb = "Updating" if _func_already_exists(context, args.function_name) else "Adding" - print(f"{verb} custom server side function...", end="") + ftype = "server" if server else "client" + print(f"{verb} custom {ftype} function...", end="") with open(args.filename, "r") as f: code = f.read() @@ -202,9 +203,6 @@ def function_add_or_update( print(f"Function {args.function_name} not found as top-level function in {args.filename}") sys.exit(1) - if requirements: - print_yellow('\nPlease note that deploying your functions will take a few minutes because it makes use of libraries other than polyapi.') - data = { "context": context, "name": args.function_name, @@ -213,18 +211,20 @@ def function_add_or_update( "language": "python", "returnType": return_type, "returnTypeSchema": return_type_schema, - "requirements": requirements, "arguments": arguments, "logsEnabled": logs_enabled, } + if server and requirements: + print_yellow('\nPlease note that deploying your functions will take a few minutes because it makes use of libraries other than polyapi.') + data["requirements"] = requirements + api_key, api_url = get_api_key_and_url() assert api_key if server: url = f"{api_url}/functions/server" else: - raise NotImplementedError("Client functions not yet implemented.") - # url = f"{base_url}/functions/client" + url = f"{api_url}/functions/client" headers = get_auth_headers(api_key) resp = requests.post(url, headers=headers, json=data) diff --git a/polyapi/generate.py b/polyapi/generate.py index bbc0e86..3466e58 100644 --- a/polyapi/generate.py +++ b/polyapi/generate.py @@ -2,9 +2,10 @@ import requests import os import shutil -from typing import Any, Dict, List, Tuple +from typing import List from polyapi.auth import render_auth_function +from polyapi.client import render_client_function from polyapi.execute import execute_post from polyapi.webhook import render_webhook_handle @@ -18,6 +19,7 @@ SUPPORTED_FUNCTION_TYPES = { "apiFunction", "authFunction", + "customFunction", "serverFunction", "webhookHandle", } @@ -38,32 +40,29 @@ def get_specs() -> List: def parse_function_specs( - specs: List, limit_ids: List[str] | None # optional list of ids to limit to -) -> List[Tuple[str, str, str, str, List[PropertySpecification], Dict[str, Any]]]: + specs: List[SpecificationDto], + limit_ids: List[str] | None, # optional list of ids to limit to +) -> List[SpecificationDto]: functions = [] for spec in specs: + if not spec or "function" not in spec: + continue + + if not spec["function"]: + continue + if limit_ids and spec["id"] not in limit_ids: continue if spec["type"] not in SUPPORTED_FUNCTION_TYPES: continue - function_type = spec["type"] - function_name = f"poly.{spec['context']}.{spec['name']}" - function_id = spec["id"] - arguments: List[PropertySpecification] = [ - arg for arg in spec["function"]["arguments"] - ] - functions.append( - ( - function_type, - function_name, - function_id, - spec["description"], - arguments, - spec["function"]["returnType"], - ) - ) + if spec["type"] == "customFunction" and spec["language"] != "python": + # poly libraries only support client functions of same language + continue + + functions.append(spec) + return functions @@ -90,11 +89,10 @@ def read_cached_specs() -> List[SpecificationDto]: return json.loads(f.read()) -def get_functions_and_parse(limit_ids: List[str] | None = None): +def get_functions_and_parse(limit_ids: List[str] | None = None) -> List[SpecificationDto]: specs = get_specs() cache_specs(specs) - functions = parse_function_specs(specs, limit_ids=limit_ids) - return functions + return parse_function_specs(specs, limit_ids=limit_ids) def get_variables() -> List[VariableSpecDto]: @@ -162,14 +160,7 @@ def save_rendered_specs() -> None: api_specs = [spec for spec in specs if spec["type"] == "apiFunction"] for spec in api_specs: assert spec["function"] - func_str, type_defs = render_spec( - spec["type"], - spec["name"], - spec["id"], - spec["description"], - spec["function"]["arguments"], - spec["function"]["returnType"], - ) + func_str, type_defs = render_spec(spec) data = { "language": "python", "apiFunctionId": spec["id"], @@ -181,14 +172,20 @@ def save_rendered_specs() -> None: assert resp.status_code == 201, (resp.text, resp.status_code) -def render_spec( - function_type: str, - function_name: str, - function_id: str, - function_description: str, - arguments: List[PropertySpecification], - return_type: Dict[str, Any], -): +def render_spec(spec: SpecificationDto): + function_type = spec["type"] + function_description = spec["description"] + function_name = spec["name"] + function_id = spec["id"] + + arguments: List[PropertySpecification] = [] + return_type = {} + if spec["function"]: + arguments = [ + arg for arg in spec["function"]["arguments"] + ] + return_type = spec["function"]["returnType"] + if function_type == "apiFunction": func_str, func_type_defs = render_api_function( function_type, @@ -198,6 +195,13 @@ def render_spec( arguments, return_type, ) + elif function_type == "customFunction": + func_str, func_type_defs = render_client_function( + function_name, + spec["code"], + arguments, + return_type, + ) elif function_type == "serverFunction": func_str, func_type_defs = render_server_function( function_type, @@ -229,25 +233,14 @@ def render_spec( def add_function_file( - function_type: str, full_path: str, function_name: str, - function_id: str, - function_description: str, - arguments: List[PropertySpecification], - return_type: Dict[str, Any], + spec: SpecificationDto, ): # first lets add the import to the __init__ init_the_init(full_path) - func_str, func_type_defs = render_spec( - function_type, - function_name, - function_id, - function_description, - arguments, - return_type, - ) + func_str, func_type_defs = render_spec(spec) if func_str: # add function to init @@ -262,27 +255,17 @@ def add_function_file( def create_function( - function_type: str, - path: str, - function_id: str, - function_description: str, - arguments: List[PropertySpecification], - return_type: Dict[str, Any], + spec: SpecificationDto ) -> None: full_path = os.path.dirname(os.path.abspath(__file__)) - - folders = path.split(".") + folders = f"poly.{spec['context']}.{spec['name']}".split(".") for idx, folder in enumerate(folders): if idx + 1 == len(folders): # special handling for final level add_function_file( - function_type, full_path, folder, - function_id, - function_description, - arguments, - return_type, + spec, ) else: full_path = os.path.join(full_path, folder) @@ -296,9 +279,6 @@ def create_function( add_import_to_init(full_path, next) -# TODO create the socket and pass to create_function? - - -def generate_functions(functions: List) -> None: +def generate_functions(functions: List[SpecificationDto]) -> None: for func in functions: - create_function(*func) + create_function(func) diff --git a/polyapi/typedefs.py b/polyapi/typedefs.py index 92a7d38..544f3c1 100644 --- a/polyapi/typedefs.py +++ b/polyapi/typedefs.py @@ -32,9 +32,11 @@ class SpecificationDto(TypedDict): context: str name: str description: str - # function is none if this is actually VariableSpecDto - function: FunctionSpecification | None + # function is none (or function key not present) if this is actually VariableSpecDto + function: NotRequired[FunctionSpecification | None] type: Literal['apiFunction', 'customFunction', 'serverFunction', 'authFunction', 'webhookHandle', 'serverVariable'] + code: NotRequired[str] + language: str class VariableSpecification(TypedDict): diff --git a/pyproject.toml b/pyproject.toml index 4d4a710..e907371 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.2.3.dev5" +version = "0.2.3.dev6" description = "The PolyAPI Python Client" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ From 6947e87733848e34194c156dcb1f990d5209bcfc Mon Sep 17 00:00:00 2001 From: Dan Fellin Date: Thu, 11 Apr 2024 08:29:15 -0700 Subject: [PATCH 10/36] fix tests --- tests/test_function_cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_function_cli.py b/tests/test_function_cli.py index 7ccdb5c..ac3ccda 100644 --- a/tests/test_function_cli.py +++ b/tests/test_function_cli.py @@ -79,12 +79,12 @@ def test_list_complex_return_type(self): def test_parse_import_basic(self): code = "import flask\n\n\ndef foobar(n: int) -> int:\n return 9\n" _, _, _, additional_requirements = _parse_code(code, "foobar") - self.assertEqual(additional_requirements, ["flask"]) + self.assertEqual(additional_requirements, ["Flask"]) def test_parse_import_from(self): code = "from flask import Request, Response\n\n\ndef foobar(n: int) -> int:\n return 9\n" _, _, _, additional_requirements = _parse_code(code, "foobar") - self.assertEqual(additional_requirements, ["flask"]) + self.assertEqual(additional_requirements, ["Flask"]) def test_parse_import_base(self): code = "import requests\n\n\ndef foobar(n: int) -> int:\n return 9\n" From a6fe1b4bf38ea59814aec1a95366eb475a908ed5 Mon Sep 17 00:00:00 2001 From: Dan Fellin Date: Fri, 12 Apr 2024 07:57:32 -0700 Subject: [PATCH 11/36] 0.2.3.dev7, dont assume client function, must specify with --client --- polyapi/cli.py | 5 +++-- polyapi/function_cli.py | 8 ++++++-- pyproject.toml | 2 +- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/polyapi/cli.py b/polyapi/cli.py index a55750c..14f21a2 100644 --- a/polyapi/cli.py +++ b/polyapi/cli.py @@ -23,7 +23,8 @@ def execute_from_cli(): ) parser.add_argument("--context", required=False, default="") parser.add_argument("--description", required=False, default="") - parser.add_argument("--server", action="store_true", help="Pass --server when adding function to add a server function. By default, new functions are client.") + parser.add_argument("--client", action="store_true", help="Pass --client when adding function to add a client function.") + parser.add_argument("--server", action="store_true", help="Pass --server when adding function to add a server function.") parser.add_argument("--logs", action="store_true", help="Pass --logs when adding function if you want to store and see the function logs.") parser.add_argument("command", choices=CLI_COMMANDS) parser.add_argument("subcommands", nargs="*") @@ -47,4 +48,4 @@ def execute_from_cli(): print("Clearing the generated library...") clear() elif command == "function": - function_add_or_update(args.context, args.description, args.server, args.logs, args.subcommands) + function_add_or_update(args.context, args.description, args.client, args.server, args.logs, args.subcommands) diff --git a/polyapi/function_cli.py b/polyapi/function_cli.py index 3c01b98..a050ff6 100644 --- a/polyapi/function_cli.py +++ b/polyapi/function_cli.py @@ -175,7 +175,7 @@ def _func_already_exists(context: str, function_name: str) -> bool: def function_add_or_update( - context: str, description: str, server: bool, logs_enabled: bool, subcommands: List + context: str, description: str, client: bool, server: bool, logs_enabled: bool, subcommands: List ): parser = argparse.ArgumentParser() parser.add_argument("subcommand", choices=["add"]) @@ -223,8 +223,12 @@ def function_add_or_update( assert api_key if server: url = f"{api_url}/functions/server" - else: + elif client: url = f"{api_url}/functions/client" + else: + print_red("ERROR") + print("Please specify type of function with --client or --server") + sys.exit(1) headers = get_auth_headers(api_key) resp = requests.post(url, headers=headers, json=data) diff --git a/pyproject.toml b/pyproject.toml index e907371..65c3f48 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.2.3.dev6" +version = "0.2.3.dev7" description = "The PolyAPI Python Client" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ From 6ffd13d675bab4ef5c8e389c20f9b5440af2683e Mon Sep 17 00:00:00 2001 From: Dan Fellin Date: Mon, 15 Apr 2024 07:48:23 -0700 Subject: [PATCH 12/36] 0.2.3.dev8 fix small bug client functions --- polyapi/client.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/polyapi/client.py b/polyapi/client.py index c7d84b8..0427e43 100644 --- a/polyapi/client.py +++ b/polyapi/client.py @@ -22,4 +22,4 @@ def render_client_function( args_def=args_def, return_type_def=return_type_def, ) - return code, func_type_defs \ No newline at end of file + return code + "\n\n", func_type_defs \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 65c3f48..916aa50 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.2.3.dev7" +version = "0.2.3.dev8" description = "The PolyAPI Python Client" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ From f10ac615499f600d635d427c4e7514c69c2d1bf3 Mon Sep 17 00:00:00 2001 From: Dan Fellin Date: Mon, 15 Apr 2024 11:36:39 -0700 Subject: [PATCH 13/36] 0.2.3.dev9, add fallbacks if unrecognized version of OpenAPI present in API functions --- polyapi/utils.py | 20 +++++++++++++++++--- pyproject.toml | 2 +- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/polyapi/utils.py b/polyapi/utils.py index 6ed6787..91c4b0a 100644 --- a/polyapi/utils.py +++ b/polyapi/utils.py @@ -1,5 +1,6 @@ import re import os +import logging from typing import Tuple, List from colorama import Fore, Style from polyapi.constants import BASIC_PYTHON_TYPES @@ -91,7 +92,11 @@ def get_type_and_def(type_spec: PropertyType) -> Tuple[str, str]: if type_spec.get("items"): items = type_spec["items"] if items.get("$ref"): - return "ResponseType", generate_schema_types(type_spec, root="ResponseType") # type: ignore + try: + return "ResponseType", generate_schema_types(type_spec, root="ResponseType") # type: ignore + except: + logging.exception(f"Error when generating schema type: {type_spec}") + return "Dict", "" else: item_type, _ = get_type_and_def(items) title = f"List[{item_type}]" @@ -108,7 +113,12 @@ def get_type_and_def(type_spec: PropertyType) -> Tuple[str, str]: if title: assert isinstance(title, str) title = clean_title(title) - return title, generate_schema_types(schema, root=title) # type: ignore + try: + return title, generate_schema_types(schema, root=title) # type: ignore + except: + logging.exception(f"Error when generating schema type: {schema}") + return "Dict", "" + elif schema.get("items"): # fallback to schema $ref name if no explicit title items = schema.get("items") # type: ignore @@ -123,7 +133,11 @@ def get_type_and_def(type_spec: PropertyType) -> Tuple[str, str]: return "List", "" title = f"List[{title}]" - return title, generate_schema_types(schema, root=title) + try: + return title, generate_schema_types(schema, root=title) + except: + logging.exception(f"Error when generating schema type: {schema}") + return "List", "" else: return "Any", "" else: diff --git a/pyproject.toml b/pyproject.toml index 916aa50..6a0a331 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.2.3.dev8" +version = "0.2.3.dev9" description = "The PolyAPI Python Client" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ From e065e0db736d52b9384acb82f22b32ea39c0727c Mon Sep 17 00:00:00 2001 From: Dan Fellin Date: Mon, 15 Apr 2024 14:35:55 -0700 Subject: [PATCH 14/36] add pyjwt to requirements --- pyproject.toml | 3 ++- requirements.txt | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6a0a331..58b9c84 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.2.3.dev9" +version = "0.2.4.dev0" description = "The PolyAPI Python Client" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ @@ -15,6 +15,7 @@ dependencies = [ "colorama==0.4.4", "python-socketio[asyncio_client]==5.11.1", "truststore==0.8.0", + "pyjwt==2.3.0", ] readme = "README.md" license = { file = "LICENSE" } diff --git a/requirements.txt b/requirements.txt index c5060d1..d2e9af2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,4 +5,5 @@ pydantic==2.6.4 stdlib_list==0.10.0 colorama==0.4.4 python-socketio[asyncio_client]==5.11.1 -truststore==0.8.0 \ No newline at end of file +truststore==0.8.0 +pyjwt==2.3.0 \ No newline at end of file From 0d634757f2c021cd6338d4290d9446ad5e5d45a8 Mon Sep 17 00:00:00 2001 From: Dan Fellin Date: Wed, 17 Apr 2024 12:05:47 -0700 Subject: [PATCH 15/36] add ability to do multiple error handlers! --- polyapi/error_handler.py | 94 +++++++++++++++++++++++++--------------- pyproject.toml | 2 +- 2 files changed, 61 insertions(+), 35 deletions(-) diff --git a/polyapi/error_handler.py b/polyapi/error_handler.py index e8869cf..e353fcd 100644 --- a/polyapi/error_handler.py +++ b/polyapi/error_handler.py @@ -1,52 +1,78 @@ import asyncio import copy import socketio # type: ignore -from typing import Any, Callable, Dict, Optional +from typing import Any, Callable, Dict, List, Optional from polyapi.config import get_api_key_and_url -local_error_handlers: Dict[str, Any] = {} +active_handlers: List[Dict[str, Any]] = [] +client = None -def on(path: str, callback: Callable, options: Optional[Dict[str, Any]] = None) -> Callable: - assert not local_error_handlers - socket = socketio.AsyncClient() - api_key, base_url = get_api_key_and_url() +def prepare(): + loop = asyncio.get_event_loop() + loop.run_until_complete(get_client_and_connect()) + print("Client initialized!") - async def _inner(): - await socket.connect(base_url, transports=["websocket"], namespaces=["/events"]) - handler_id = None - data = copy.deepcopy(options or {}) - data["path"] = path - data["apiKey"] = api_key - def registerCallback(id: int): - nonlocal handler_id, socket - handler_id = id - socket.on(f"handleError:{handler_id}", callback, namespace="/events") +async def get_client_and_connect(): + _, base_url = get_api_key_and_url() + global client + client = socketio.AsyncClient() + await client.connect(base_url, transports=["websocket"], namespaces=["/events"]) - await socket.emit("registerErrorHandler", data, "/events", registerCallback) - if local_error_handlers.get(path): - local_error_handlers[path].append(callback) - else: - local_error_handlers[path] = [callback] - async def unregister(): - nonlocal handler_id, socket - if handler_id and socket: - await socket.emit( - "unregisterErrorHandler", - {"id": handler_id, "path": path, "apiKey": api_key}, - namespace="/events", - ) +async def unregister(data: Dict[str, Any]): + print(f"stopping error handler for '{data['path']}'...") + assert client + await client.emit( + "unregisterErrorHandler", + data, + "/events", + ) - if local_error_handlers.get(path): - local_error_handlers[path].remove(callback) - await socket.wait() +async def unregister_all(): + _, base_url = get_api_key_and_url() + # need to reconnect because maybe socketio client disconnected after Ctrl+C? + await client.connect(base_url, transports=["websocket"], namespaces=["/events"]) + await asyncio.gather(*[unregister(handler) for handler in active_handlers]) - return unregister - return asyncio.run(_inner()) \ No newline at end of file +async def on( + path: str, callback: Callable, options: Optional[Dict[str, Any]] = None +) -> None: + print(f"starting error handler for {path}...") + + if not client: + raise Exception("Client not initialized. Please call error_handler.prepare() first.") + + api_key, _ = get_api_key_and_url() + handler_id = None + data = copy.deepcopy(options or {}) + data["path"] = path + data["apiKey"] = api_key + + def registerCallback(id: int): + nonlocal handler_id + handler_id = id + client.on(f"handleError:{handler_id}", callback, namespace="/events") + active_handlers.append({"path": path, "id": handler_id, "apiKey": api_key}) + + await client.emit("registerErrorHandler", data, "/events", registerCallback) + + +def start(*args): + loop = asyncio.get_event_loop() + loop.run_until_complete(get_client_and_connect()) + asyncio.gather(*args) + + try: + loop.run_forever() + except KeyboardInterrupt: + pass + finally: + loop.run_until_complete(unregister_all()) + loop.stop() \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 58b9c84..d6d225e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.2.4.dev0" +version = "0.2.4.dev1" description = "The PolyAPI Python Client" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ From 3a24f7dbcf4d6386cae218e511ffb6c2459b85bb Mon Sep 17 00:00:00 2001 From: Dan Fellin Date: Wed, 17 Apr 2024 12:10:05 -0700 Subject: [PATCH 16/36] dont require pyjwt anymore --- pyproject.toml | 1 - requirements.txt | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d6d225e..2e54431 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,6 @@ dependencies = [ "colorama==0.4.4", "python-socketio[asyncio_client]==5.11.1", "truststore==0.8.0", - "pyjwt==2.3.0", ] readme = "README.md" license = { file = "LICENSE" } diff --git a/requirements.txt b/requirements.txt index d2e9af2..c5060d1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,5 +5,4 @@ pydantic==2.6.4 stdlib_list==0.10.0 colorama==0.4.4 python-socketio[asyncio_client]==5.11.1 -truststore==0.8.0 -pyjwt==2.3.0 \ No newline at end of file +truststore==0.8.0 \ No newline at end of file From c9120ef1b200cb31b1809435236a1418fd9929fb Mon Sep 17 00:00:00 2001 From: Dan Fellin Date: Thu, 18 Apr 2024 12:48:19 -0700 Subject: [PATCH 17/36] 0.2.4.dev2, replace hacky manual mimetypes with real mimetypes --- polyapi/server.py | 13 ++++--------- pyproject.toml | 2 +- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/polyapi/server.py b/polyapi/server.py index 682bb0e..6a359bf 100644 --- a/polyapi/server.py +++ b/polyapi/server.py @@ -18,7 +18,10 @@ def {function_name}( Function ID: {function_id} \""" resp = execute("{function_type}", "{function_id}", {data}) - return {return_action} + try: + return {return_action} + except: + return resp.text """ @@ -54,14 +57,6 @@ def render_server_function( def _get_server_return_action(return_type_name: str) -> str: if return_type_name == "str": return_action = "resp.text" - elif return_type_name == "Any": - return_action = "resp.text" - elif return_type_name == "int": - return_action = "int(resp.text.replace('(int) ', ''))" - elif return_type_name == "float": - return_action = "float(resp.text.replace('(float) ', ''))" - elif return_type_name == "bool": - return_action = "False if resp.text == 'False' else True" else: return_action = "resp.json()" return return_action \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 2e54431..3851eff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.2.4.dev1" +version = "0.2.4.dev2" description = "The PolyAPI Python Client" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ From 6d5a11e6f0f0358c50f7500818ae4580380e87f7 Mon Sep 17 00:00:00 2001 From: Dan Fellin Date: Fri, 19 Apr 2024 10:56:35 -0700 Subject: [PATCH 18/36] add polycustom --- polyapi/poly_custom.py | 6 ++++++ pyproject.toml | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 polyapi/poly_custom.py diff --git a/polyapi/poly_custom.py b/polyapi/poly_custom.py new file mode 100644 index 0000000..7eb6eaf --- /dev/null +++ b/polyapi/poly_custom.py @@ -0,0 +1,6 @@ +polyCustom = { + "executionId": None, + "executionApiKey": None, + "responseStatusCode": 200, + "responseContentType": None, +} \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 3851eff..49a7519 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.2.4.dev2" +version = "0.2.4.dev3" description = "The PolyAPI Python Client" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ From 44e63d61344d5d4f1e9da945e8a88f858a3c0389 Mon Sep 17 00:00:00 2001 From: Dan Fellin Date: Mon, 22 Apr 2024 08:24:31 -0700 Subject: [PATCH 19/36] multi webhook (#9) * towards multiple webhook handlers simultaneously * next --- polyapi/error_handler.py | 5 +- polyapi/webhook.py | 141 ++++++++++++++++++++++++--------------- 2 files changed, 93 insertions(+), 53 deletions(-) diff --git a/polyapi/error_handler.py b/polyapi/error_handler.py index e353fcd..ad2a451 100644 --- a/polyapi/error_handler.py +++ b/polyapi/error_handler.py @@ -6,7 +6,10 @@ from polyapi.config import get_api_key_and_url +# all active webhook handlers, used by unregister_all to cleanup active_handlers: List[Dict[str, Any]] = [] + +# global client shared by all error handlers, will be initialized by webhook.start client = None @@ -47,7 +50,7 @@ async def on( print(f"starting error handler for {path}...") if not client: - raise Exception("Client not initialized. Please call error_handler.prepare() first.") + raise Exception("Client not initialized. Abort!") api_key, _ = get_api_key_and_url() handler_id = None diff --git a/polyapi/webhook.py b/polyapi/webhook.py index 40a215e..3d74df5 100644 --- a/polyapi/webhook.py +++ b/polyapi/webhook.py @@ -1,77 +1,100 @@ +import asyncio +import socketio # type: ignore import uuid from typing import Any, Dict, List, Tuple +from polyapi.config import get_api_key_and_url from polyapi.typedefs import PropertySpecification +# all active webhook handlers, used by unregister_all to cleanup +active_handlers: List[Dict[str, Any]] = [] + +# global client shared by all webhooks, will be initialized by webhook.start +client = None + + WEBHOOK_TEMPLATE = """ -import asyncio -def {function_name}(callback, options=None): +async def {function_name}(callback, options=None): \"""{description} Function ID: {function_id} \""" + from polyapi.webhook import client, active_handlers + + print("Starting webhook for {function_name}...") + + if not client: + raise Exception("Client not initialized. Abort!") + options = options or {{}} eventsClientId = "{client_id}" function_id = "{function_id}" api_key, base_url = get_api_key_and_url() - async def _inner(): - socket = socketio.AsyncClient() - await socket.connect(base_url, transports=['websocket'], namespaces=['/events']) - - def registerCallback(registered: bool): - nonlocal socket - if registered: - socket.on('handleWebhookEvent:{function_id}', handleEvent, namespace="/events") - else: - print("Could not set register webhook event handler for {function_id}") - - async def handleEvent(data): - nonlocal api_key - nonlocal options - polyCustom = {{}} - resp = await callback(data.get("body"), data.get("headers"), data.get("params"), polyCustom) - if options.get("waitForResponse"): - await socket.emit('setWebhookListenerResponse', {{ - "webhookHandleID": function_id, - "apiKey": api_key, - "clientID": eventsClientId, - "executionId": data.get("executionId"), - "response": {{ - "data": resp, - "statusCode": polyCustom.get("responseStatusCode", 200), - "contentType": polyCustom.get("responseContentType", None), - }}, - }}, namespace="/events") - - data = {{ - "clientID": eventsClientId, - "webhookHandleID": function_id, - "apiKey": api_key, - "waitForResponse": options.get("waitForResponse"), - }} - await socket.emit('registerWebhookEventHandler', data, namespace="/events", callback=registerCallback) - - async def closeEventHandler(): - nonlocal socket - if not socket: - return - - await socket.emit('unregisterWebhookEventHandler', {{ - "clientID": eventsClientId, + def registerCallback(registered: bool): + if registered: + client.on('handleWebhookEvent:{function_id}', handleEvent, namespace="/events") + else: + print("Could not set register webhook event handler for {function_id}") + + async def handleEvent(data): + nonlocal api_key + nonlocal options + polyCustom = {{}} + resp = callback(data.get("body"), data.get("headers"), data.get("params"), polyCustom) + if options.get("waitForResponse"): + await client.emit('setWebhookListenerResponse', {{ "webhookHandleID": function_id, - "apiKey": api_key + "apiKey": api_key, + "clientID": eventsClientId, + "executionId": data.get("executionId"), + "response": {{ + "data": resp, + "statusCode": polyCustom.get("responseStatusCode", 200), + "contentType": polyCustom.get("responseContentType", None), + }}, }}, namespace="/events") - await socket.wait() + data = {{ + "clientID": eventsClientId, + "webhookHandleID": function_id, + "apiKey": api_key, + "waitForResponse": options.get("waitForResponse"), + }} + await client.emit('registerWebhookEventHandler', data, namespace="/events", callback=registerCallback) + active_handlers.append({{"clientID": eventsClientId, "webhookHandleID": function_id, "apiKey": api_key}}) +""" - return closeEventHandler - return asyncio.run(_inner()) -""" +async def get_client_and_connect(): + _, base_url = get_api_key_and_url() + global client + client = socketio.AsyncClient() + await client.connect(base_url, transports=["websocket"], namespaces=["/events"]) + + +async def unregister(data: Dict[str, Any]): + print(f"stopping error handler for '{data['webhookHandleID']}'...") + assert client + await client.emit( + "unregisterWebhookEventHandler", + { + "clientID": data["clientID"], + "webhookHandleID": data["webhookHandleID"], + "apiKey": data["apiKey"], + }, + "/events", + ) + + +async def unregister_all(): + _, base_url = get_api_key_and_url() + # need to reconnect because maybe socketio client disconnected after Ctrl+C? + await client.connect(base_url, transports=["websocket"], namespaces=["/events"]) + await asyncio.gather(*[unregister(handler) for handler in active_handlers]) def render_webhook_handle( @@ -89,4 +112,18 @@ def render_webhook_handle( function_name=function_name, ) - return func_str, "" \ No newline at end of file + return func_str, "" + + +def start(*args): + loop = asyncio.get_event_loop() + loop.run_until_complete(get_client_and_connect()) + asyncio.gather(*args) + + try: + loop.run_forever() + except KeyboardInterrupt: + pass + finally: + loop.run_until_complete(unregister_all()) + loop.stop() From 76e4e8280e616895dfa1ea61b4c109a1c90cb8d7 Mon Sep 17 00:00:00 2001 From: Dan Fellin Date: Mon, 22 Apr 2024 08:32:11 -0700 Subject: [PATCH 20/36] update pyproject.toml --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 49a7519..767103e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,8 +3,8 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.2.4.dev3" -description = "The PolyAPI Python Client" +version = "0.2.4.dev4" +description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ "requests==2.31.0", From d9c79d6d45dbe92ede65a409a574fdd8d225c510 Mon Sep 17 00:00:00 2001 From: Dan Fellin Date: Mon, 22 Apr 2024 09:45:22 -0700 Subject: [PATCH 21/36] 0.2.4.dev5, accept non-200 and non-201 status codes from polyCustom --- polyapi/execute.py | 2 +- polyapi/poly_custom.py | 5 ++++- pyproject.toml | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/polyapi/execute.py b/polyapi/execute.py index 0cce6a1..1e0e46f 100644 --- a/polyapi/execute.py +++ b/polyapi/execute.py @@ -11,7 +11,7 @@ def execute(function_type, function_id, data) -> Response: headers = {"Authorization": f"Bearer {api_key}"} url = f"{api_url}/functions/{function_type}/{function_id}/execute" resp = requests.post(url, json=data, headers=headers) - if resp.status_code != 200 and resp.status_code != 201: + if resp.status_code < 200 or resp.status_code >= 400: error_content = resp.content.decode("utf-8", errors="ignore") raise PolyApiException(f"{resp.status_code}: {error_content}") return resp diff --git a/polyapi/poly_custom.py b/polyapi/poly_custom.py index 7eb6eaf..619c344 100644 --- a/polyapi/poly_custom.py +++ b/polyapi/poly_custom.py @@ -1,4 +1,7 @@ -polyCustom = { +from typing import Dict, Any + + +polyCustom: Dict[str, Any] = { "executionId": None, "executionApiKey": None, "responseStatusCode": 200, diff --git a/pyproject.toml b/pyproject.toml index 767103e..8a7d809 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.2.4.dev4" +version = "0.2.4.dev5" description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ From c4f0e5bd071aeaece598c6670012a68249aff08e Mon Sep 17 00:00:00 2001 From: Dan Fellin Date: Mon, 22 Apr 2024 09:56:57 -0700 Subject: [PATCH 22/36] 0.2.4.dev6 bring python client in line with execute server and consider 3xx to be error --- polyapi/execute.py | 4 +++- pyproject.toml | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/polyapi/execute.py b/polyapi/execute.py index 1e0e46f..abd1cee 100644 --- a/polyapi/execute.py +++ b/polyapi/execute.py @@ -11,7 +11,9 @@ def execute(function_type, function_id, data) -> Response: headers = {"Authorization": f"Bearer {api_key}"} url = f"{api_url}/functions/{function_type}/{function_id}/execute" resp = requests.post(url, json=data, headers=headers) - if resp.status_code < 200 or resp.status_code >= 400: + # print(resp.status_code) + # print(resp.headers["content-type"]) + if resp.status_code < 200 or resp.status_code >= 300: error_content = resp.content.decode("utf-8", errors="ignore") raise PolyApiException(f"{resp.status_code}: {error_content}") return resp diff --git a/pyproject.toml b/pyproject.toml index 8a7d809..d105cb2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.2.4.dev5" +version = "0.2.4.dev6" description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ From 3eaab8a36bfc540ad3281c33b14999d690520de1 Mon Sep 17 00:00:00 2001 From: Dan Fellin Date: Mon, 22 Apr 2024 12:33:36 -0700 Subject: [PATCH 23/36] 0.2.4.dev7, make TypedDict input and response types more picky if wrong TypedDict is used --- polyapi/function_cli.py | 60 +++++++++++++++++++++++++++++++---------- pyproject.toml | 2 +- 2 files changed, 47 insertions(+), 15 deletions(-) diff --git a/polyapi/function_cli.py b/polyapi/function_cli.py index a050ff6..0cbbb7a 100644 --- a/polyapi/function_cli.py +++ b/polyapi/function_cli.py @@ -4,6 +4,7 @@ import types import sys from typing import Dict, List, Mapping, Optional, Tuple +from typing import _TypedDictMeta as BaseTypedDict # type: ignore from typing_extensions import _TypedDictMeta # type: ignore import requests from stdlib_list import stdlib_list @@ -18,9 +19,18 @@ # these libraries are already installed in the base docker image # and shouldnt be included in additional requirements -BASE_REQUIREMENTS = {"polyapi", "requests", "typing_extensions", "jsonschema-gentypes", "pydantic", "cloudevents"} -all_stdlib_symbols = stdlib_list('.'.join([str(v) for v in sys.version_info[0:2]])) -BASE_REQUIREMENTS.update(all_stdlib_symbols) # dont need to pip install stuff in the python standard library +BASE_REQUIREMENTS = { + "polyapi", + "requests", + "typing_extensions", + "jsonschema-gentypes", + "pydantic", + "cloudevents", +} +all_stdlib_symbols = stdlib_list(".".join([str(v) for v in sys.version_info[0:2]])) +BASE_REQUIREMENTS.update( + all_stdlib_symbols +) # dont need to pip install stuff in the python standard library def _get_schemas(code: str) -> List[Dict]: @@ -28,7 +38,14 @@ def _get_schemas(code: str) -> List[Dict]: user_code = types.SimpleNamespace() exec(code, user_code.__dict__) for name, obj in user_code.__dict__.items(): - if ( + if isinstance(obj, BaseTypedDict): + print_red("ERROR") + print_red("\nERROR DETAILS: ") + print( + "It looks like you have used TypedDict in a custom function. Please use `from typing_extensions import TypedDict` instead. The `typing_extensions` version is more powerful and better allows us to provide rich types for your function." + ) + sys.exit(1) + elif ( isinstance(obj, type) and isinstance(obj, _TypedDictMeta) and name != "TypedDict" @@ -76,6 +93,13 @@ def get_python_type_from_ast(expr: ast.expr) -> str: if name == "List": slice = getattr(expr.slice, "id", "Any") return f"List[{slice}]" + elif name == "Dict": + if expr.slice and isinstance(expr.slice, ast.Tuple): + key = get_python_type_from_ast(expr.slice.dims[0]) + value = get_python_type_from_ast(expr.slice.dims[1]) + return f"Dict[{key}, {value}]" + else: + return "Dict" return "Any" else: return "Any" @@ -104,7 +128,9 @@ def _get_type(expr: ast.expr | None, schemas: List[Dict]) -> Tuple[str, Dict | N return json_type, _get_type_schema(json_type, python_type, schemas) -def _get_req_name_if_not_in_base(n: Optional[str], pip_name_lookup: Mapping[str, List[str]]) -> Optional[str]: +def _get_req_name_if_not_in_base( + n: Optional[str], pip_name_lookup: Mapping[str, List[str]] +) -> Optional[str]: if not n: return None @@ -175,7 +201,12 @@ def _func_already_exists(context: str, function_name: str) -> bool: def function_add_or_update( - context: str, description: str, client: bool, server: bool, logs_enabled: bool, subcommands: List + context: str, + description: str, + client: bool, + server: bool, + logs_enabled: bool, + subcommands: List, ): parser = argparse.ArgumentParser() parser.add_argument("subcommand", choices=["add"]) @@ -191,16 +222,15 @@ def function_add_or_update( code = f.read() # OK! let's parse the code and generate the arguments - ( - arguments, - return_type, - return_type_schema, - requirements - ) = _parse_code(code, args.function_name) + (arguments, return_type, return_type_schema, requirements) = _parse_code( + code, args.function_name + ) if not return_type: print_red("ERROR") - print(f"Function {args.function_name} not found as top-level function in {args.filename}") + print( + f"Function {args.function_name} not found as top-level function in {args.filename}" + ) sys.exit(1) data = { @@ -216,7 +246,9 @@ def function_add_or_update( } if server and requirements: - print_yellow('\nPlease note that deploying your functions will take a few minutes because it makes use of libraries other than polyapi.') + print_yellow( + "\nPlease note that deploying your functions will take a few minutes because it makes use of libraries other than polyapi." + ) data["requirements"] = requirements api_key, api_url = get_api_key_and_url() diff --git a/pyproject.toml b/pyproject.toml index d105cb2..b13dca4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.2.4.dev6" +version = "0.2.4.dev7" description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ From fbf9171440c6ab59ab66b4c9cf6eab0d23861a1e Mon Sep 17 00:00:00 2001 From: Dan Fellin Date: Tue, 23 Apr 2024 09:13:39 -0700 Subject: [PATCH 24/36] actually use args in webhook --- polyapi/generate.py | 6 +++++- polyapi/webhook.py | 17 ++++++++++++++--- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/polyapi/generate.py b/polyapi/generate.py index 3466e58..c6197dc 100644 --- a/polyapi/generate.py +++ b/polyapi/generate.py @@ -124,7 +124,9 @@ def generate() -> None: remove_old_library() - functions = get_functions_and_parse() + limit_ids = ['e51d81cc-ffbf-4a04-8a5c-2f54866ad322'] + + functions = get_functions_and_parse(limit_ids) if functions: generate_functions(functions) else: @@ -132,6 +134,8 @@ def generate() -> None: "No functions exist yet in this tenant! Empty library initialized. Let's add some functions!" ) exit() + print("MINIMAL DEBUG GENERATE COMPLETE") + return variables = get_variables() if variables: diff --git a/polyapi/webhook.py b/polyapi/webhook.py index 3d74df5..dffe140 100644 --- a/polyapi/webhook.py +++ b/polyapi/webhook.py @@ -5,6 +5,7 @@ from polyapi.config import get_api_key_and_url from polyapi.typedefs import PropertySpecification +from polyapi.utils import parse_arguments # all active webhook handlers, used by unregister_all to cleanup active_handlers: List[Dict[str, Any]] = [] @@ -13,10 +14,18 @@ client = None +WEBHOOK_DEFS_TEMPLATE = """ +from typing import List, Dict, Any, TypedDict +{function_args_def} +""" + + WEBHOOK_TEMPLATE = """ -async def {function_name}(callback, options=None): +async def {function_name}( +{function_args} +): \"""{description} Function ID: {function_id} @@ -105,14 +114,16 @@ def render_webhook_handle( arguments: List[PropertySpecification], return_type: Dict[str, Any], ) -> Tuple[str, str]: + function_args, function_args_def = parse_arguments(function_name, arguments) func_str = WEBHOOK_TEMPLATE.format( description=function_description, client_id=uuid.uuid4().hex, function_id=function_id, function_name=function_name, + function_args=function_args ) - - return func_str, "" + func_defs = WEBHOOK_DEFS_TEMPLATE.format(function_args_def=function_args_def) + return func_str, func_defs def start(*args): From 60cad53e98edcefe68d467e4c9536dcb348212db Mon Sep 17 00:00:00 2001 From: Dan Fellin Date: Tue, 23 Apr 2024 09:40:56 -0700 Subject: [PATCH 25/36] 0.2.4.dev8, fix webhook stopping text --- polyapi/webhook.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/polyapi/webhook.py b/polyapi/webhook.py index 3d74df5..b03dfda 100644 --- a/polyapi/webhook.py +++ b/polyapi/webhook.py @@ -77,7 +77,7 @@ async def get_client_and_connect(): async def unregister(data: Dict[str, Any]): - print(f"stopping error handler for '{data['webhookHandleID']}'...") + print(f"stopping webhook handler for '{data['webhookHandleID']}'...") assert client await client.emit( "unregisterWebhookEventHandler", diff --git a/pyproject.toml b/pyproject.toml index b13dca4..df0b3af 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.2.4.dev7" +version = "0.2.4.dev8" description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ From 39cc2c0028f9f5e4766f2e9d28c6f41c4072cead Mon Sep 17 00:00:00 2001 From: Dan Fellin Date: Tue, 23 Apr 2024 12:37:00 -0700 Subject: [PATCH 26/36] getting there --- polyapi/constants.py | 4 ++++ polyapi/generate.py | 6 +----- polyapi/server.py | 2 +- polyapi/typedefs.py | 1 + polyapi/utils.py | 25 ++++++++++++++++++++++++- polyapi/webhook.py | 7 ++++++- tests/test_utils.py | 7 +++++++ 7 files changed, 44 insertions(+), 8 deletions(-) diff --git a/polyapi/constants.py b/polyapi/constants.py index a58b64d..16bbeaf 100644 --- a/polyapi/constants.py +++ b/polyapi/constants.py @@ -5,6 +5,8 @@ "boolean": "bool", "array": "List", "object": "Dict", + "function": "Callable", + "void": "None", } @@ -15,6 +17,8 @@ "bool": "boolean", "List": "array", "Dict": "object", + "Callable": "function", + "None": "void", } BASIC_PYTHON_TYPES = set(PYTHON_TO_JSONSCHEMA_TYPE_MAP.keys()) diff --git a/polyapi/generate.py b/polyapi/generate.py index c6197dc..3466e58 100644 --- a/polyapi/generate.py +++ b/polyapi/generate.py @@ -124,9 +124,7 @@ def generate() -> None: remove_old_library() - limit_ids = ['e51d81cc-ffbf-4a04-8a5c-2f54866ad322'] - - functions = get_functions_and_parse(limit_ids) + functions = get_functions_and_parse() if functions: generate_functions(functions) else: @@ -134,8 +132,6 @@ def generate() -> None: "No functions exist yet in this tenant! Empty library initialized. Let's add some functions!" ) exit() - print("MINIMAL DEBUG GENERATE COMPLETE") - return variables = get_variables() if variables: diff --git a/polyapi/server.py b/polyapi/server.py index 6a359bf..b42094a 100644 --- a/polyapi/server.py +++ b/polyapi/server.py @@ -4,7 +4,7 @@ from polyapi.utils import camelCase, add_type_import_path, parse_arguments, get_type_and_def SERVER_DEFS_TEMPLATE = """ -from typing import List, Dict, Any, TypedDict +from typing import List, Dict, Any, TypedDict, Callable {args_def} {return_type_def} """ diff --git a/polyapi/typedefs.py b/polyapi/typedefs.py index 544f3c1..e23113a 100644 --- a/polyapi/typedefs.py +++ b/polyapi/typedefs.py @@ -12,6 +12,7 @@ class PropertySpecification(TypedDict): class PropertyType(TypedDict): kind: Literal['void', 'primitive', 'array', 'object', 'function', 'plain'] + spec: NotRequired[Dict] name: NotRequired[str] type: NotRequired[str] items: NotRequired['PropertyType'] diff --git a/polyapi/utils.py b/polyapi/utils.py index 91c4b0a..67e7e39 100644 --- a/polyapi/utils.py +++ b/polyapi/utils.py @@ -10,7 +10,7 @@ # this string should be in every __init__ file. # it contains all the imports needed for the function or variable code to run -CODE_IMPORTS = "from typing import List, Dict, Any, TypedDict, Optional\nimport requests\nimport socketio # type: ignore\nfrom polyapi.config import get_api_key_and_url\nfrom polyapi.execute import execute, execute_post, variable_get, variable_update\n\n" +CODE_IMPORTS = "from typing import List, Dict, Any, TypedDict, Optional, Callable\nimport requests\nimport socketio # type: ignore\nfrom polyapi.config import get_api_key_and_url\nfrom polyapi.execute import execute, execute_post, variable_get, variable_update\n\n" def init_the_init(full_path: str) -> None: @@ -61,6 +61,10 @@ def print_red(s: str): def add_type_import_path(function_name: str, arg: str) -> str: """ if not basic type, coerce to camelCase and add the import path """ + # for now, just treat Callables as basic types + if arg.startswith("Callable"): + return arg + if arg in BASIC_PYTHON_TYPES: return arg @@ -142,6 +146,25 @@ def get_type_and_def(type_spec: PropertyType) -> Tuple[str, str]: return "Any", "" else: return "Dict", "" + elif type_spec["kind"] == "function": + arg_types = [] + arg_defs = [] + if "spec" in type_spec: + return_type, _ = get_type_and_def(type_spec["spec"]["returnType"]) + if return_type not in BASIC_PYTHON_TYPES: + # for now only Python only supports basic types as return types + return_type = "Any" + + for argument in type_spec["spec"]["arguments"]: + arg_type, arg_def = get_type_and_def(argument["type"]) + arg_types.append(arg_type) + if arg_def: + arg_defs.append(arg_def) + + final_arg_type = "Callable[[{}], {}]".format(", ".join(arg_types), return_type) + return final_arg_type, "\n".join(arg_defs) + else: + return "Callable", "" elif type_spec["kind"] == "any": return "Any", "" else: diff --git a/polyapi/webhook.py b/polyapi/webhook.py index fd3f787..ad5cb97 100644 --- a/polyapi/webhook.py +++ b/polyapi/webhook.py @@ -15,7 +15,7 @@ WEBHOOK_DEFS_TEMPLATE = """ -from typing import List, Dict, Any, TypedDict +from typing import List, Dict, Any, TypedDict, Callable {function_args_def} """ @@ -115,6 +115,11 @@ def render_webhook_handle( return_type: Dict[str, Any], ) -> Tuple[str, str]: function_args, function_args_def = parse_arguments(function_name, arguments) + + if "WebhookEventTypeElement" in function_args: + # let's add the function name import! + function_args = function_args.replace("WebhookEventTypeElement", f"_{function_name}.WebhookEventTypeElement") + func_str = WEBHOOK_TEMPLATE.format( description=function_description, client_id=uuid.uuid4().hex, diff --git a/tests/test_utils.py b/tests/test_utils.py index 7e920bb..c3662a5 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,5 +1,8 @@ import unittest from polyapi.schema import _fix_title +from polyapi.utils import get_type_and_def + +OPENAPI_FUNCTION = {'kind': 'function', 'spec': {'arguments': [{'name': 'event', 'required': False, 'type': {'kind': 'object', 'schema': {'$schema': 'http://json-schema.org/draft-06/schema#', 'type': 'array', 'items': {'$ref': '#/definitions/WebhookEventTypeElement'}, 'definitions': {'WebhookEventTypeElement': {'type': 'object', 'additionalProperties': False, 'properties': {'title': {'type': 'string'}, 'manufacturerName': {'type': 'string'}, 'carType': {'type': 'string'}, 'id': {'type': 'integer'}}, 'required': ['carType', 'id', 'manufacturerName', 'title'], 'title': 'WebhookEventTypeElement'}}}}}, {'name': 'headers', 'required': False, 'type': {'kind': 'object', 'typeName': 'Record'}}, {'name': 'params', 'required': False, 'type': {'kind': 'object', 'typeName': 'Record'}}, {'name': 'polyCustom', 'required': False, 'type': {'kind': 'object', 'properties': [{'name': 'responseStatusCode', 'type': {'type': 'number', 'kind': 'primitive'}, 'required': True}, {'name': 'responseContentType', 'type': {'type': 'string', 'kind': 'primitive'}, 'required': True, 'nullable': True}]}}], 'returnType': {'kind': 'void'}, 'synchronous': True}} class T(unittest.TestCase): @@ -8,3 +11,7 @@ def test_fix_titles(self): output = 'from typing import TypedDict\nfrom typing_extensions import Required\n\n\nclass Numofcars(TypedDict, total=False):\n """ numOfCars. """\n\n requestNumber: Required[int]\n """\n Requestnumber.\n\n Required property\n """\n\n' fixed = _fix_title(input_data, output) self.assertIn("class numOfCars", fixed) + + def test_get_type_and_def(self): + arg_type, arg_def = get_type_and_def(OPENAPI_FUNCTION) + self.assertEqual(arg_type, "Callable[[List[WebhookEventTypeElement], Dict, Dict, Dict], None]") From 9631e64e897347977ef2a6e4206fcbfd823ceb48 Mon Sep 17 00:00:00 2001 From: Dan Fellin Date: Tue, 23 Apr 2024 12:40:29 -0700 Subject: [PATCH 27/36] 0.2.4.dev9, handle already connected error on windows --- polyapi/webhook.py | 9 +++++++-- pyproject.toml | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/polyapi/webhook.py b/polyapi/webhook.py index b03dfda..c28652f 100644 --- a/polyapi/webhook.py +++ b/polyapi/webhook.py @@ -1,5 +1,6 @@ import asyncio import socketio # type: ignore +from socketio.exceptions import ConnectionError # type: ignore import uuid from typing import Any, Dict, List, Tuple @@ -92,8 +93,12 @@ async def unregister(data: Dict[str, Any]): async def unregister_all(): _, base_url = get_api_key_and_url() - # need to reconnect because maybe socketio client disconnected after Ctrl+C? - await client.connect(base_url, transports=["websocket"], namespaces=["/events"]) + # maybe need to reconnect because maybe socketio client disconnected after Ctrl+C? + # feels like Linux disconnects but Windows stays connected + try: + await client.connect(base_url, transports=["websocket"], namespaces=["/events"]) + except ConnectionError: + pass await asyncio.gather(*[unregister(handler) for handler in active_handlers]) diff --git a/pyproject.toml b/pyproject.toml index df0b3af..65410be 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.2.4.dev8" +version = "0.2.4.dev9" description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ From 51c33cd644d8c004625fb5cacf29ffd4a5ebe119 Mon Sep 17 00:00:00 2001 From: Dan Fellin Date: Tue, 23 Apr 2024 13:23:31 -0700 Subject: [PATCH 28/36] 0.2.4.deva1, better user facing messages for webhooks and error handlers --- polyapi/error_handler.py | 10 +++++++--- polyapi/generate.py | 2 ++ polyapi/utils.py | 11 ++++++++++- polyapi/webhook.py | 9 ++++++--- pyproject.toml | 2 +- 5 files changed, 26 insertions(+), 8 deletions(-) diff --git a/polyapi/error_handler.py b/polyapi/error_handler.py index ad2a451..33e7891 100644 --- a/polyapi/error_handler.py +++ b/polyapi/error_handler.py @@ -1,6 +1,7 @@ import asyncio import copy import socketio # type: ignore +from socketio.exceptions import ConnectionError # type: ignore from typing import Any, Callable, Dict, List, Optional from polyapi.config import get_api_key_and_url @@ -28,7 +29,7 @@ async def get_client_and_connect(): async def unregister(data: Dict[str, Any]): - print(f"stopping error handler for '{data['path']}'...") + print(f"Stopping error handler for {data['path']}...") assert client await client.emit( "unregisterErrorHandler", @@ -40,14 +41,17 @@ async def unregister(data: Dict[str, Any]): async def unregister_all(): _, base_url = get_api_key_and_url() # need to reconnect because maybe socketio client disconnected after Ctrl+C? - await client.connect(base_url, transports=["websocket"], namespaces=["/events"]) + try: + await client.connect(base_url, transports=["websocket"], namespaces=["/events"]) + except ConnectionError: + pass await asyncio.gather(*[unregister(handler) for handler in active_handlers]) async def on( path: str, callback: Callable, options: Optional[Dict[str, Any]] = None ) -> None: - print(f"starting error handler for {path}...") + print(f"Starting error handler for {path}...") if not client: raise Exception("Client not initialized. Abort!") diff --git a/polyapi/generate.py b/polyapi/generate.py index 3466e58..94c52c6 100644 --- a/polyapi/generate.py +++ b/polyapi/generate.py @@ -176,6 +176,7 @@ def render_spec(spec: SpecificationDto): function_type = spec["type"] function_description = spec["description"] function_name = spec["name"] + function_context = spec["context"] function_id = spec["id"] arguments: List[PropertySpecification] = [] @@ -223,6 +224,7 @@ def render_spec(spec: SpecificationDto): elif function_type == "webhookHandle": func_str, func_type_defs = render_webhook_handle( function_type, + function_context, function_name, function_id, function_description, diff --git a/polyapi/utils.py b/polyapi/utils.py index 91c4b0a..166f5f9 100644 --- a/polyapi/utils.py +++ b/polyapi/utils.py @@ -165,4 +165,13 @@ def parse_arguments(function_name: str, arguments: List[PropertySpecification]) arg_string += f", # {description}\n" else: arg_string += ",\n" - return arg_string.rstrip("\n"), "\n\n".join(args_def) \ No newline at end of file + return arg_string.rstrip("\n"), "\n\n".join(args_def) + + +def poly_full_path(context, name) -> str: + """get the functions path as it will be exposed in the poly library""" + if context: + path = context + "." + name + else: + path = name + return f"poly.{path}" \ No newline at end of file diff --git a/polyapi/webhook.py b/polyapi/webhook.py index c28652f..d24e0cf 100644 --- a/polyapi/webhook.py +++ b/polyapi/webhook.py @@ -6,6 +6,7 @@ from polyapi.config import get_api_key_and_url from polyapi.typedefs import PropertySpecification +from polyapi.utils import poly_full_path # all active webhook handlers, used by unregister_all to cleanup active_handlers: List[Dict[str, Any]] = [] @@ -24,7 +25,7 @@ async def {function_name}(callback, options=None): \""" from polyapi.webhook import client, active_handlers - print("Starting webhook for {function_name}...") + print("Starting webhook for {function_path}...") if not client: raise Exception("Client not initialized. Abort!") @@ -66,7 +67,7 @@ async def handleEvent(data): "waitForResponse": options.get("waitForResponse"), }} await client.emit('registerWebhookEventHandler', data, namespace="/events", callback=registerCallback) - active_handlers.append({{"clientID": eventsClientId, "webhookHandleID": function_id, "apiKey": api_key}}) + active_handlers.append({{"clientID": eventsClientId, "webhookHandleID": function_id, "apiKey": api_key, "path": "{function_path}"}}) """ @@ -78,7 +79,7 @@ async def get_client_and_connect(): async def unregister(data: Dict[str, Any]): - print(f"stopping webhook handler for '{data['webhookHandleID']}'...") + print(f"Stopping webhook handler for {data['path']}...") assert client await client.emit( "unregisterWebhookEventHandler", @@ -104,6 +105,7 @@ async def unregister_all(): def render_webhook_handle( function_type: str, + function_context: str, function_name: str, function_id: str, function_description: str, @@ -115,6 +117,7 @@ def render_webhook_handle( client_id=uuid.uuid4().hex, function_id=function_id, function_name=function_name, + function_path=poly_full_path(function_context, function_name), ) return func_str, "" diff --git a/pyproject.toml b/pyproject.toml index 65410be..6541066 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.2.4.dev9" +version = "0.2.4.deva1" description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ From e6eff399ebef8564568210abfacf848ad0d7f35f Mon Sep 17 00:00:00 2001 From: Dan Fellin Date: Tue, 23 Apr 2024 13:27:00 -0700 Subject: [PATCH 29/36] woops lets go version 10 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 6541066..526412b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.2.4.deva1" +version = "0.2.4.dev10" description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ From 1fe0b58ca04f08df8f512b6d2127acb154d5bbd9 Mon Sep 17 00:00:00 2001 From: Dan Fellin Date: Tue, 23 Apr 2024 13:33:54 -0700 Subject: [PATCH 30/36] 0.2.4.dev11 --- polyapi/webhook.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/polyapi/webhook.py b/polyapi/webhook.py index d24e0cf..e4bdf5c 100644 --- a/polyapi/webhook.py +++ b/polyapi/webhook.py @@ -25,7 +25,7 @@ async def {function_name}(callback, options=None): \""" from polyapi.webhook import client, active_handlers - print("Starting webhook for {function_path}...") + print("Starting webhook handler for {function_path}...") if not client: raise Exception("Client not initialized. Abort!") diff --git a/pyproject.toml b/pyproject.toml index 526412b..2bace72 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.2.4.dev10" +version = "0.2.4.dev11" description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ From a49780eb86e4fadb8c2ee5e091f9f547f8b315be Mon Sep 17 00:00:00 2001 From: Dan Fellin Date: Wed, 24 Apr 2024 08:07:44 -0700 Subject: [PATCH 31/36] move polyCustom to own file --- polyapi/__init__.py | 11 ++++++++++- polyapi/poly_custom.py | 9 --------- pyproject.toml | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) delete mode 100644 polyapi/poly_custom.py diff --git a/polyapi/__init__.py b/polyapi/__init__.py index 984a00b..f818b53 100644 --- a/polyapi/__init__.py +++ b/polyapi/__init__.py @@ -1,6 +1,7 @@ import os import sys import truststore +from typing import Dict, Any truststore.inject_into_ssl() from .cli import CLI_COMMANDS @@ -11,4 +12,12 @@ currdir = os.path.dirname(os.path.abspath(__file__)) if not os.path.isdir(os.path.join(currdir, "poly")): print("No 'poly' found. Please run 'python3 -m polyapi generate' to generate the 'poly' library for your tenant.") - sys.exit(1) \ No newline at end of file + sys.exit(1) + + +polyCustom: Dict[str, Any] = { + "executionId": None, + "executionApiKey": None, + "responseStatusCode": 200, + "responseContentType": None, +} \ No newline at end of file diff --git a/polyapi/poly_custom.py b/polyapi/poly_custom.py deleted file mode 100644 index 619c344..0000000 --- a/polyapi/poly_custom.py +++ /dev/null @@ -1,9 +0,0 @@ -from typing import Dict, Any - - -polyCustom: Dict[str, Any] = { - "executionId": None, - "executionApiKey": None, - "responseStatusCode": 200, - "responseContentType": None, -} \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 2bace72..5c59e33 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.2.4.dev11" +version = "0.2.4.dev12" description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ From 5c66cfaeb72524f12da36d9ab769eea0714c9e2a Mon Sep 17 00:00:00 2001 From: Dan Fellin Date: Thu, 25 Apr 2024 10:15:20 -0700 Subject: [PATCH 32/36] onward --- polyapi/api.py | 2 ++ polyapi/auth.py | 6 +++--- polyapi/server.py | 2 ++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/polyapi/api.py b/polyapi/api.py index e8018f9..8cceff3 100644 --- a/polyapi/api.py +++ b/polyapi/api.py @@ -24,6 +24,8 @@ def {function_name}( \""" resp = execute("{function_type}", "{function_id}", {data}) return {api_response_type}(resp.json()) # type: ignore + + """ diff --git a/polyapi/auth.py b/polyapi/auth.py index a405510..199cfef 100644 --- a/polyapi/auth.py +++ b/polyapi/auth.py @@ -16,9 +16,9 @@ class AuthFunctionResponse(TypedDict): - status: int - data: Any - headers: Dict[str, str] + status: int + data: Any + headers: Dict[str, str] async def getToken(clientId: str, clientSecret: str, scopes: List[str], callback, options: Optional[Dict[str, Any]] = None): diff --git a/polyapi/server.py b/polyapi/server.py index b42094a..f616f26 100644 --- a/polyapi/server.py +++ b/polyapi/server.py @@ -22,6 +22,8 @@ def {function_name}( return {return_action} except: return resp.text + + """ From 9d8e053d7cb3a892bb48fb87950701804b3b7206 Mon Sep 17 00:00:00 2001 From: Dan Fellin Date: Thu, 25 Apr 2024 10:31:47 -0700 Subject: [PATCH 33/36] fix it --- polyapi/webhook.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/polyapi/webhook.py b/polyapi/webhook.py index eb6915b..1524655 100644 --- a/polyapi/webhook.py +++ b/polyapi/webhook.py @@ -122,9 +122,9 @@ def render_webhook_handle( ) -> Tuple[str, str]: function_args, function_args_def = parse_arguments(function_name, arguments) - if "WebhookEventTypeElement" in function_args: + if "WebhookEventType" in function_args: # let's add the function name import! - function_args = function_args.replace("WebhookEventTypeElement", f"_{function_name}.WebhookEventTypeElement") + function_args = function_args.replace("WebhookEventType", f"_{function_name}.WebhookEventType") func_str = WEBHOOK_TEMPLATE.format( description=function_description, From 947c07dec906a8768547e8432f7d2b653bfeba21 Mon Sep 17 00:00:00 2001 From: Dan Fellin Date: Thu, 25 Apr 2024 11:42:06 -0700 Subject: [PATCH 34/36] wip --- polyapi/cli.py | 3 ++- polyapi/function_cli.py | 2 ++ polyapi/generate.py | 18 ------------------ polyapi/rendered_spec.py | 34 ++++++++++++++++++++++++++++++++++ 4 files changed, 38 insertions(+), 19 deletions(-) create mode 100644 polyapi/rendered_spec.py diff --git a/polyapi/cli.py b/polyapi/cli.py index 14f21a2..ef2a2a9 100644 --- a/polyapi/cli.py +++ b/polyapi/cli.py @@ -3,8 +3,9 @@ from polyapi.utils import print_green from .config import clear_config, set_api_key_and_url -from .generate import generate, clear, save_rendered_specs +from .generate import generate, clear from .function_cli import function_add_or_update +from .rendered_spec import save_rendered_specs CLI_COMMANDS = ["setup", "generate", "function", "clear", "help", "save_rendered_specs"] diff --git a/polyapi/function_cli.py b/polyapi/function_cli.py index 0cbbb7a..ea82871 100644 --- a/polyapi/function_cli.py +++ b/polyapi/function_cli.py @@ -13,6 +13,7 @@ from polyapi.generate import get_functions_and_parse, generate_functions from polyapi.config import get_api_key_and_url from polyapi.constants import PYTHON_TO_JSONSCHEMA_TYPE_MAP +from polyapi.rendered_spec import update_rendered_spec from polyapi.utils import get_auth_headers, print_green, print_red, print_yellow import importlib @@ -270,6 +271,7 @@ def function_add_or_update( print(f"Function ID: {function_id}") print("Generating new custom function...", end="") functions = get_functions_and_parse(limit_ids=[function_id]) + update_rendered_spec(functions[0]) generate_functions(functions) print_green("DONE") else: diff --git a/polyapi/generate.py b/polyapi/generate.py index 94c52c6..2cb4fd3 100644 --- a/polyapi/generate.py +++ b/polyapi/generate.py @@ -154,24 +154,6 @@ def clear() -> None: print("Cleared!") -def save_rendered_specs() -> None: - specs = read_cached_specs() - # right now we just support rendered apiFunctions - api_specs = [spec for spec in specs if spec["type"] == "apiFunction"] - for spec in api_specs: - assert spec["function"] - func_str, type_defs = render_spec(spec) - data = { - "language": "python", - "apiFunctionId": spec["id"], - "signature": func_str, - "typedefs": type_defs, - } - resp = execute_post("/functions/rendered-specs", data) - print("adding", spec["context"], spec["name"]) - assert resp.status_code == 201, (resp.text, resp.status_code) - - def render_spec(spec: SpecificationDto): function_type = spec["type"] function_description = spec["description"] diff --git a/polyapi/rendered_spec.py b/polyapi/rendered_spec.py new file mode 100644 index 0000000..f5e9543 --- /dev/null +++ b/polyapi/rendered_spec.py @@ -0,0 +1,34 @@ +from typing import Dict +from polyapi.generate import read_cached_specs, render_spec +from polyapi.execute import execute_post +from polyapi.typedefs import SpecificationDto + + +def update_rendered_spec(spec: SpecificationDto): + print("Updating rendered spec...") + func_str, type_defs = render_spec(spec) + data = { + "language": "python", + "signature": func_str, + "typedefs": type_defs, + } + if spec["type"] == "apiFunction": + data["apiFunctionId"] = spec["id"] + elif spec["type"] == "serverFunction": + data["customFunctionId"] = spec["id"] + else: + raise NotImplementedError("todo") + + resp = execute_post("/functions/rendered-specs", data) + assert resp.status_code == 201, (resp.text, resp.status_code) + # this needs to run with something like `kn func run...` + + +def save_rendered_specs() -> None: + specs = read_cached_specs() + # right now we just support rendered apiFunctions + api_specs = [spec for spec in specs if spec["type"] == "apiFunction"] + for spec in api_specs: + assert spec["function"] + print("adding", spec["context"], spec["name"]) + update_rendered_spec(spec) From 8d34db5761f80887acc8c5bdb0bb8bebae855b03 Mon Sep 17 00:00:00 2001 From: Dan Fellin Date: Wed, 8 May 2024 11:46:35 -0700 Subject: [PATCH 35/36] next --- polyapi/function_cli.py | 4 ++-- pyproject.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/polyapi/function_cli.py b/polyapi/function_cli.py index ea82871..9e461c3 100644 --- a/polyapi/function_cli.py +++ b/polyapi/function_cli.py @@ -13,7 +13,7 @@ from polyapi.generate import get_functions_and_parse, generate_functions from polyapi.config import get_api_key_and_url from polyapi.constants import PYTHON_TO_JSONSCHEMA_TYPE_MAP -from polyapi.rendered_spec import update_rendered_spec +# from polyapi.rendered_spec import update_rendered_spec from polyapi.utils import get_auth_headers, print_green, print_red, print_yellow import importlib @@ -271,7 +271,7 @@ def function_add_or_update( print(f"Function ID: {function_id}") print("Generating new custom function...", end="") functions = get_functions_and_parse(limit_ids=[function_id]) - update_rendered_spec(functions[0]) + # update_rendered_spec(functions[0]) generate_functions(functions) print_green("DONE") else: diff --git a/pyproject.toml b/pyproject.toml index 7cff949..d77e224 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.2.5.dev1" +version = "0.2.5.dev2" description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ From 0bfc7d33371f26afaadd28bada5e4506a69ae2b5 Mon Sep 17 00:00:00 2001 From: Dan Fellin Date: Wed, 8 May 2024 11:48:44 -0700 Subject: [PATCH 36/36] there we go --- polyapi/generate.py | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/polyapi/generate.py b/polyapi/generate.py index de518e0..a4f429e 100644 --- a/polyapi/generate.py +++ b/polyapi/generate.py @@ -6,7 +6,6 @@ from polyapi.auth import render_auth_function from polyapi.client import render_client_function -from polyapi.execute import execute_post from polyapi.webhook import render_webhook_handle from .typedefs import PropertySpecification, SpecificationDto, VariableSpecDto @@ -157,24 +156,6 @@ def clear() -> None: print("Cleared!") -def save_rendered_specs() -> None: - specs = read_cached_specs() - # right now we just support rendered apiFunctions - api_specs = [spec for spec in specs if spec["type"] == "apiFunction"] - for spec in api_specs: - assert spec["function"] - func_str, type_defs = render_spec(spec) - data = { - "language": "python", - "apiFunctionId": spec["id"], - "signature": func_str, - "typedefs": type_defs, - } - resp = execute_post("/functions/rendered-specs", data) - print("adding", spec["context"], spec["name"]) - assert resp.status_code == 201, (resp.text, resp.status_code) - - def render_spec(spec: SpecificationDto): function_type = spec["type"] function_description = spec["description"]