diff --git a/polyapi/api.py b/polyapi/api.py index e8018f9..8cceff3 100644 --- a/polyapi/api.py +++ b/polyapi/api.py @@ -24,6 +24,8 @@ def {function_name}( \""" resp = execute("{function_type}", "{function_id}", {data}) return {api_response_type}(resp.json()) # type: ignore + + """ diff --git a/polyapi/auth.py b/polyapi/auth.py index a405510..199cfef 100644 --- a/polyapi/auth.py +++ b/polyapi/auth.py @@ -16,9 +16,9 @@ class AuthFunctionResponse(TypedDict): - status: int - data: Any - headers: Dict[str, str] + status: int + data: Any + headers: Dict[str, str] async def getToken(clientId: str, clientSecret: str, scopes: List[str], callback, options: Optional[Dict[str, Any]] = None): diff --git a/polyapi/cli.py b/polyapi/cli.py index 14f21a2..cd0d2ac 100644 --- a/polyapi/cli.py +++ b/polyapi/cli.py @@ -3,11 +3,12 @@ from polyapi.utils import print_green from .config import clear_config, set_api_key_and_url -from .generate import generate, clear, save_rendered_specs +from .generate import generate, clear from .function_cli import function_add_or_update +from .rendered_spec import get_and_update_rendered_spec -CLI_COMMANDS = ["setup", "generate", "function", "clear", "help", "save_rendered_specs"] +CLI_COMMANDS = ["setup", "generate", "function", "clear", "help", "update_rendered_spec"] CLIENT_DESC = """Commands python -m polyapi setup Setup your Poly connection @@ -17,7 +18,7 @@ """ -def execute_from_cli(): +def execute_from_cli() -> None: parser = argparse.ArgumentParser( prog="python -m polyapi", description=CLIENT_DESC, formatter_class=argparse.RawTextHelpFormatter ) @@ -42,8 +43,14 @@ def execute_from_cli(): elif command == "setup": clear_config() generate() - elif command == "save_rendered_specs": - save_rendered_specs() + elif command == "update_rendered_spec": + assert len(args.subcommands) == 2 + updated = get_and_update_rendered_spec(args.subcommands[0], args.subcommands[1]) + if updated: + print("Updated rendered spec!") + else: + print("Failed to update rendered spec!") + exit(1) elif command == "clear": print("Clearing the generated library...") clear() diff --git a/polyapi/constants.py b/polyapi/constants.py index a58b64d..16bbeaf 100644 --- a/polyapi/constants.py +++ b/polyapi/constants.py @@ -5,6 +5,8 @@ "boolean": "bool", "array": "List", "object": "Dict", + "function": "Callable", + "void": "None", } @@ -15,6 +17,8 @@ "bool": "boolean", "List": "array", "Dict": "object", + "Callable": "function", + "None": "void", } BASIC_PYTHON_TYPES = set(PYTHON_TO_JSONSCHEMA_TYPE_MAP.keys()) diff --git a/polyapi/generate.py b/polyapi/generate.py index de518e0..a4f429e 100644 --- a/polyapi/generate.py +++ b/polyapi/generate.py @@ -6,7 +6,6 @@ from polyapi.auth import render_auth_function from polyapi.client import render_client_function -from polyapi.execute import execute_post from polyapi.webhook import render_webhook_handle from .typedefs import PropertySpecification, SpecificationDto, VariableSpecDto @@ -157,24 +156,6 @@ def clear() -> None: print("Cleared!") -def save_rendered_specs() -> None: - specs = read_cached_specs() - # right now we just support rendered apiFunctions - api_specs = [spec for spec in specs if spec["type"] == "apiFunction"] - for spec in api_specs: - assert spec["function"] - func_str, type_defs = render_spec(spec) - data = { - "language": "python", - "apiFunctionId": spec["id"], - "signature": func_str, - "typedefs": type_defs, - } - resp = execute_post("/functions/rendered-specs", data) - print("adding", spec["context"], spec["name"]) - assert resp.status_code == 201, (resp.text, resp.status_code) - - def render_spec(spec: SpecificationDto): function_type = spec["type"] function_description = spec["description"] diff --git a/polyapi/rendered_spec.py b/polyapi/rendered_spec.py new file mode 100644 index 0000000..5b3a394 --- /dev/null +++ b/polyapi/rendered_spec.py @@ -0,0 +1,73 @@ +import os +from typing import Optional + +import requests +from polyapi.config import get_api_key_and_url +from polyapi.generate import read_cached_specs, render_spec +from polyapi.typedefs import SpecificationDto + + +def update_rendered_spec(api_key: str, spec: SpecificationDto): + print("Updating rendered spec...") + func_str, type_defs = render_spec(spec) + data = { + "language": "python", + "signature": func_str, + "typedefs": type_defs, + } + if spec["type"] == "apiFunction": + data["apiFunctionId"] = spec["id"] + elif spec["type"] == "serverFunction": + data["customFunctionId"] = spec["id"] + elif spec["type"] == "clientFunction": + data["customFunctionId"] = spec["id"] + elif spec["type"] == "webhookHandle": + data["webhookHandleId"] = spec["id"] + else: + raise NotImplementedError("todo") + + # use super key on develop-k8s here! + _, base_url = get_api_key_and_url() + if not base_url: + base_url = os.environ.get("HOST_URL") + + url = f"{base_url}/functions/rendered-specs" + headers = {"Authorization": f"Bearer {api_key}"} + resp = requests.post(url, json=data, headers=headers) + assert resp.status_code == 201, (resp.text, resp.status_code) + + +def _get_spec(api_key: str, spec_id: str) -> Optional[SpecificationDto]: + _, base_url = get_api_key_and_url() + if not base_url: + base_url = os.environ.get("HOST_URL") + + url = f"{base_url}/specs" + headers = {"Authorization": f"Bearer {api_key}"} + resp = requests.get(url, headers=headers) + if resp.status_code == 200: + specs = resp.json() + for spec in specs: + if spec['id'] == spec_id: + return spec + return None + else: + raise NotImplementedError(resp.content) + + +def get_and_update_rendered_spec(api_key: str, spec_id: str) -> bool: + spec = _get_spec(api_key, spec_id) + if spec: + update_rendered_spec(api_key, spec) + return True + return False + + +def save_rendered_specs() -> None: + specs = read_cached_specs() + # right now we just support rendered apiFunctions + api_specs = [spec for spec in specs if spec["type"] == "apiFunction"] + for spec in api_specs: + assert spec["function"] + print("adding", spec["context"], spec["name"]) + update_rendered_spec("FIXME", spec) diff --git a/polyapi/schema.py b/polyapi/schema.py index 87d8c6b..4633369 100644 --- a/polyapi/schema.py +++ b/polyapi/schema.py @@ -1,3 +1,4 @@ +import logging import contextlib from typing import Dict from jsonschema_gentypes.cli import process_config @@ -18,10 +19,8 @@ def _cleanup_input_for_gentypes(input_data: Dict): # jsonschema_gentypes doesn't like double quotes in enums # TODO fix this upstream for idx, enum in enumerate(v): - assert isinstance(enum, str) - v[idx] = enum.replace('"', "'") - - + if isinstance(enum, str): + v[idx] = enum.replace('"', "'") def _temp_store_input_data(input_data: Dict) -> str: @@ -33,6 +32,23 @@ def _temp_store_input_data(input_data: Dict) -> str: return temp_file.name +def wrapped_generate_schema_types(type_spec: dict, root, fallback_type): + if not root: + root = "MyList" if fallback_type == "List" else "MyDict" + + root = clean_title(root) + + try: + return root, generate_schema_types(type_spec, root=root) + except RecursionError: + # some schemas are so huge, our library cant handle it + # TODO identify critical recursion penalty and maybe switch underlying logic to iterative? + return fallback_type, "" + except: + logging.exception(f"Error when generating schema type: {type_spec}") + return fallback_type, "" + + def generate_schema_types(input_data: Dict, root=None): """takes in a Dict representing a schema as input then appends the resulting python code to the output file""" _cleanup_input_for_gentypes(input_data) @@ -48,6 +64,7 @@ def generate_schema_types(input_data: Dict, root=None): "source": tmp_input, "destination": tmp_output, "root_name": root, + "api_arguments": {"get_name_properties": "UpperFirst"}, } ], } @@ -55,7 +72,7 @@ def generate_schema_types(input_data: Dict, root=None): # jsonschema_gentypes prints source to stdout # no option to surpress so we do this with contextlib.redirect_stdout(None): - process_config(config) + process_config(config, [tmp_input]) with open(tmp_output) as f: output = f.read() diff --git a/polyapi/server.py b/polyapi/server.py index 6a359bf..f616f26 100644 --- a/polyapi/server.py +++ b/polyapi/server.py @@ -4,7 +4,7 @@ from polyapi.utils import camelCase, add_type_import_path, parse_arguments, get_type_and_def SERVER_DEFS_TEMPLATE = """ -from typing import List, Dict, Any, TypedDict +from typing import List, Dict, Any, TypedDict, Callable {args_def} {return_type_def} """ @@ -22,6 +22,8 @@ def {function_name}( return {return_action} except: return resp.text + + """ diff --git a/polyapi/typedefs.py b/polyapi/typedefs.py index 544f3c1..e23113a 100644 --- a/polyapi/typedefs.py +++ b/polyapi/typedefs.py @@ -12,6 +12,7 @@ class PropertySpecification(TypedDict): class PropertyType(TypedDict): kind: Literal['void', 'primitive', 'array', 'object', 'function', 'plain'] + spec: NotRequired[Dict] name: NotRequired[str] type: NotRequired[str] items: NotRequired['PropertyType'] diff --git a/polyapi/utils.py b/polyapi/utils.py index 604a700..d7b105b 100644 --- a/polyapi/utils.py +++ b/polyapi/utils.py @@ -1,16 +1,16 @@ import re import os -import logging from typing import Tuple, List from colorama import Fore, Style from polyapi.constants import BASIC_PYTHON_TYPES from polyapi.typedefs import PropertySpecification, PropertyType -from polyapi.schema import generate_schema_types, clean_title, map_primitive_types +from polyapi.schema import wrapped_generate_schema_types, clean_title, map_primitive_types # this string should be in every __init__ file. # it contains all the imports needed for the function or variable code to run -CODE_IMPORTS = "from typing import List, Dict, Any, TypedDict, Optional\nimport logging\nimport requests\nimport socketio # type: ignore\nfrom polyapi.config import get_api_key_and_url\nfrom polyapi.execute import execute, execute_post, variable_get, variable_update\n\n" +CODE_IMPORTS = "from typing import List, Dict, Any, TypedDict, Optional, Callable\nimport logging\nimport requests\nimport socketio # type: ignore\nfrom polyapi.config import get_api_key_and_url\nfrom polyapi.execute import execute, execute_post, variable_get, variable_update\n\n" +FALLBACK_TYPES = {"Dict", "List"} def init_the_init(full_path: str) -> None: @@ -61,6 +61,10 @@ def print_red(s: str): def add_type_import_path(function_name: str, arg: str) -> str: """ if not basic type, coerce to camelCase and add the import path """ + # for now, just treat Callables as basic types + if arg.startswith("Callable"): + return arg + if arg in BASIC_PYTHON_TYPES: return arg @@ -92,11 +96,7 @@ def get_type_and_def(type_spec: PropertyType) -> Tuple[str, str]: if type_spec.get("items"): items = type_spec["items"] if items.get("$ref"): - try: - return "ResponseType", generate_schema_types(type_spec, root="ResponseType") # type: ignore - except: - logging.exception(f"Error when generating schema type: {type_spec}") - return "Dict", "" + return wrapped_generate_schema_types(type_spec, "ResponseType", "Dict") # type: ignore else: item_type, _ = get_type_and_def(items) title = f"List[{item_type}]" @@ -112,12 +112,7 @@ def get_type_and_def(type_spec: PropertyType) -> Tuple[str, str]: title = schema.get("title", "") if title: assert isinstance(title, str) - title = clean_title(title) - try: - return title, generate_schema_types(schema, root=title) # type: ignore - except: - logging.exception(f"Error when generating schema type: {schema}") - return "Dict", "" + return wrapped_generate_schema_types(schema, title, "Dict") # type: ignore elif schema.get("items"): # fallback to schema $ref name if no explicit title @@ -128,20 +123,34 @@ def get_type_and_def(type_spec: PropertyType) -> Tuple[str, str]: title = items.get("$ref", "") # type: ignore title = title.rsplit("/", 1)[-1] - title = clean_title(title) if not title: return "List", "" title = f"List[{title}]" - try: - return title, generate_schema_types(schema, root=title) - except: - logging.exception(f"Error when generating schema type: {schema}") - return "List", "" + return wrapped_generate_schema_types(schema, title, "List") else: return "Any", "" else: return "Dict", "" + elif type_spec["kind"] == "function": + arg_types = [] + arg_defs = [] + if "spec" in type_spec: + return_type, _ = get_type_and_def(type_spec["spec"]["returnType"]) + if return_type not in BASIC_PYTHON_TYPES: + # for now only Python only supports basic types as return types + return_type = "Any" + + for argument in type_spec["spec"]["arguments"]: + arg_type, arg_def = get_type_and_def(argument["type"]) + arg_types.append(arg_type) + if arg_def: + arg_defs.append(arg_def) + + final_arg_type = "Callable[[{}], {}]".format(", ".join(arg_types), return_type) + return final_arg_type, "\n".join(arg_defs) + else: + return "Callable", "" elif type_spec["kind"] == "any": return "Any", "" else: diff --git a/polyapi/webhook.py b/polyapi/webhook.py index e4bdf5c..1524655 100644 --- a/polyapi/webhook.py +++ b/polyapi/webhook.py @@ -6,7 +6,7 @@ from polyapi.config import get_api_key_and_url from polyapi.typedefs import PropertySpecification -from polyapi.utils import poly_full_path +from polyapi.utils import parse_arguments, poly_full_path # all active webhook handlers, used by unregister_all to cleanup active_handlers: List[Dict[str, Any]] = [] @@ -15,10 +15,18 @@ client = None +WEBHOOK_DEFS_TEMPLATE = """ +from typing import List, Dict, Any, TypedDict, Callable +{function_args_def} +""" + + WEBHOOK_TEMPLATE = """ -async def {function_name}(callback, options=None): +async def {function_name}( +{function_args} +): \"""{description} Function ID: {function_id} @@ -112,15 +120,22 @@ def render_webhook_handle( arguments: List[PropertySpecification], return_type: Dict[str, Any], ) -> Tuple[str, str]: + function_args, function_args_def = parse_arguments(function_name, arguments) + + if "WebhookEventType" in function_args: + # let's add the function name import! + function_args = function_args.replace("WebhookEventType", f"_{function_name}.WebhookEventType") + func_str = WEBHOOK_TEMPLATE.format( description=function_description, client_id=uuid.uuid4().hex, function_id=function_id, function_name=function_name, + function_args=function_args, function_path=poly_full_path(function_context, function_name), ) - - return func_str, "" + func_defs = WEBHOOK_DEFS_TEMPLATE.format(function_args_def=function_args_def) + return func_str, func_defs def start(*args): diff --git a/pyproject.toml b/pyproject.toml index e889d20..376e0e1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,13 +3,13 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.2.4" +version = "0.2.5" description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ "requests==2.31.0", "typing_extensions==4.10.0", - "jsonschema-gentypes==2.4.0", + "jsonschema-gentypes==2.6.0", "pydantic==2.6.4", "stdlib_list==0.10.0", "colorama==0.4.4", diff --git a/requirements.txt b/requirements.txt index c5060d1..3e890b0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ requests==2.31.0 typing_extensions==4.10.0 -jsonschema-gentypes==2.4.0 +jsonschema-gentypes==2.6.0 pydantic==2.6.4 stdlib_list==0.10.0 colorama==0.4.4 diff --git a/tests/test_rendered_spec.py b/tests/test_rendered_spec.py new file mode 100644 index 0000000..9bf7003 --- /dev/null +++ b/tests/test_rendered_spec.py @@ -0,0 +1,52 @@ +import unittest +from mock import patch, Mock + +from polyapi.rendered_spec import get_and_update_rendered_spec + +GET_PRODUCTS_COUNT = { + "id": "8f7d24b0-4a29-40c0-9091", + "type": "serverFunction", + "context": "test", + "name": "getProductsCount111", + "description": "An API call to retrieve the count of products in the product list.", + "requirements": ["snabbdom"], + "function": { + "arguments": [ + { + "name": "products", + "required": False, + "type": { + "kind": "array", + "items": {"kind": "primitive", "type": "string"}, + }, + } + ], + "returnType": {"kind": "plain", "value": "number"}, + "synchronous": True, + }, + "code": "", + "language": "javascript", + "visibilityMetadata": {"visibility": "ENVIRONMENT"}, +} + + +class T(unittest.TestCase): + @patch("polyapi.rendered_spec._get_spec") + def test_get_and_update_rendered_spec_fail(self, _get_spec): + """ pass in a bad id to update and make sure it returns False + """ + _get_spec.return_value = None + updated = get_and_update_rendered_spec("abc", "123") + self.assertEqual(_get_spec.call_count, 1) + self.assertFalse(updated) + + @patch("polyapi.rendered_spec.requests.post") + @patch("polyapi.rendered_spec._get_spec") + def test_get_and_update_rendered_spec_success(self, _get_spec, post): + """ pass in a bad id to update and make sure it returns False + """ + _get_spec.return_value = GET_PRODUCTS_COUNT + post.return_value = Mock(status_code=201, text="Created") + updated = get_and_update_rendered_spec("abc", "123") + self.assertEqual(_get_spec.call_count, 1) + self.assertTrue(updated) \ No newline at end of file diff --git a/tests/test_schema.py b/tests/test_schema.py new file mode 100644 index 0000000..8602fd6 --- /dev/null +++ b/tests/test_schema.py @@ -0,0 +1,20 @@ +import unittest +from polyapi.schema import wrapped_generate_schema_types + +SCHEMA = { + "$schema": "http://json-schema.org/draft-06/schema#", + "type": "object", + "properties": {"name": {"type": "string"}}, + "required": ["name"], + "additionalProperties": False, + "definitions": {}, +} + + +class T(unittest.TestCase): + def test_fix_titles(self): + output = wrapped_generate_schema_types(SCHEMA, "", "Dict") + self.assertEqual("MyDict", output[0]) + self.assertIn("class MyDict", output[1]) + + # should not throw with unknown dialect error diff --git a/tests/test_utils.py b/tests/test_utils.py index 7e920bb..c3662a5 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,5 +1,8 @@ import unittest from polyapi.schema import _fix_title +from polyapi.utils import get_type_and_def + +OPENAPI_FUNCTION = {'kind': 'function', 'spec': {'arguments': [{'name': 'event', 'required': False, 'type': {'kind': 'object', 'schema': {'$schema': 'http://json-schema.org/draft-06/schema#', 'type': 'array', 'items': {'$ref': '#/definitions/WebhookEventTypeElement'}, 'definitions': {'WebhookEventTypeElement': {'type': 'object', 'additionalProperties': False, 'properties': {'title': {'type': 'string'}, 'manufacturerName': {'type': 'string'}, 'carType': {'type': 'string'}, 'id': {'type': 'integer'}}, 'required': ['carType', 'id', 'manufacturerName', 'title'], 'title': 'WebhookEventTypeElement'}}}}}, {'name': 'headers', 'required': False, 'type': {'kind': 'object', 'typeName': 'Record'}}, {'name': 'params', 'required': False, 'type': {'kind': 'object', 'typeName': 'Record'}}, {'name': 'polyCustom', 'required': False, 'type': {'kind': 'object', 'properties': [{'name': 'responseStatusCode', 'type': {'type': 'number', 'kind': 'primitive'}, 'required': True}, {'name': 'responseContentType', 'type': {'type': 'string', 'kind': 'primitive'}, 'required': True, 'nullable': True}]}}], 'returnType': {'kind': 'void'}, 'synchronous': True}} class T(unittest.TestCase): @@ -8,3 +11,7 @@ def test_fix_titles(self): output = 'from typing import TypedDict\nfrom typing_extensions import Required\n\n\nclass Numofcars(TypedDict, total=False):\n """ numOfCars. """\n\n requestNumber: Required[int]\n """\n Requestnumber.\n\n Required property\n """\n\n' fixed = _fix_title(input_data, output) self.assertIn("class numOfCars", fixed) + + def test_get_type_and_def(self): + arg_type, arg_def = get_type_and_def(OPENAPI_FUNCTION) + self.assertEqual(arg_type, "Callable[[List[WebhookEventTypeElement], Dict, Dict, Dict], None]")