From dd588b53cee1da3d5dac3346868e9d48adeedb88 Mon Sep 17 00:00:00 2001 From: Dan Fellin Date: Thu, 20 Mar 2025 09:19:22 -0700 Subject: [PATCH 01/14] onward --- .gitignore | 5 +++-- polyapi/generate.py | 2 +- polyapi/poly/__init__.py | 0 polyapi/poly_schemas.py | 43 ++++++++++++++++++++++++++++++++++++++++ polyapi/schema.py | 2 ++ 5 files changed, 49 insertions(+), 3 deletions(-) delete mode 100644 polyapi/poly/__init__.py create mode 100644 polyapi/poly_schemas.py diff --git a/.gitignore b/.gitignore index 2ab87b6..49ae8f4 100644 --- a/.gitignore +++ b/.gitignore @@ -34,5 +34,6 @@ __pycache__ .polyapi-python function_add_test.py lib_test*.py -polyapi/poly/ -polyapi/vari/ +polyapi/poly +polyapi/vari +polyapi/schemas diff --git a/polyapi/generate.py b/polyapi/generate.py index 0869c8b..e76556c 100644 --- a/polyapi/generate.py +++ b/polyapi/generate.py @@ -23,7 +23,7 @@ "webhookHandle", } -SUPPORTED_TYPES = SUPPORTED_FUNCTION_TYPES | {"serverVariable"} +SUPPORTED_TYPES = SUPPORTED_FUNCTION_TYPES | {"serverVariable", "schema"} def get_specs() -> List: diff --git a/polyapi/poly/__init__.py b/polyapi/poly/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/polyapi/poly_schemas.py b/polyapi/poly_schemas.py new file mode 100644 index 0000000..a60dc0b --- /dev/null +++ b/polyapi/poly_schemas.py @@ -0,0 +1,43 @@ +from typing import Any, Dict, List, Tuple + +from polyapi.typedefs import PropertySpecification +from polyapi.utils import parse_arguments, get_type_and_def + +DEFS_TEMPLATE = """ +from typing import List, Dict, Any, TypedDict +{args_def} +{return_type_def} +""" + + +def _wrap_code_in_try_except(code: str) -> str: + """ this is necessary because client functions with imports will blow up ALL server functions, + even if they don't use them. + because the server function will try to load all client functions when loading the library + """ + prefix = """logger = logging.getLogger("poly") +try: + """ + suffix = """except ImportError as e: + logger.debug(e)""" + + lines = code.split("\n") + code = "\n ".join(lines) + return "".join([prefix, code, "\n", suffix]) + + +def render_poly_schema( + function_name: str, + code: str, + arguments: List[PropertySpecification], + return_type: Dict[str, Any], +) -> Tuple[str, str]: + # args, args_def = parse_arguments(function_name, arguments) + # return_type_name, return_type_def = get_type_and_def(return_type) # type: ignore + # func_type_defs = DEFS_TEMPLATE.format( + # args_def=args_def, + # return_type_def=return_type_def, + # ) + # code = _wrap_code_in_try_except(code) + # return code + "\n\n", func_type_defs + return "'TODO'", "'TODO'" diff --git a/polyapi/schema.py b/polyapi/schema.py index 860bbaa..d99890a 100644 --- a/polyapi/schema.py +++ b/polyapi/schema.py @@ -1,3 +1,5 @@ +""" NOTE: this file represents the schema parsing logic for jsonschema_gentypes +""" import logging import contextlib from typing import Dict From 721c1f986ffea808c265aef1c476b3cb64d6a89f Mon Sep 17 00:00:00 2001 From: Dan Fellin Date: Thu, 20 Mar 2025 10:48:08 -0700 Subject: [PATCH 02/14] adding schemas for Pythonland! --- polyapi/cli.py | 3 +- polyapi/generate.py | 30 +++++++++------- polyapi/poly_schemas.py | 77 +++++++++++++++++++++-------------------- polyapi/typedefs.py | 10 ++++++ polyapi/utils.py | 9 +++-- 5 files changed, 76 insertions(+), 53 deletions(-) diff --git a/polyapi/cli.py b/polyapi/cli.py index 836d701..9b6e0cf 100644 --- a/polyapi/cli.py +++ b/polyapi/cli.py @@ -13,6 +13,7 @@ CLI_COMMANDS = ["setup", "generate", "function", "clear", "help", "update_rendered_spec"] + def execute_from_cli(): # First we setup all our argument parsing logic # Then we parse the arguments (waaay at the bottom) @@ -46,7 +47,7 @@ def setup(args): def generate_command(args): initialize_config() - print("Generating Poly functions...", end="") + print("Generating Poly Python SDK...", end="") generate() print_green("DONE") diff --git a/polyapi/generate.py b/polyapi/generate.py index e76556c..2aec32c 100644 --- a/polyapi/generate.py +++ b/polyapi/generate.py @@ -2,13 +2,14 @@ import requests import os import shutil -from typing import List +from typing import List, cast from polyapi.auth import render_auth_function from polyapi.client import render_client_function +from polyapi.poly_schemas import generate_schemas from polyapi.webhook import render_webhook_handle -from .typedefs import PropertySpecification, SpecificationDto, VariableSpecDto +from .typedefs import PropertySpecification, SchemaSpecDto, SpecificationDto, VariableSpecDto from .api import render_api_function from .server import render_server_function from .utils import add_import_to_init, get_auth_headers, init_the_init, to_func_namespace @@ -98,16 +99,13 @@ def get_functions_and_parse(limit_ids: List[str] | None = None) -> List[Specific def get_variables() -> List[VariableSpecDto]: - api_key, api_url = get_api_key_and_url() - headers = {"Authorization": f"Bearer {api_key}"} - # TODO do some caching so this and get_functions just do 1 function call - url = f"{api_url}/specs" - resp = requests.get(url, headers=headers) - if resp.status_code == 200: - specs = resp.json() - return [spec for spec in specs if spec["type"] == "serverVariable"] - else: - raise NotImplementedError(resp.content) + specs = read_cached_specs() + return [cast(VariableSpecDto, spec) for spec in specs if spec["type"] == "serverVariable"] + + +def get_schemas() -> List[SchemaSpecDto]: + specs = read_cached_specs() + return [cast(SchemaSpecDto, spec) for spec in specs if spec["type"] == "schema"] def remove_old_library(): @@ -120,6 +118,10 @@ def remove_old_library(): if os.path.exists(path): shutil.rmtree(path) + path = os.path.join(currdir, "schemas") + if os.path.exists(path): + shutil.rmtree(path) + def generate() -> None: @@ -138,6 +140,10 @@ def generate() -> None: if variables: generate_variables(variables) + schemas = get_schemas() + if schemas: + generate_schemas(schemas) + # indicator to vscode extension that this is a polyapi-python project file_path = os.path.join(os.getcwd(), ".polyapi-python") open(file_path, "w").close() diff --git a/polyapi/poly_schemas.py b/polyapi/poly_schemas.py index a60dc0b..aadcce1 100644 --- a/polyapi/poly_schemas.py +++ b/polyapi/poly_schemas.py @@ -1,43 +1,46 @@ +import os from typing import Any, Dict, List, Tuple -from polyapi.typedefs import PropertySpecification -from polyapi.utils import parse_arguments, get_type_and_def +from polyapi.utils import add_import_to_init, init_the_init, pascalCase -DEFS_TEMPLATE = """ -from typing import List, Dict, Any, TypedDict -{args_def} -{return_type_def} +from .typedefs import SchemaSpecDto + + +SPEC_TEMPLATE = """class {name}(TypedDict): + pass """ -def _wrap_code_in_try_except(code: str) -> str: - """ this is necessary because client functions with imports will blow up ALL server functions, - even if they don't use them. - because the server function will try to load all client functions when loading the library - """ - prefix = """logger = logging.getLogger("poly") -try: - """ - suffix = """except ImportError as e: - logger.debug(e)""" - - lines = code.split("\n") - code = "\n ".join(lines) - return "".join([prefix, code, "\n", suffix]) - - -def render_poly_schema( - function_name: str, - code: str, - arguments: List[PropertySpecification], - return_type: Dict[str, Any], -) -> Tuple[str, str]: - # args, args_def = parse_arguments(function_name, arguments) - # return_type_name, return_type_def = get_type_and_def(return_type) # type: ignore - # func_type_defs = DEFS_TEMPLATE.format( - # args_def=args_def, - # return_type_def=return_type_def, - # ) - # code = _wrap_code_in_try_except(code) - # return code + "\n\n", func_type_defs - return "'TODO'", "'TODO'" +def generate_schemas(specs: List[SchemaSpecDto]): + for spec in specs: + create_schema(spec) + + +def create_schema(spec: SchemaSpecDto) -> None: + folders = ["schemas"] + if spec["context"]: + folders += [pascalCase(s) for s in spec["context"].split(".")] + + # build up the full_path by adding all the folders + full_path = os.path.join(os.path.dirname(os.path.abspath(__file__))) + + for idx, folder in enumerate(folders): + full_path = os.path.join(full_path, folder) + if not os.path.exists(full_path): + os.makedirs(full_path) + next = folders[idx + 1] if idx + 1 < len(folders) else None + if next: + add_import_to_init(full_path, next) + + add_schema_to_init(full_path, spec) + + +def add_schema_to_init(full_path: str, spec: SchemaSpecDto): + init_the_init(full_path) + init_path = os.path.join(full_path, "__init__.py") + with open(init_path, "a") as f: + f.write(render_poly_schema(spec) + "\n\n") + + +def render_poly_schema(spec: SchemaSpecDto) -> str: + return SPEC_TEMPLATE.format(name=pascalCase(spec["name"])) \ No newline at end of file diff --git a/polyapi/typedefs.py b/polyapi/typedefs.py index c8d77cf..1f95c22 100644 --- a/polyapi/typedefs.py +++ b/polyapi/typedefs.py @@ -55,6 +55,15 @@ class VariableSpecDto(TypedDict): variable: VariableSpecification type: Literal['serverVariable'] + +class SchemaSpecDto(TypedDict): + id: str + context: str + name: str + type: Literal['schema'] + # TODO add more + + Visibility = Union[Literal['PUBLIC'], Literal['TENANT'], Literal['ENVIRONMENT']] @@ -69,6 +78,7 @@ class PolyServerFunction(PolyDeployable): always_on: NotRequired[bool] visibility: NotRequired[Visibility] + class PolyClientFunction(PolyDeployable): logs_enabled: NotRequired[bool] visibility: NotRequired[Visibility] diff --git a/polyapi/utils.py b/polyapi/utils.py index a5141a6..9aa5dab 100644 --- a/polyapi/utils.py +++ b/polyapi/utils.py @@ -11,8 +11,7 @@ # this string should be in every __init__ file. # it contains all the imports needed for the function or variable code to run -CODE_IMPORTS = "from typing import List, Dict, Any, TypedDict, Optional, Callable\nimport logging\nimport requests\nimport socketio # type: ignore\nfrom polyapi.config import get_api_key_and_url\nfrom polyapi.execute import execute, execute_post, variable_get, variable_update\n\n" -FALLBACK_TYPES = {"Dict", "List"} +CODE_IMPORTS = "from typing import List, Dict, Any, Optional, Callable\nfrom typing_extensions import TypedDict, NotRequired\nimport logging\nimport requests\nimport socketio # type: ignore\nfrom polyapi.config import get_api_key_and_url\nfrom polyapi.execute import execute, execute_post, variable_get, variable_update\n\n" def init_the_init(full_path: str) -> None: @@ -38,7 +37,7 @@ def get_auth_headers(api_key: str): return {"Authorization": f"Bearer {api_key}"} -def camelCase(s): +def camelCase(s: str) -> str: s = s.strip() if " " in s or "-" in s: s = re.sub(r"(_|-)+", " ", s).title().replace(" ", "") @@ -48,6 +47,10 @@ def camelCase(s): return s +def pascalCase(s) -> str: + return re.sub(r"(^|_)([a-z])", lambda match: match.group(2).upper(), s) + + def print_green(s: str): print(Fore.GREEN + s + Style.RESET_ALL) From cb0cf77fe596d4f6617db66fd2efcf7bd6bafad5 Mon Sep 17 00:00:00 2001 From: Dan Fellin Date: Thu, 20 Mar 2025 11:28:12 -0700 Subject: [PATCH 03/14] onward --- polyapi/poly_schemas.py | 26 ++++++++++++++++++++------ polyapi/typedefs.py | 4 ++++ polyapi/utils.py | 9 +++++---- pyproject.toml | 4 ++-- requirements.txt | 4 ++-- 5 files changed, 33 insertions(+), 14 deletions(-) diff --git a/polyapi/poly_schemas.py b/polyapi/poly_schemas.py index aadcce1..753101f 100644 --- a/polyapi/poly_schemas.py +++ b/polyapi/poly_schemas.py @@ -1,12 +1,20 @@ import os from typing import Any, Dict, List, Tuple -from polyapi.utils import add_import_to_init, init_the_init, pascalCase +from polyapi.schema import generate_schema_types +from polyapi.utils import add_import_to_init, init_the_init +from tests.test_schema import SCHEMA from .typedefs import SchemaSpecDto +SCHEMA_CODE_IMPORTS = """from typing_extensions import TypedDict, NotRequired -SPEC_TEMPLATE = """class {name}(TypedDict): + +""" + + +FALLBACK_SPEC_TEMPLATE = """class {name}(TypedDict, total=False): + ''' unable to generate schema for {name}, defaulting to permissive type ''' pass """ @@ -19,7 +27,7 @@ def generate_schemas(specs: List[SchemaSpecDto]): def create_schema(spec: SchemaSpecDto) -> None: folders = ["schemas"] if spec["context"]: - folders += [pascalCase(s) for s in spec["context"].split(".")] + folders += [s for s in spec["context"].split(".")] # build up the full_path by adding all the folders full_path = os.path.join(os.path.dirname(os.path.abspath(__file__))) @@ -30,17 +38,23 @@ def create_schema(spec: SchemaSpecDto) -> None: os.makedirs(full_path) next = folders[idx + 1] if idx + 1 < len(folders) else None if next: - add_import_to_init(full_path, next) + add_import_to_init(full_path, next, code_imports=SCHEMA_CODE_IMPORTS) add_schema_to_init(full_path, spec) def add_schema_to_init(full_path: str, spec: SchemaSpecDto): - init_the_init(full_path) + init_the_init( + full_path, + code_imports=SCHEMA_CODE_IMPORTS + ) init_path = os.path.join(full_path, "__init__.py") with open(init_path, "a") as f: f.write(render_poly_schema(spec) + "\n\n") def render_poly_schema(spec: SchemaSpecDto) -> str: - return SPEC_TEMPLATE.format(name=pascalCase(spec["name"])) \ No newline at end of file + print(spec['name']) + definition = spec["definition"] + return generate_schema_types(definition, root=spec["name"]) + # return FALLBACK_SPEC_TEMPLATE.format(name=spec["name"]) diff --git a/polyapi/typedefs.py b/polyapi/typedefs.py index 1f95c22..3a6d84a 100644 --- a/polyapi/typedefs.py +++ b/polyapi/typedefs.py @@ -60,7 +60,11 @@ class SchemaSpecDto(TypedDict): id: str context: str name: str + contextName: str type: Literal['schema'] + definition: Dict[Any, Any] + visibilityMetadata: object + unresolvedPolySchemaRefs: List # TODO add more diff --git a/polyapi/utils.py b/polyapi/utils.py index 9aa5dab..73d39a3 100644 --- a/polyapi/utils.py +++ b/polyapi/utils.py @@ -14,15 +14,16 @@ CODE_IMPORTS = "from typing import List, Dict, Any, Optional, Callable\nfrom typing_extensions import TypedDict, NotRequired\nimport logging\nimport requests\nimport socketio # type: ignore\nfrom polyapi.config import get_api_key_and_url\nfrom polyapi.execute import execute, execute_post, variable_get, variable_update\n\n" -def init_the_init(full_path: str) -> None: +def init_the_init(full_path: str, code_imports="") -> None: init_path = os.path.join(full_path, "__init__.py") if not os.path.exists(init_path): + code_imports = code_imports or CODE_IMPORTS with open(init_path, "w") as f: - f.write(CODE_IMPORTS) + f.write(code_imports) -def add_import_to_init(full_path: str, next: str) -> None: - init_the_init(full_path) +def add_import_to_init(full_path: str, next: str, code_imports="") -> None: + init_the_init(full_path, code_imports=code_imports) init_path = os.path.join(full_path, "__init__.py") with open(init_path, "a+") as f: diff --git a/pyproject.toml b/pyproject.toml index 0a5384d..6e3491e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,11 +3,11 @@ requires = ["setuptools>=61.2", "wheel"] [project] name = "polyapi-python" -version = "0.3.2" +version = "0.3.3.dev0" description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ - "requests==2.31.0", + "requests>=2.31.0", "typing_extensions>=4.10.0", "jsonschema-gentypes==2.6.0", "pydantic==2.6.4", diff --git a/requirements.txt b/requirements.txt index ee8dbde..47168ef 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ -requests==2.31.0 +requests==2.32.3 typing_extensions>=4.10.0 -jsonschema-gentypes==2.6.0 +jsonschema-gentypes==2.10.0 pydantic==2.6.4 stdlib_list==0.10.0 colorama==0.4.4 From 7a240ddd28101c1ba1d9dda8734bc8713896d5ce Mon Sep 17 00:00:00 2001 From: Dan Fellin Date: Thu, 20 Mar 2025 13:19:32 -0700 Subject: [PATCH 04/14] next --- .flake8 | 4 +-- README.md | 10 +++++++ polyapi/api.py | 1 + polyapi/generate.py | 64 ++++++++++++++++++++++++++++++++++++++--- polyapi/poly_schemas.py | 15 +++++----- polyapi/schema.py | 14 ++++++++- polyapi/utils.py | 11 +++++-- 7 files changed, 102 insertions(+), 17 deletions(-) diff --git a/.flake8 b/.flake8 index 5d7316b..da87de0 100644 --- a/.flake8 +++ b/.flake8 @@ -1,4 +1,4 @@ [flake8] -extend-ignore = E203,E303,E402,E501,E722,W391,F401,W292 +ignore = E203,E303,E402,E501,E722,W391,F401,W292,F811 max-line-length = 150 -max-complexity = 20 +max-complexity = 22 diff --git a/README.md b/README.md index 9eea17b..c929f3f 100644 --- a/README.md +++ b/README.md @@ -143,6 +143,16 @@ To run this library's unit tests, please clone the repo then run: python -m unittest discover ``` +## Linting + +The flake8 config is at the root of this repo at `.flake8`. + +When hacking on this library, please enable flake8 and add this line to your flake8 args (e.g., in your VSCode Workspace Settings): + +``` +--config=.flake8 +``` + ## Support If you run into any issues or want help getting started with this project, please contact support@polyapi.io \ No newline at end of file diff --git a/polyapi/api.py b/polyapi/api.py index 8dc4c85..7ddebc5 100644 --- a/polyapi/api.py +++ b/polyapi/api.py @@ -8,6 +8,7 @@ from typing import List, Dict, Any, TypedDict {args_def} {return_type_def} + class {api_response_type}(TypedDict): status: int headers: Dict diff --git a/polyapi/generate.py b/polyapi/generate.py index 2aec32c..1df5c67 100644 --- a/polyapi/generate.py +++ b/polyapi/generate.py @@ -4,6 +4,7 @@ import shutil from typing import List, cast +from polyapi import schema from polyapi.auth import render_auth_function from polyapi.client import render_client_function from polyapi.poly_schemas import generate_schemas @@ -27,6 +28,15 @@ SUPPORTED_TYPES = SUPPORTED_FUNCTION_TYPES | {"serverVariable", "schema"} +X_POLY_REF_WARNING = '''""" +x-poly-ref: + path:''' + +X_POLY_REF_BETTER_WARNING = '''""" +Unresolved schema, please add the following schema to complete it: + path:''' + + def get_specs() -> List: api_key, api_url = get_api_key_and_url() assert api_key @@ -39,6 +49,40 @@ def get_specs() -> List: raise NotImplementedError(resp.content) +def build_schema_index(items): + index = {} + for item in items: + if item.get("type") == "schema" and "contextName" in item: + index[item["contextName"]] = {**item.get("definition", {}), "name": item.get("name")} + return index + + +def resolve_poly_refs(obj, schema_index): + if isinstance(obj, dict): + if "x-poly-ref" in obj: + ref = obj["x-poly-ref"] + if isinstance(ref, dict) and "path" in ref: + path = ref["path"] + if path in schema_index: + return resolve_poly_refs(schema_index[path], schema_index) + else: + return obj + return {k: resolve_poly_refs(v, schema_index) for k, v in obj.items()} + elif isinstance(obj, list): + return [resolve_poly_refs(item, schema_index) for item in obj] + else: + return obj + + +def replace_poly_refs_in_functions(data: List[SpecificationDto], schema_index): + for item in data: + if item.get("type") == "apiFunction": + func = item.get("function") + if func: + item["function"] = resolve_poly_refs(func, schema_index) + return data + + def parse_function_specs( specs: List[SpecificationDto], limit_ids: List[str] | None, # optional list of ids to limit to @@ -127,7 +171,16 @@ def generate() -> None: remove_old_library() - functions = get_functions_and_parse() + limit_ids: List[str] = [] # useful for narrowing down generation to a single function to debug + functions = get_functions_and_parse(limit_ids=limit_ids) + + schemas = get_schemas() + if schemas: + generate_schemas(schemas) + + schema_index = build_schema_index(schemas) + functions = replace_poly_refs_in_functions(functions, schema_index) + if functions: generate_functions(functions) else: @@ -140,9 +193,6 @@ def generate() -> None: if variables: generate_variables(variables) - schemas = get_schemas() - if schemas: - generate_schemas(schemas) # indicator to vscode extension that this is a polyapi-python project file_path = os.path.join(os.getcwd(), ".polyapi-python") @@ -220,6 +270,12 @@ def render_spec(spec: SpecificationDto): arguments, return_type, ) + + if X_POLY_REF_WARNING in func_type_defs: + # this indicates that jsonschema_gentypes has detected an x-poly-ref + # let's add a more user friendly error explaining what is going on + func_type_defs = func_type_defs.replace(X_POLY_REF_WARNING, X_POLY_REF_BETTER_WARNING) + return func_str, func_type_defs diff --git a/polyapi/poly_schemas.py b/polyapi/poly_schemas.py index 753101f..0d0d224 100644 --- a/polyapi/poly_schemas.py +++ b/polyapi/poly_schemas.py @@ -1,7 +1,7 @@ import os from typing import Any, Dict, List, Tuple -from polyapi.schema import generate_schema_types +from polyapi.schema import wrapped_generate_schema_types from polyapi.utils import add_import_to_init, init_the_init from tests.test_schema import SCHEMA @@ -44,17 +44,18 @@ def create_schema(spec: SchemaSpecDto) -> None: def add_schema_to_init(full_path: str, spec: SchemaSpecDto): - init_the_init( - full_path, - code_imports=SCHEMA_CODE_IMPORTS - ) + init_the_init(full_path, code_imports="") init_path = os.path.join(full_path, "__init__.py") with open(init_path, "a") as f: f.write(render_poly_schema(spec) + "\n\n") def render_poly_schema(spec: SchemaSpecDto) -> str: - print(spec['name']) definition = spec["definition"] - return generate_schema_types(definition, root=spec["name"]) + if not definition.get("type"): + definition["type"] = "object" + root, schema_types = wrapped_generate_schema_types( + definition, root=spec["name"], fallback_type=Dict + ) + return schema_types # return FALLBACK_SPEC_TEMPLATE.format(name=spec["name"]) diff --git a/polyapi/schema.py b/polyapi/schema.py index d99890a..93e0faa 100644 --- a/polyapi/schema.py +++ b/polyapi/schema.py @@ -1,5 +1,7 @@ """ NOTE: this file represents the schema parsing logic for jsonschema_gentypes """ +import random +import string import logging import contextlib from typing import Dict @@ -35,8 +37,18 @@ def _temp_store_input_data(input_data: Dict) -> str: def wrapped_generate_schema_types(type_spec: dict, root, fallback_type): + from polyapi.utils import pascalCase if not root: - root = "MyList" if fallback_type == "List" else "MyDict" + root = "List" if fallback_type == "List" else "Dict" + if type_spec.get("x-poly-ref") and type_spec["x-poly-ref"].get("path"): + # x-poly-ref occurs when we have an unresolved reference + # lets name the root after the reference for some level of visibility + root += pascalCase(type_spec["x-poly-ref"]["path"].replace(".", " ")) + else: + # add three random letters for uniqueness + root += random.choice(string.ascii_letters).upper() + root += random.choice(string.ascii_letters).upper() + root += random.choice(string.ascii_letters).upper() root = clean_title(root) diff --git a/polyapi/utils.py b/polyapi/utils.py index 73d39a3..06dc0fc 100644 --- a/polyapi/utils.py +++ b/polyapi/utils.py @@ -115,15 +115,20 @@ def get_type_and_def(type_spec: PropertyType) -> Tuple[str, str]: elif type_spec["kind"] == "object": if type_spec.get("schema"): schema = type_spec["schema"] - title = schema.get("title", "") + title = schema.get("title", schema.get("name", "")) if title: assert isinstance(title, str) return wrapped_generate_schema_types(schema, title, "Dict") # type: ignore - + elif schema.get("allOf") and len(schema['allOf']): + # we are in a case of a single allOf, lets strip off the allOf and move on! + # our library doesn't handle allOf well yet + allOf = schema['allOf'][0] + title = allOf.get("title", allOf.get("name", "")) + return wrapped_generate_schema_types(allOf, title, "Dict") elif schema.get("items"): # fallback to schema $ref name if no explicit title items = schema.get("items") # type: ignore - title = items.get("title", "") # type: ignore + title = items.get("title") # type: ignore if not title: # title is actually a reference to another schema title = items.get("$ref", "") # type: ignore From 8b68dca793636b62710d0f382a345fe84ede3b25 Mon Sep 17 00:00:00 2001 From: Dan Fellin Date: Thu, 20 Mar 2025 13:29:14 -0700 Subject: [PATCH 05/14] next --- pyproject.toml | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6e3491e..3e3549d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ version = "0.3.3.dev0" description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers" authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }] dependencies = [ - "requests>=2.31.0", + "requests>=2.32.3", "typing_extensions>=4.10.0", "jsonschema-gentypes==2.6.0", "pydantic==2.6.4", diff --git a/requirements.txt b/requirements.txt index 47168ef..d967c75 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -requests==2.32.3 +requests>=2.32.3 typing_extensions>=4.10.0 jsonschema-gentypes==2.10.0 pydantic==2.6.4 From 2cd5c7053ec68579cd59d0f627152f185d4c31ba Mon Sep 17 00:00:00 2001 From: Dan Fellin Date: Fri, 21 Mar 2025 11:30:03 -0700 Subject: [PATCH 06/14] next --- polyapi/generate.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/polyapi/generate.py b/polyapi/generate.py index 1df5c67..c7beef3 100644 --- a/polyapi/generate.py +++ b/polyapi/generate.py @@ -74,13 +74,26 @@ def resolve_poly_refs(obj, schema_index): return obj -def replace_poly_refs_in_functions(data: List[SpecificationDto], schema_index): - for item in data: - if item.get("type") == "apiFunction": - func = item.get("function") +def replace_poly_refs_in_functions(specs: List[SpecificationDto], schema_index): + spec_idxs_to_remove = [] + for idx, spec in enumerate(specs): + if spec.get("type") == "apiFunction": + func = spec.get("function") if func: - item["function"] = resolve_poly_refs(func, schema_index) - return data + try: + spec["function"] = resolve_poly_refs(func, schema_index) + except Exception: + print() + print(f"{spec['context']}.{spec['name']} (id: {spec['id']}) failed to resolve poly refs, skipping!") + spec_idxs_to_remove.append(idx) + + # reverse the list so we pop off later indexes first + spec_idxs_to_remove.reverse() + + for idx in spec_idxs_to_remove: + specs.pop(idx) + + return specs def parse_function_specs( From 2da5f856df667c4ebe44e712f01e47e2e0a79dbf Mon Sep 17 00:00:00 2001 From: Dan Fellin Date: Fri, 21 Mar 2025 12:10:06 -0700 Subject: [PATCH 07/14] next --- polyapi/generate.py | 2 ++ polyapi/schema.py | 10 +++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/polyapi/generate.py b/polyapi/generate.py index c7beef3..d1b63da 100644 --- a/polyapi/generate.py +++ b/polyapi/generate.py @@ -86,6 +86,8 @@ def replace_poly_refs_in_functions(specs: List[SpecificationDto], schema_index): print() print(f"{spec['context']}.{spec['name']} (id: {spec['id']}) failed to resolve poly refs, skipping!") spec_idxs_to_remove.append(idx) + # if 'mews' in spec['context']: + # spec_idxs_to_remove(spec) # reverse the list so we pop off later indexes first spec_idxs_to_remove.reverse() diff --git a/polyapi/schema.py b/polyapi/schema.py index 93e0faa..4477745 100644 --- a/polyapi/schema.py +++ b/polyapi/schema.py @@ -7,9 +7,12 @@ from typing import Dict from jsonschema_gentypes.cli import process_config from jsonschema_gentypes import configuration +import referencing import tempfile import json +import referencing.exceptions + from polyapi.constants import JSONSCHEMA_TO_PYTHON_TYPE_MAP @@ -58,8 +61,13 @@ def wrapped_generate_schema_types(type_spec: dict, root, fallback_type): # some schemas are so huge, our library cant handle it # TODO identify critical recursion penalty and maybe switch underlying logic to iterative? return fallback_type, "" + except referencing.exceptions.CannotDetermineSpecification: + # just go with fallback_type here + # we couldn't match the right $ref earlier in resolve_poly_refs + # {'$ref': '#/definitions/FinanceAccountListModel'} + return fallback_type, "" except: - logging.exception(f"Error when generating schema type: {type_spec}") + logging.error(f"Error when generating schema type: {type_spec}\nusing fallback type '{fallback_type}'") return fallback_type, "" From e6afa69b790b1e76a7e2a1261e17c15b7611eb39 Mon Sep 17 00:00:00 2001 From: Dan Fellin Date: Fri, 21 Mar 2025 12:30:08 -0700 Subject: [PATCH 08/14] test --- tests/test_generate.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 tests/test_generate.py diff --git a/tests/test_generate.py b/tests/test_generate.py new file mode 100644 index 0000000..ecc7d05 --- /dev/null +++ b/tests/test_generate.py @@ -0,0 +1,14 @@ +import unittest +from polyapi.utils import get_type_and_def, rewrite_reserved + +OPENAPI_FUNCTION = {'kind': 'function', 'spec': {'arguments': [{'name': 'event', 'required': False, 'type': {'kind': 'object', 'schema': {'$schema': 'http://json-schema.org/draft-06/schema#', 'type': 'array', 'items': {'$ref': '#/definitions/WebhookEventTypeElement'}, 'definitions': {'WebhookEventTypeElement': {'type': 'object', 'additionalProperties': False, 'properties': {'title': {'type': 'string'}, 'manufacturerName': {'type': 'string'}, 'carType': {'type': 'string'}, 'id': {'type': 'integer'}}, 'required': ['carType', 'id', 'manufacturerName', 'title'], 'title': 'WebhookEventTypeElement'}}}}}, {'name': 'headers', 'required': False, 'type': {'kind': 'object', 'typeName': 'Record'}}, {'name': 'params', 'required': False, 'type': {'kind': 'object', 'typeName': 'Record'}}, {'name': 'polyCustom', 'required': False, 'type': {'kind': 'object', 'properties': [{'name': 'responseStatusCode', 'type': {'type': 'number', 'kind': 'primitive'}, 'required': True}, {'name': 'responseContentType', 'type': {'type': 'string', 'kind': 'primitive'}, 'required': True, 'nullable': True}]}}], 'returnType': {'kind': 'void'}, 'synchronous': True}} + + +class T(unittest.TestCase): + def test_get_type_and_def(self): + arg_type, arg_def = get_type_and_def(OPENAPI_FUNCTION) + self.assertEqual(arg_type, "Callable[[List[WebhookEventTypeElement], Dict, Dict, Dict], None]") + + def test_rewrite_reserved(self): + rv = rewrite_reserved("from") + self.assertEqual(rv, "_from") \ No newline at end of file From 80466475e0e6ceaaa75f5fb04f6e32bbf2756ba2 Mon Sep 17 00:00:00 2001 From: Dan Fellin Date: Mon, 24 Mar 2025 07:40:51 -0700 Subject: [PATCH 09/14] next --- polyapi/generate.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/polyapi/generate.py b/polyapi/generate.py index d1b63da..c7beef3 100644 --- a/polyapi/generate.py +++ b/polyapi/generate.py @@ -86,8 +86,6 @@ def replace_poly_refs_in_functions(specs: List[SpecificationDto], schema_index): print() print(f"{spec['context']}.{spec['name']} (id: {spec['id']}) failed to resolve poly refs, skipping!") spec_idxs_to_remove.append(idx) - # if 'mews' in spec['context']: - # spec_idxs_to_remove(spec) # reverse the list so we pop off later indexes first spec_idxs_to_remove.reverse() From 3e58f94a40acbd7eb8e0f7fccb972c6a5e4d72d9 Mon Sep 17 00:00:00 2001 From: Dan Fellin Date: Mon, 24 Mar 2025 08:19:19 -0700 Subject: [PATCH 10/14] next --- polyapi/generate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polyapi/generate.py b/polyapi/generate.py index c7beef3..6092c57 100644 --- a/polyapi/generate.py +++ b/polyapi/generate.py @@ -25,7 +25,7 @@ "webhookHandle", } -SUPPORTED_TYPES = SUPPORTED_FUNCTION_TYPES | {"serverVariable", "schema"} +SUPPORTED_TYPES = SUPPORTED_FUNCTION_TYPES | {"serverVariable", "schema", "snippet"} X_POLY_REF_WARNING = '''""" From 69ced1733fc87f278af34986813cc8611d71072c Mon Sep 17 00:00:00 2001 From: Dan Fellin Date: Mon, 24 Mar 2025 08:43:17 -0700 Subject: [PATCH 11/14] next --- polyapi/function_cli.py | 7 +++++-- polyapi/generate.py | 13 +++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/polyapi/function_cli.py b/polyapi/function_cli.py index f7fd677..1256075 100644 --- a/polyapi/function_cli.py +++ b/polyapi/function_cli.py @@ -1,7 +1,7 @@ import sys from typing import Any, List, Optional import requests -from polyapi.generate import get_functions_and_parse, generate_functions +from polyapi.generate import cache_specs, generate_functions, get_specs, parse_function_specs from polyapi.config import get_api_key_and_url from polyapi.utils import get_auth_headers, print_green, print_red, print_yellow from polyapi.parser import parse_function_code, get_jsonschema_type @@ -88,7 +88,10 @@ def function_add_or_update( print(f"Function ID: {function_id}") if generate: print("Generating new custom function...", end="") - functions = get_functions_and_parse(limit_ids=[function_id]) + # TODO do something more efficient here rather than regetting ALL the specs again + specs = get_specs() + cache_specs(specs) + functions = parse_function_specs(specs) generate_functions(functions) print_green("DONE") else: diff --git a/polyapi/generate.py b/polyapi/generate.py index 6092c57..cb82a13 100644 --- a/polyapi/generate.py +++ b/polyapi/generate.py @@ -98,7 +98,7 @@ def replace_poly_refs_in_functions(specs: List[SpecificationDto], schema_index): def parse_function_specs( specs: List[SpecificationDto], - limit_ids: List[str] | None, # optional list of ids to limit to + limit_ids: List[str] | None = None, # optional list of ids to limit to ) -> List[SpecificationDto]: functions = [] for spec in specs: @@ -149,12 +149,6 @@ def read_cached_specs() -> List[SpecificationDto]: return json.loads(f.read()) -def get_functions_and_parse(limit_ids: List[str] | None = None) -> List[SpecificationDto]: - specs = get_specs() - cache_specs(specs) - return parse_function_specs(specs, limit_ids=limit_ids) - - def get_variables() -> List[VariableSpecDto]: specs = read_cached_specs() return [cast(VariableSpecDto, spec) for spec in specs if spec["type"] == "serverVariable"] @@ -185,7 +179,10 @@ def generate() -> None: remove_old_library() limit_ids: List[str] = [] # useful for narrowing down generation to a single function to debug - functions = get_functions_and_parse(limit_ids=limit_ids) + + specs = get_specs() + cache_specs(specs) + functions = parse_function_specs(specs, limit_ids=limit_ids) schemas = get_schemas() if schemas: From 63e1a282101232a494a52708363fc9e32576ecc0 Mon Sep 17 00:00:00 2001 From: Dan Fellin Date: Mon, 24 Mar 2025 10:23:30 -0700 Subject: [PATCH 12/14] little tweak for A-Aron --- polyapi/generate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polyapi/generate.py b/polyapi/generate.py index cb82a13..9a00742 100644 --- a/polyapi/generate.py +++ b/polyapi/generate.py @@ -77,7 +77,7 @@ def resolve_poly_refs(obj, schema_index): def replace_poly_refs_in_functions(specs: List[SpecificationDto], schema_index): spec_idxs_to_remove = [] for idx, spec in enumerate(specs): - if spec.get("type") == "apiFunction": + if spec.get("type") in ("apiFunction", "clientFunction", "serverFunction"): func = spec.get("function") if func: try: From 54ab9b51b4f99aec76e94bdaa8183f4d4f9c1169 Mon Sep 17 00:00:00 2001 From: Dan Fellin Date: Mon, 24 Mar 2025 10:25:33 -0700 Subject: [PATCH 13/14] fix --- polyapi/generate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polyapi/generate.py b/polyapi/generate.py index 9a00742..a3626c1 100644 --- a/polyapi/generate.py +++ b/polyapi/generate.py @@ -77,7 +77,7 @@ def resolve_poly_refs(obj, schema_index): def replace_poly_refs_in_functions(specs: List[SpecificationDto], schema_index): spec_idxs_to_remove = [] for idx, spec in enumerate(specs): - if spec.get("type") in ("apiFunction", "clientFunction", "serverFunction"): + if spec.get("type") in ("apiFunction", "customFunction", "serverFunction"): func = spec.get("function") if func: try: From 1d38472d834135ffa80ebacfafccef2a01d9d211 Mon Sep 17 00:00:00 2001 From: Dan Fellin Date: Mon, 24 Mar 2025 10:26:05 -0700 Subject: [PATCH 14/14] next --- polyapi/generate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polyapi/generate.py b/polyapi/generate.py index a3626c1..78d0e07 100644 --- a/polyapi/generate.py +++ b/polyapi/generate.py @@ -20,7 +20,7 @@ SUPPORTED_FUNCTION_TYPES = { "apiFunction", "authFunction", - "customFunction", + "customFunction", # client function - this is badly named in /specs atm "serverFunction", "webhookHandle", }