Skip to content
71 changes: 69 additions & 2 deletions docs/describing_models.rst
Original file line number Diff line number Diff line change
Expand Up @@ -295,12 +295,79 @@ look in ``models/`` for a file named ``macro_micro.ymmsl`` and load a program or
named ``macro_micro`` from it. Once imported, ``uq_driver`` and ``macro_micro`` can then
be used as implementation of a component.

To find files, ymmsl-python will inspect the ``YMMSLPATH`` environment variable for
To find files, ymmsl-python will look in two places:

1. Installed Python packages can provide "Entry Points" to provide importable yMMSL
components.
2. The ``YMMSL_PATH`` environment variable can contain directories with importable yMMSL
files.

These mechanisms are described in more detail below.


YMMSL Path
^^^^^^^^^^

Ymmsl-python will inspect the ``YMMSL_PATH`` environment variable for
directories to search. This should contain one or more colon-separated paths pointing to
directories with yMMSL files, in the same way that ``PATH`` points to directories with
executables and ``PYTHONPATH`` to directories with Python files to be imported.

For example, if ``YMMSLPATH`` equals ``/home/user/ymmsl:/home/user/my_project`` then the
Comment thread
maarten-ic marked this conversation as resolved.
For example, if ``YMMSL_PATH`` equals ``/home/user/ymmsl:/home/user/my_project`` then the
first import statement would first look for ``/home/user/ymmsl/utils/uq.ymmsl`` and then
for ``/home/user/my_project/utils/uq.ymmsl`` if that did not exist.


Python Entry Points
^^^^^^^^^^^^^^^^^^^

Installed Python packages can provide entry points for ymmsl-python to advertise that
they provide importable yMMSL components.
For example, the first import statement above would look for an entry point named
``utils.uq`` and try to load that configuration.

If you are a developer of a Python package and want to use the entry point mechanism so
users can import your component, you will need to:

1. Configure the entry point in your ``pyproject.toml`` (or ``setup.py``) file.
2. Provide the yMMSL configuration as a string inside your python distribution.

Below code listings provide an example how to do this.

.. code-block:: toml
:caption: Entry point configuration in ``pyproject.toml``

# Indicate you want to provide an entry point for "ymmsl.module":
[project.entry-points."ymmsl.module"]
# Provide one or more "name = value" entries, pointing to a valid yMMSL
# configuration string (see next code listing). For more details, see
# https://setuptools.pypa.io/en/latest/userguide/entry_point.html#entry-points-syntax
"utils.uq" = "my_package.utils.uq:YMMSL_CONFIG"

.. code-block:: python
:caption: yMMSL configuration string in ``my_package/utils/uq.py``

import sys

YMMSL_CONFIG = f"""
ymmsl_version: v0.2
description: Uncertainty Quantification utilities from my_package
programs:
uq_driver:
description: |
This component creates a sample of initial states for the simulation, then
sends them on initial_state-out to the model to be run. It then collects the
final state for analysis on final_state_in.
executable: {sys.executable}
args: -m my_package.utils.uq
ports:
o_i: initial_state_out
s: final_state_in
"""


.. seealso::
- User guide on Entry Points from setuptools:
https://setuptools.pypa.io/en/latest/userguide/entry_point.html
- The Entry Points specification:
https://packaging.python.org/en/latest/specifications/entry-points/
1 change: 1 addition & 0 deletions setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
test_suite='tests',
install_requires=[
'click>=6.5',
'importlib-metadata; python_version < "3.10"',
'yatiml>=0.12.0,<0.13.0',
# 'yatiml @ git+https://github.com/yatiml/yatiml@develop#egg=yatiml'
],
Expand Down
120 changes: 91 additions & 29 deletions ymmsl/v0_2/resolver.py
Original file line number Diff line number Diff line change
@@ -1,19 +1,33 @@
import logging
import os
import sys
from collections.abc import MutableMapping
from difflib import get_close_matches
import os
from pathlib import Path
from textwrap import indent
from typing import Dict, List, Tuple, TypeVar
from typing import Dict, List, Optional, Tuple, TypeVar, Union

from yatiml import RecognitionError

import ymmsl
from ymmsl.v0_2.configuration import Configuration
from ymmsl.v0_2.identity import Reference
from ymmsl.v0_2.imports import ImportKind, ImportStatement
from ymmsl.v0_2.implementation import Implementation
from ymmsl.v0_2.imports import ImportKind, ImportStatement
from ymmsl.v0_2.model import Model
Comment thread
IrisvdWerf marked this conversation as resolved.
from ymmsl.v0_2.program import Program

if sys.version_info < (3, 10):
from importlib_metadata import EntryPoint, entry_points
else:
from importlib.metadata import EntryPoint, entry_points

_logger = logging.getLogger(__name__)


ModuleSource = Union[Path, EntryPoint]
"""Source file (Path) or EntryPoint for a yMMSL module"""


class ResolutionContext:
"""Keeps track of where in the resolution process we are.
Expand All @@ -29,17 +43,17 @@ def __init__(self) -> None:
else:
self.ymmsl_path = list()

# _files is a list of (file_path, module_name)]
self._files: List[Tuple[Path, Reference]] = list()
# _modules is a list of (file_path or entry_point, module_name)
self._modules: List[Tuple[ModuleSource, Reference]] = list()
self._imports: List[ImportStatement] = list()

def push_file(self, file: Path, module: Reference) -> None:
def push_module(self, file: ModuleSource, module: Reference) -> None:
"""Push a file/module onto the stack."""
self._files.append((file, module))
self._modules.append((file, module))

def pop_file(self) -> None:
def pop_module(self) -> None:
"""Pop the topmost file/module off of the stack."""
self._files.pop()
self._modules.pop()

def push_import(self, imp_st: ImportStatement) -> None:
"""Push an import statement onto the stack."""
Expand All @@ -56,7 +70,7 @@ def trace(self) -> str:

result: List[str] = list()

for i, (file_path, _) in enumerate(self._files):
for i, (file_path, _) in enumerate(self._modules):
if i < len(self._imports):
imp_mod = self._imports[i].module
imp_name = self._imports[i].name
Expand Down Expand Up @@ -93,7 +107,8 @@ def resolve(module: Reference, config: Configuration) -> None:


def do_resolve(
file: Path, module: Reference, config: Configuration, ctx: ResolutionContext
file: ModuleSource, module: Reference, config: Configuration,
ctx: ResolutionContext
) -> None:
"""Implementation of resolve().

Expand All @@ -106,9 +121,9 @@ def do_resolve(
module: The module corresponding to this configuration
config: The configuration to resolve
"""
ctx.push_file(file, module)
ctx.push_module(file, module)
resolve_impls(module, config, ctx)
ctx.pop_file()
ctx.pop_module()


def resolve_impls(
Expand Down Expand Up @@ -159,10 +174,10 @@ def resolve_impl_imports(
for imp_st in config.imports:
ctx.push_import(imp_st)
if imp_st.kind == ImportKind.IMPLEMENTATION:
imp_cfg, loaded_file = load_resolve_file(
imp_cfg, loaded_file = load_resolve_module(
imp_st.module, imp_st.module_path(), ctx)

ctx.push_file(loaded_file, imp_st.module)
ctx.push_module(loaded_file, imp_st.module)

impls = find_impls(imp_cfg, imp_st.full_name(), ctx)
for impl in impls:
Expand All @@ -180,7 +195,7 @@ def resolve_impl_imports(

ylocals[Reference([imp_st.name])] = imp_st.full_name()

ctx.pop_file()
ctx.pop_module()

ctx.pop_import()

Expand Down Expand Up @@ -243,12 +258,58 @@ def find_impl(
raise RuntimeError(msg)


ymmsl_cache: Dict[Path, Tuple[Configuration, Path]] = dict()
ymmsl_cache: Dict[Path, Tuple[Configuration, ModuleSource]] = dict()


def _load_from_entrypoints(
module: Reference) -> Optional[Tuple[Configuration, EntryPoint]]:
# Find entry point
entrypoints = entry_points(group="ymmsl.module", name=str(module))
if not entrypoints:
return None
if len(entrypoints) > 1:
_logger.warning(
Comment thread
maarten-ic marked this conversation as resolved.
"Multiple entry points found for yMMSL import module '%s'. "
"Taking the first of %s",
module,
", ".join(
f"'{ep.value}' (from {ep.dist.name if ep.dist else '<unknown>'})"
for ep in entrypoints
)
)
entrypoint = next(iter(entrypoints))

# Load entry point
try:
ymmsl_txt = entrypoint.load()
except Exception as exc:
raise RuntimeError(
f"Error while loading the entrypoint '{entrypoint.value}' "
f"(from {entrypoint.dist.name if entrypoint.dist else '<unknown>'})"
) from exc
if not isinstance(ymmsl_txt, str):
raise TypeError(f"Entrypoint {entrypoint.value} is not a string")

config = ymmsl.load_as(Configuration, ymmsl_txt)
return config, entrypoint


def _load_from_ymmsl_path(
module_path: Path, ymmsl_path: list[Path]
) -> Optional[Tuple[Configuration, Path]]:
for yp in ymmsl_path:
try:
loaded_file = yp / module_path
config = ymmsl.load_as(Configuration, loaded_file)
return config, loaded_file
except FileNotFoundError:
pass
return None


def load_resolve_file(
def load_resolve_module(
module: Reference, module_path: Path, ctx: ResolutionContext
) -> Tuple[Configuration, Path]:
) -> Tuple[Configuration, ModuleSource]:
"""Load and resolve an imported ymmsl file.

Caches the result, without functools.lru_cache because ctx shouldn't hash.
Expand All @@ -262,21 +323,22 @@ def load_resolve_file(
if module_path not in ymmsl_cache:
# TODO: relative to working directory?
try:
for yp in ctx.ymmsl_path:
try:
loaded_file = yp / module_path
config = ymmsl.load_as(Configuration, loaded_file)
break
except FileNotFoundError:
pass
else:
config_and_loaded_file = (
_load_from_entrypoints(module)
or _load_from_ymmsl_path(module_path, ctx.ymmsl_path)
)
if config_and_loaded_file is None:
msg = ctx.trace()
msg += f' Failed to find a file {module_path} for module {module}.\n'
msg += ' Based on the YMMSL_PATH environment variable, I\'ve'
msg += ' searched at:\n'
msg += ' Based on the YMMSL_PATH environment variable and Python'
msg += ' entry points. I\'ve searched at:\n'
msg += ctx.search_paths(module_path, 8)
msg += '\n and in entry points:\n'
msg += '\n'.join(
8*' ' + ep.name for ep in entry_points(group="ymmsl.module"))
raise RuntimeError(msg)

config, loaded_file = config_and_loaded_file
do_resolve(loaded_file, module, config, ctx)
ymmsl_cache[module_path] = config, loaded_file

Expand Down
Loading
Loading