From 48fbaba0c9a74a3fe03c618fc3fccb68f3ae2248 Mon Sep 17 00:00:00 2001 From: Jeff Allen Date: Fri, 30 Dec 2022 16:55:27 +0000 Subject: [PATCH 1/6] Rename PyJavaMethod to PyJavaFunction PyJavaMethod represents bound and unbound *methods* (name 'builtin_function_or_method'). CPython calls the same idea PyCFunction. PEP 573 introduces a PyCMethod (name 'builtin_method') for methods of types in modules. It's going to be confusing if we don't go with PyJavaFunction. --- .../main/java/org/python/core/Exposer.java | 2 +- .../org/python/core/MethodDescriptor.java | 2 +- .../java/org/python/core/MethodSignature.java | 2 +- ...{PyJavaMethod.java => PyJavaFunction.java} | 29 ++++++++++--------- .../java/org/python/core/PyMethodDescr.java | 10 +++---- .../python/core/TypeExposerMethodTest.java | 4 +-- 6 files changed, 25 insertions(+), 24 deletions(-) rename core/src/main/java/org/python/core/{PyJavaMethod.java => PyJavaFunction.java} (95%) diff --git a/core/src/main/java/org/python/core/Exposer.java b/core/src/main/java/org/python/core/Exposer.java index c072579f8..4ba381f15 100644 --- a/core/src/main/java/org/python/core/Exposer.java +++ b/core/src/main/java/org/python/core/Exposer.java @@ -1166,7 +1166,7 @@ static class StaticMethodSpec extends CallableSpec { StaticMethodSpec(String name, ScopeKind scopeKind) { super(name, scopeKind); } @Override - PyJavaMethod asAttribute(PyType objclass, Lookup lookup) { + PyJavaFunction asAttribute(PyType objclass, Lookup lookup) { // TODO Auto-generated method stub return null; } diff --git a/core/src/main/java/org/python/core/MethodDescriptor.java b/core/src/main/java/org/python/core/MethodDescriptor.java index 760a77926..5c652bf6e 100644 --- a/core/src/main/java/org/python/core/MethodDescriptor.java +++ b/core/src/main/java/org/python/core/MethodDescriptor.java @@ -9,7 +9,7 @@ * Java. This class provides some common behaviour and support * methods that would otherwise be duplicated. This is also home to * some static methods in support of both sub-classes and other - * callable objects (e.g. {@link PyJavaMethod}). + * callable objects (e.g. {@link PyJavaFunction}). */ abstract class MethodDescriptor extends Descriptor implements FastCall { diff --git a/core/src/main/java/org/python/core/MethodSignature.java b/core/src/main/java/org/python/core/MethodSignature.java index 5766cdb96..7767e415c 100644 --- a/core/src/main/java/org/python/core/MethodSignature.java +++ b/core/src/main/java/org/python/core/MethodSignature.java @@ -17,7 +17,7 @@ /** * The {@code enum MethodSignature} enumerates the method signatures * for which an optimised implementation is possible. Sub-classes of - * {@link PyJavaMethod} and {@link PyMethodDescr} correspond to + * {@link PyJavaFunction} and {@link PyMethodDescr} correspond to * these values. It is not required that each value have a distinct * optimised sub-class. This {@code enum} is used internally to * choose between these sub-classes. diff --git a/core/src/main/java/org/python/core/PyJavaMethod.java b/core/src/main/java/org/python/core/PyJavaFunction.java similarity index 95% rename from core/src/main/java/org/python/core/PyJavaMethod.java rename to core/src/main/java/org/python/core/PyJavaFunction.java index 2b861dcd4..d3083994c 100644 --- a/core/src/main/java/org/python/core/PyJavaMethod.java +++ b/core/src/main/java/org/python/core/PyJavaFunction.java @@ -15,7 +15,7 @@ * sub-classes represent either a built-in function or a built-in * method bound to a particular object. */ -public abstract class PyJavaMethod implements CraftedPyObject, FastCall { +public abstract class PyJavaFunction implements CraftedPyObject, FastCall { /** The type of Python object this class implements. */ static final PyType TYPE = PyType.fromSpec( // @@ -40,14 +40,14 @@ public abstract class PyJavaMethod implements CraftedPyObject, FastCall { /** * A Java {@code MethodHandle} that implements the function or bound * method. The type of this handle varies according to the sub-class - * of {@code PyJavaMethod}, but it is definitely "prepared" to + * of {@code PyJavaFunction}, but it is definitely "prepared" to * accept {@code Object.class} instances or arrays, not the actual * parameter types of the method definition in Java. */ final MethodHandle handle; /** - * An argument parser supplied to this {@code PyJavaMethod} at + * An argument parser supplied to this {@code PyJavaFunction} at * construction, from Java reflection of the definition in Java and * from annotations on it. Full information on the signature is * available from this structure, and it is available to parse the @@ -72,7 +72,7 @@ public abstract class PyJavaMethod implements CraftedPyObject, FastCall { * method) * @param module name of the module supplying the definition */ - protected PyJavaMethod(ArgParser argParser, MethodHandle handle, Object self, String module) { + protected PyJavaFunction(ArgParser argParser, MethodHandle handle, Object self, String module) { this.argParser = argParser; this.handle = handle; this.self = self; @@ -80,7 +80,7 @@ protected PyJavaMethod(ArgParser argParser, MethodHandle handle, Object self, St } /** - * Construct a {@code PyJavaMethod} from an {@link ArgParser} and + * Construct a {@code PyJavaFunction} from an {@link ArgParser} and * {@code MethodHandle} for the implementation method. The arguments * described by the parser do not include "self". This is the * factory we use to create a function in a module. @@ -94,7 +94,8 @@ protected PyJavaMethod(ArgParser argParser, MethodHandle handle, Object self, St * @return A method descriptor supporting the signature */ // Compare CPython PyCFunction_NewEx in methodobject.c - static PyJavaMethod fromParser(ArgParser ap, MethodHandle method, Object self, String module) { + static PyJavaFunction fromParser(ArgParser ap, MethodHandle method, Object self, + String module) { /* * Note this is a recommendation on the assumption all optimisations * are supported. The actual choice is made in the switch statement. @@ -130,11 +131,11 @@ static PyJavaMethod fromParser(ArgParser ap, MethodHandle method, Object self, S } /** - * Construct a {@code PyJavaMethod} from a {@link PyMethodDescr} and - * optional object to bind. The {@link PyMethodDescr} provides the - * parser and unbound prepared {@code MethodHandle}. The arguments - * described by the parser do not include "self". This is the - * factory that supports descriptor {@code __get__}. + * Construct a {@code PyJavaFunction} from a {@link PyMethodDescr} + * and optional object to bind. The {@link PyMethodDescr} provides + * the parser and unbound prepared {@code MethodHandle}. The + * arguments described by the parser do not include "self". This is + * the factory that supports descriptor {@code __get__}. * * @param descr descriptor being bound * @param self object to which bound (or {@code null} if a static @@ -145,7 +146,7 @@ static PyJavaMethod fromParser(ArgParser ap, MethodHandle method, Object self, S * @throws Throwable on other errors while chasing the MRO */ // Compare CPython PyCFunction_NewEx in methodobject.c - static PyJavaMethod from(PyMethodDescr descr, Object self) throws TypeError, Throwable { + static PyJavaFunction from(PyMethodDescr descr, Object self) throws TypeError, Throwable { ArgParser ap = descr.argParser; assert ap.methodKind == MethodKind.INSTANCE; MethodHandle handle = descr.getHandle(self).bindTo(self); @@ -255,7 +256,7 @@ public TypeError typeError(ArgumentError ae, Object[] args, String[] names) { * The implementation may have any signature allowed by * {@link ArgParser}. */ - private static class General extends PyJavaMethod { + private static class General extends PyJavaFunction { /** * Construct a method object, identifying the implementation by a @@ -302,7 +303,7 @@ public Object call(Object[] args, String[] names) throws TypeError, Throwable { * @ImplNote Sub-classes must define {@link #call(Object[])}: the * default definition in {@link FastCall} is not enough. */ - private static abstract class AbstractPositional extends PyJavaMethod { + private static abstract class AbstractPositional extends PyJavaFunction { /** Default values of the trailing arguments. */ protected final Object[] d; diff --git a/core/src/main/java/org/python/core/PyMethodDescr.java b/core/src/main/java/org/python/core/PyMethodDescr.java index 867dbcfcc..7f5cf6244 100644 --- a/core/src/main/java/org/python/core/PyMethodDescr.java +++ b/core/src/main/java/org/python/core/PyMethodDescr.java @@ -17,7 +17,7 @@ * from Python. A {@code PyMethodDescr} is a callable object itself, * and provides binding behaviour through * {@link #__get__(Object, PyType) __get__}, which usually creates a - * {@link PyJavaMethod}. + * {@link PyJavaFunction}. *

* It suits us to sub-class {@code PyMethodDescr} to express the * multiplicity of implementations and to respond to the signature @@ -70,8 +70,8 @@ abstract class PyMethodDescr extends MethodDescriptor { /** * Deduced method signature (useful to have cached when constructing - * a {@link PyJavaMethod}). Note that this is allowed to differ from - * {@link MethodSignature#fromParser(ArgParser) + * a {@link PyJavaFunction}). Note that this is allowed to differ + * from {@link MethodSignature#fromParser(ArgParser) * MethodSignature.fromParser(argParser)}. */ final MethodSignature signature; @@ -376,7 +376,7 @@ Object simple__call__(Object[] args, String[] names) throws TypeError, Throwable /** * Return the described method, bound to {@code obj} as its "self" * argument, or if {@code obj==null}, return this descriptor. In the - * non-null case, {@code __get__} returns a {@link PyJavaMethod}. + * non-null case, {@code __get__} returns a {@link PyJavaFunction}. * Calling the returned object invokes the same Java method as this * descriptor, with {@code obj} as first argument, and other * arguments to the call appended. @@ -396,7 +396,7 @@ Object __get__(Object obj, PyType type) throws TypeError, Throwable { else { // Return a callable binding the method and the target check(obj); - return PyJavaMethod.from(this, obj); + return PyJavaFunction.from(this, obj); } } diff --git a/core/src/test/java/org/python/core/TypeExposerMethodTest.java b/core/src/test/java/org/python/core/TypeExposerMethodTest.java index f19986c3a..d06c6cbcf 100644 --- a/core/src/test/java/org/python/core/TypeExposerMethodTest.java +++ b/core/src/test/java/org/python/core/TypeExposerMethodTest.java @@ -46,7 +46,7 @@ abstract static class Standard { /** The object on which to invoke the method. */ Object obj; /** The function to examine or call (bound to {@code obj}). */ - PyJavaMethod func; + PyJavaFunction func; /** The parser we examine. */ ArgParser ap; /** The expected result of calling the method. */ @@ -157,7 +157,7 @@ void setup(String name, Object o) throws AttributeError, Throwable { descr = (PyMethodDescr)PyType.of(o).lookup(name); ap = descr.argParser; obj = o; - func = (PyJavaMethod)Abstract.getAttr(obj, name); + func = (PyJavaFunction)Abstract.getAttr(obj, name); } /** From e81569384db18643a17fb9c548b7b73630efd0ab Mon Sep 17 00:00:00 2001 From: Jeff Allen Date: Fri, 30 Dec 2022 19:43:56 +0000 Subject: [PATCH 2/6] Introduce the ModuleExposer and a test Port the basic apparatus of modules defined in Java from VSJ and a test. Rationalise ArgParser to eliminate a legacy constructor. --- .../main/java/org/python/core/ArgParser.java | 409 ++++++++---------- .../main/java/org/python/core/Exposer.java | 53 ++- .../main/java/org/python/core/JavaModule.java | 23 + .../main/java/org/python/core/ModuleDef.java | 157 +++++++ .../java/org/python/core/ModuleExposer.java | 88 ++++ .../java/org/python/core/ArgParserTest.java | 109 +++-- .../org/python/core/ModuleExposerTest.java | 220 ++++++++++ 7 files changed, 799 insertions(+), 260 deletions(-) create mode 100644 core/src/main/java/org/python/core/JavaModule.java create mode 100644 core/src/main/java/org/python/core/ModuleDef.java create mode 100644 core/src/main/java/org/python/core/ModuleExposer.java create mode 100644 core/src/test/java/org/python/core/ModuleExposerTest.java diff --git a/core/src/main/java/org/python/core/ArgParser.java b/core/src/main/java/org/python/core/ArgParser.java index 652ed2c2e..563be1680 100644 --- a/core/src/main/java/org/python/core/ArgParser.java +++ b/core/src/main/java/org/python/core/ArgParser.java @@ -5,7 +5,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.StringJoiner; @@ -25,26 +24,78 @@ * in a Python implementation, and arranges them into an array. This * array is either created by the parser, or designated by the * caller. The parser may therefore be used to prepare arguments for - * a pure a Java method (or {@code MethodHandle}) accepting an - * array, or to insert them as initial values in an interpreter - * frame ({@code PyFrame}). + * a pure a Java method (or {@code MethodHandle}) that accepts an + * array, or to insert arguments as initial values of local + * variables in an an optimised interpreter frame ({@link PyFrame}). *

* The fields of the parser that determine the acceptable numbers of * positional arguments and their names are essentially those of a * {@code code} object ({@link PyCode}). Defaults are provided * values that mirror the defaults built into a {@code function} - * object ({@code PyFunction}). + * object ({@link PyFunction}). + *

+ * Consider for example a function that in Python would have the + * function definition:

+ * def func(a, b, c=3, d=4, /, e=5, f=6, *aa, g=7, h, i=9, **kk):
+ *     pass
+ *
This could be described by a constructor call and + * modifiers: 
+ * String[] names = {"a", "b", "c", "d",  "e", "f",  "g", "h", "i",
+ *         "aa", "kk"};
+ * ArgParser ap = new ArgParser("func", names,
+ *         names.length - 2, 4, 3, true, true) //
+ *                 .defaults(3, 4, 5, 6) //
+ *                 .kwdefaults(7, null, 9);
+ *
Note that "aa" and "kk" are at the end of the parameter + * names. (This is how a CPython frame is laid out.) + *

+ * Defaults are provided, after the parser has been constructed, as + * values corresponding to parameter names, when right-justified in + * the space to which they apply. (See diagram below.) Both the + * positional and keyword defaults are given by position in this + * formulation. The {@link #kwdefaults(Object...)} call is allowed + * to supply {@code null} values at positions it does not define. + *

+ * When parsed to an array, the layout of the argument values, in + * relation to fields of the parser will be as follows. + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
A Python {@code frame}
namesabcdefghiaakk
layoutposOnlykwOnly
defaultskwdefaults
*

* The most readable way of specifying a parser (although one that * is a little costly to construct) is to list the parameters as * they would be declared in Python, including the furniture that * marks up the positional-only, keyword-only, positional varargs, * and keyword varargs. This is the API offered by - * {@link #fromSignature(String, String...)} but it only features in - * unit tests. - *

- * In practice, we construct the parser with a complex of arguments - * derived by inspection of the method signature. + * {@link #fromSignature(String, String...)}. In practice we only + * use this in unit tests. For serious applications we construct the + * {@code ArgParser} with a complex of arguments derived by + * inspection of the Java or Python signature. */ class ArgParser { @@ -76,16 +127,19 @@ class ArgParser { * Names of parameters that could be satisfied by position or * keyword, including the collector parameters. Elements are * guaranteed to be interned, and not {@code null} or empty. The - * length of this array is the number of named parameters: + * array must name all the parameters, of which there are: * {@code argcount + kwonlyargcount + (hasVarArgs() ? 1 : 0) + (hasVarKeywords() ? 1 : 0)} + *

+ * It is often is longer since it suits us to re-use an array that + * names all the local variables of a frame. */ /* - * Here and elsewhere we use the same names as the CPython code, - * even though it tends to say "argument" when it could mean that or - * "parameter". In comments and documentation "positional parameter" - * means a parameter eligible to be satisfied by an argument given - * by position. + * Here and elsewhere we use the same field names as the CPython + * code, even though it tends to say "argument" when it could mean + * that or "parameter". In comments and documentation + * "positional parameter" means a parameter eligible to be satisfied + * by an argument given by position. */ final String[] argnames; @@ -142,40 +196,45 @@ class ArgParser { final int varKeywordsIndex; /** - * Create a parser, for a named function, with defined numbers of - * positional-only and keyword-only parameters, and naming the - * parameters. Parameters that may only be satisfied by arguments - * given by position need not be named. ("" is acceptable in the - * names array.) + * Construct a parser for a named function, with defined numbers of + * positional-only and keyword-only parameters, and parameter names + * in an array prepared by client code. *

- * Overflow of positional and/or keyword arguments into a - * {@code tuple} or {@code dict} may also be allowed. For example, a - * function that in Python would have the signature with the - * function definition:

+     * The array of names is used in-place (not copied). The client code
+     * must therefore ensure that it cannot be modified after the parser
+     * has been constructed.
+     * 

+     * The array of names may be longer than is necessary: the caller
+     * specifies how much of the array should be treated as regular
+     * parameter names, and whether zero, one or two further elements
+     * will name collectors for excess positional or keyword arguments.
+     * The rest of the elements will not be examined by the parser. The
+     * motivation for this design is to permit efficient construction
+     * when the the array of names is the local variable names in a
+     * Python {@code code} object.
+     *
+     * @param name of the function
+     * @param names of the parameters including any collectors (varargs)
+     * @param regargcount number of regular (non-collector) parameters
+     * @param posOnly number of positional-only parameters
+     * @param kwOnly number of keyword-only parameters
+     * @param varargs whether there is positional collector
+     * @param varkw whether there is a keywords collector
+     */
+    ArgParser(String name, String[] names, int regargcount, int posOnly, int kwOnly,
+            boolean varargs, boolean varkw) {
+        this(name, ScopeKind.TYPE, MethodKind.STATIC, names, regargcount, posOnly, kwOnly, varargs,
+                varkw);
+    }
+
+    /**
+     * Construct a parser from descriptive parameters that may be
+     * derived from a the annotated declarations ({@link Exposed}
+     * methods) that appear in type and module definitions written in
+     * Java. For;
      * def func(a, b, c=3, d=4, /, e=5, f=6, *aa, g=7, h, i=9, **kk):
      *     pass
-     * # func.__defaults__ == (3, 4, 5, 6)
-     * # func.__kwdefaults__ == {'g': 7, 'i': 9}
-     * 
would be described by a constructor call: 
-     * private static ArgParser parser =
-     *     new ArgParser("func", "aa", "kk", 4, 3, //
-     *             "a", "b", "c", "d", "e", "f", "g", "h", "i")
-     *                     .defaults(3, 4, 5, 6) //
-     *                     .kwdefaults(7, null, 9);
-     *
-     * 
 Note that "aa" and "kk" are given separately, not amongst
-     * the parameter names. In the parsing result array, they will be at
-     * the end. (This is how a CPython frame is laid out.)
-     * 

-     * Defaults are provided, after the parser has been constructed, as
-     * values corresponding to parameter names, when right-justified in
-     * the space to which they apply. (See diagram below.) Both the
-     * positional and keyword defaults are given by position in this
-     * formulation. The {@link #kwdefaults(Object...)} call is allowed
-     * to supply {@code null} values at positions it does not define.
-     * 

-     * When parsed to an array, the layout of the argument values, in
-     * relation to fields of the parser will be as follows.
+     *
The constructor arguments should specify this layout: * * * @@ -197,6 +256,8 @@ class ArgParser { * * * + * + * * * * @@ -205,150 +266,37 @@ class ArgParser { * *
A Python {@code frame}
posOnlykwOnlyvarargsvarkw
* - * @param name of the function - * @param varargs name of the positional collector or {@code null} - * @param varkw name of the keywords collector or {@code null} - * @param posOnly number of positional-only parameters - * @param kwdOnly number of keyword-only parameters - * @param names of the (non-collector) parameters - */ - ArgParser(String name, String varargs, String varkw, int posOnly, int kwdOnly, - String... names) { - this(name, ScopeKind.TYPE, MethodKind.STATIC, varargs, varkw, posOnly, kwdOnly, names); - } - - /** * @param name of the function * @param scopeKind whether module, etc. * @param methodKind whether static, etc. - * @param varargs name of the positional collector or {@code null} - * @param varkw name of the keywords collector or {@code null} - * @param posOnly number of positional-only parameters - * @param kwdOnly number of keyword-only parameters - * @param names of the (non-collector) parameters - */ - ArgParser(String name, ScopeKind scopeKind, MethodKind methodKind, String varargs, String varkw, - int posOnly, int kwdOnly, String... names) { - - // Name of function - this.name = name; - this.methodKind = methodKind; - this.scopeKind = scopeKind; - - // Total parameter count *except* possible varargs, varkwargs - int N = names.length; - this.regargcount = N; - - // Fill in other cardinal points - this.posonlyargcount = posOnly; - this.kwonlyargcount = kwdOnly; - this.argcount = N - kwdOnly; - - // There may be positional and/or keyword collectors - this.varArgsIndex = varargs != null ? N++ : -1; - this.varKeywordsIndex = varkw != null ? N++ : -1; - - // Make a new array of the names, including the collectors. - String[] argnames = this.argnames = interned(names, N); - - if (varargs != null) - argnames[varArgsIndex] = varargs.intern(); - if (varkw != null) - argnames[varKeywordsIndex] = varkw.intern(); - - // Check for empty names - for (int i = posOnly; i < N; i++) { - if (argnames[i].length() == 0) { - // We found a "" name beyond positional only. - throw new InterpreterError(MISPLACED_EMPTY, name, argnames.toString()); - } - } - - assert argnames.length == argcount + kwonlyargcount + (hasVarArgs() ? 1 : 0) - + (hasVarKeywords() ? 1 : 0); - } - - /** - * Construct a parser for a named function, with defined numbers of - * positional-only and keyword-only parameters, and parameter names - * in an array prepared by client code. - *

- * The capabilities of this parser, are exactly the same as one - * defined by - * {@link #ArgParser(String, String, String, int, int, String...)}. - * The parser in the example there may be generated by: 

-     * String[] names = {"a", "b", "c", "d", "e", "f", "g", "h", "i",
-     *         "aa", "kk"};
-     * ArgParser ap = new ArgParser("func", true, true, 4, 3, names,
-     *         names.length - 2) //
-     *                 .defaults(3, 4, 5, 6) //
-     *                 .kwdefaults(7, null, 9);
-     *
The differences allow the array of names to be used - * in-place (not copied). The client code must therefore ensure that - * it cannot be modified after the parser has been constructed. - *

- * The array of names may be longer than is necessary: the caller - * specifies how much of the array should be treated as regular - * parameter names, and whether zero, one or two further elements - * will name collectors for excess positional or keyword arguments. - * The rest of the elements will not be examined by the parser. The - * motivation for this design is to permit efficient construction - * when the the array of names is the local variable names in a - * Python {@code code} object. - * - * @param name of the function - * @param varargs whether there is positional collector - * @param varkw whether there is a keywords collector - * @param posOnly number of positional-only parameters - * @param kwdOnly number of keyword-only parameters * @param names of the parameters including any collectors (varargs) - * @param count number of regular (non-collector) parameters - */ - ArgParser(String name, boolean varargs, boolean varkw, int posOnly, int kwdOnly, String[] names, - int count) { - this(name, ScopeKind.TYPE, MethodKind.STATIC, varargs, varkw, posOnly, kwdOnly, names, - count); - } - - /** - * Construct a parser from descriptive parameters that may be - * derived from a the annotated declarations ({@link Exposed} - * methods) that appear in type and module definitions written in - * Java. - * - * @param name of the function - * @param scopeKind whether module, etc. - * @param methodKind whether static, etc. + * @param regargcount number of regular (non-collector) parameters + * @param posOnly number of positional-only parameters + * @param kwOnly number of keyword-only parameters * @param varargs whether there is positional collector * @param varkw whether there is a keywords collector - * @param posOnly number of positional-only parameters - * @param kwdOnly number of keyword-only parameters - * @param names of the parameters including any collectors (varargs) - * @param count number of regular (non-collector) parameters */ - ArgParser(String name, ScopeKind scopeKind, MethodKind methodKind, boolean varargs, - boolean varkw, int posOnly, int kwdOnly, String[] names, int count) { + ArgParser(String name, ScopeKind scopeKind, MethodKind methodKind, String[] names, + int regargcount, int posOnly, int kwOnly, boolean varargs, boolean varkw) { // Name of function this.name = name; this.methodKind = methodKind; this.scopeKind = scopeKind; + this.argnames = names; // Total parameter count *except* possible varargs, varkwargs - int N = Math.min(count, names.length); + int N = Math.min(regargcount, names.length); this.regargcount = N; this.posonlyargcount = posOnly; - this.kwonlyargcount = kwdOnly; - this.argcount = N - kwdOnly; + this.kwonlyargcount = kwOnly; + this.argcount = N - kwOnly; // There may be positional and/or keyword collectors this.varArgsIndex = varargs ? N++ : -1; this.varKeywordsIndex = varkw ? N++ : -1; - // Make a new array of the names, including the collectors. - this.argnames = interned(names, N); - - assert argnames.length == argcount + kwonlyargcount + (hasVarArgs() ? 1 : 0) + assert argnames.length >= argcount + kwonlyargcount + (hasVarArgs() ? 1 : 0) + (hasVarKeywords() ? 1 : 0); } @@ -412,11 +360,35 @@ static ArgParser fromSignature(String name, String... decl) { // Total parameter count *except* possible varargs, varkwargs int N = args.size(); + + /* + * If there was no "/" or "*", all are positional arguments. This is + * consistent with the output of inspect.signature, where e.g. + * inspect.signature(exec) is (source, globals=None, locals=None, + * /). + */ if (posCount == 0) { posCount = N; } + // Number of regular arguments (not *, **) + int regArgCount = N; + int kwOnly = N - posCount; + + // Add any *args to the names + if (varargs != null) { + args.add(varargs); + N++; + } + + // Add any **kwargs to the names + if (varkw != null) { + args.add(varkw); + N++; + } + String[] names = N == 0 ? NO_STRINGS : args.toArray(new String[N]); - return new ArgParser(name, ScopeKind.TYPE, MethodKind.STATIC, varargs, varkw, posOnly, - N - posCount, names); + + return new ArgParser(name, ScopeKind.TYPE, MethodKind.STATIC, names, regArgCount, posOnly, + kwOnly, varargs != null, varkw != null); } /** @@ -522,21 +494,6 @@ String textSignature() { return sj.toString(); } - /** - * Return a copy of a {@code String} array in which every element is - * interned, in a new , possibly larger, array. We intern the - * strings to enable the fast path in keyword argument processing. - * - * @param a to intern - * @param N size of array to return - * @return array of equivalent interned strings - */ - private String[] interned(String[] a, int N) { - String[] s = new String[Math.max(N, a.length)]; - for (int i = 0; i < a.length; i++) { s[i] = a[i].intern(); } - return s; - } - /** * Return ith positional parameter name and default value if * available. Helper to {@link #sigString()}. @@ -633,26 +590,23 @@ Object[] parse(PyTuple args, PyDict kwargs) { return a; } - private static final String MISPLACED_EMPTY = - "Misplaced empty keyword in ArgParser spec for %s %s"; - /** - * Provide the positional defaults. If L values are provided, they - * correspond to {@code arg[max-L] ... arg[max-1]}, where - * {@code max} is the index of the first keyword-only parameter, or - * the number of parameters if there are no keyword-only parameters. - * The minimum number of positional arguments will then be - * {@code max-L}. + * Provide the positional defaults. * The {@code ArgParser} keeps a + * reference to this array, so that subsequent changes to it will + * affect argument parsing. (Concurrent access to the array and + * parser is a client issue.) + *

+ * If L values are provided, they correspond to + * {@code arg[max-L] ... arg[max-1]}, where {@code max} is the index + * of the first keyword-only parameter, or the number of parameters + * if there are no keyword-only parameters. The minimum number of + * positional arguments will then be {@code max-L}. * * @param values replacement positional defaults (or {@code null}) * @return {@code this} */ ArgParser defaults(Object... values) { - if (values == null || values.length == 0) { - defaults = null; - } else { - defaults = Arrays.copyOf(values, values.length); - } + defaults = values; checkShape(); return this; } @@ -682,18 +636,17 @@ ArgParser kwdefaults(Object... values) { } /** - * Provide the keyword-only defaults, perhaps as a {@code dict}. + * Provide the keyword-only defaults, perhaps as a {@code dict}. The + * {@code ArgParser} keeps a reference to this map, so that + * subsequent changes to it will affect argument parsing, as + * required for a Python {@link PyFunction function}. (Concurrent + * access to the mapping and parser is a client issue.) * * @param kwd replacement keyword defaults (or {@code null}) * @return {@code this} */ ArgParser kwdefaults(Map kwd) { - if (kwd == null || kwd.isEmpty()) - kwdefaults = null; - else { - kwdefaults = new HashMap<>(); - kwdefaults.putAll(kwd); - } + kwdefaults = kwd; checkShape(); return this; } @@ -706,6 +659,7 @@ ArgParser kwdefaults(Map kwd) { * number of parameters. */ private void checkShape() { + // XXX This may be too fussy, given that Python function is not final int N = argcount; final int L = defaults == null ? 0 : defaults.length; final int K = kwonlyargcount; @@ -781,12 +735,12 @@ void setPositionalArguments(PyTuple args) { * positional or keyword defaults to make up the shortfall. * * @param stack positional and keyword arguments - * @param start position of arguments in the array + * @param pos position of arguments in the array * @param nargs number of positional arguments */ - void setPositionalArguments(Object[] stack, int start, int nargs) { + void setPositionalArguments(Object[] stack, int pos, int nargs) { int n = Math.min(nargs, argcount); - for (int i = 0, j = start; i < n; i++) + for (int i = 0, j = pos; i < n; i++) setLocal(i, stack[j++]); } @@ -868,12 +822,13 @@ void setKeywordArguments(PyDict kwargs) { * @param kwnames keywords used in the call (or {@code **kwargs}) */ void setKeywordArguments(Object[] stack, int kwstart, String[] kwnames) { - /* - * Create a dictionary for the excess keyword parameters, and insert - * it in the local variables at the proper position. - */ + PyDict kwdict = null; if (varKeywordsIndex >= 0) { + /* + * Create a dictionary for the excess keyword parameters, and insert + * it in the local variables at the proper position. + */ kwdict = Py.dict(); setLocal(varKeywordsIndex, kwdict); } @@ -906,9 +861,6 @@ void setKeywordArguments(Object[] stack, int kwstart, String[] kwnames) { throw new TypeError(MULTIPLE_VALUES, name, key); } } - - if (varKeywordsIndex >= 0) { setLocal(varKeywordsIndex, kwdict); } - } /** @@ -1226,8 +1178,7 @@ class ArrayFrameWrapper extends FrameWrapper { * intended use is that {@code start = 1} allows space for a * {@code self} reference not in the argument list. The capacity of * the array, between the start index and the end, must be - * sufficient to hold the parse result. The destination array must - * be sufficient to hold the parse result and may be larger, e.g. to + * sufficient to hold the parse result may be larger, e.g. to * accommodate other local variables. * * @param vars destination array @@ -1259,6 +1210,12 @@ void setPositionalArguments(PyTuple argsTuple) { int n = Math.min(argsTuple.value.length, argcount); System.arraycopy(argsTuple.value, 0, vars, start, n); } + + @Override + void setPositionalArguments(Object[] stack, int pos, int nargs) { + int n = Math.min(nargs, argcount); + System.arraycopy(stack, pos, vars, start, n); + } } /** @@ -1335,11 +1292,12 @@ void parseToFrame(FrameWrapper frame, Object[] stack, int start, int nargs, Stri */ // Set parameters from the positional arguments in the call. - frame.setPositionalArguments(stack, start, nargs); + if (nargs > 0) { frame.setPositionalArguments(stack, start, nargs); } // Set parameters from the keyword arguments in the call. - if (nkwargs > 0) + if (varKeywordsIndex >= 0 || nkwargs > 0) { frame.setKeywordArguments(stack, start + nargs, kwnames); + } if (nargs > argcount) { @@ -1376,7 +1334,7 @@ void parseToFrame(FrameWrapper frame, Object[] stack, int start, int nargs, Stri * * @param frame to populate with argument values * @param args all arguments, positional then keyword - * @param kwnames of keyword arguments + * @param kwnames of keyword arguments (or {@code null}) */ void parseToFrame(FrameWrapper frame, Object[] args, String[] kwnames) { @@ -1393,11 +1351,12 @@ void parseToFrame(FrameWrapper frame, Object[] args, String[] kwnames) { */ // Set parameters from the positional arguments in the call. - frame.setPositionalArguments(args, 0, nargs); + if (nargs > 0) { frame.setPositionalArguments(args, 0, nargs); } // Set parameters from the keyword arguments in the call. - if (nkwargs > 0) + if (varKeywordsIndex >= 0 || nkwargs > 0) { frame.setKeywordArguments(args, nargs, kwnames); + } if (nargs > argcount) { diff --git a/core/src/main/java/org/python/core/Exposer.java b/core/src/main/java/org/python/core/Exposer.java index 4ba381f15..e1fb1e554 100644 --- a/core/src/main/java/org/python/core/Exposer.java +++ b/core/src/main/java/org/python/core/Exposer.java @@ -36,6 +36,7 @@ import org.python.core.Exposed.PositionalOnly; import org.python.core.Exposed.PythonMethod; import org.python.core.Exposed.PythonStaticMethod; +import org.python.core.ModuleDef.MethodDef; /** * An object for tabulating the attributes of classes that define @@ -67,6 +68,25 @@ protected Exposer() { /** @return which {@link ScopeKind} of {@code Exposer} is this? */ abstract ScopeKind kind(); + /** + * On behalf of the given module defined in Java, build a + * description of the attributes discovered by introspection of the + * class provided. + *

+ * Attributes are identified by annotations. (See {@link Exposed}.) + * + * @param definingClass to introspect for members + * @return exposure result + * @throws InterpreterError on errors of definition + */ + static ModuleExposer exposeModule(Class definingClass) throws InterpreterError { + // Create an instance of Exposer to hold specs, type, etc. + ModuleExposer exposer = new ModuleExposer(); + // Let the exposer control the logic + exposer.expose(definingClass); + return exposer; + } + /** * On behalf of the given type defined in Java, build a description * of the attributes discovered by introspection of the class (or @@ -617,14 +637,35 @@ private boolean isDefined() { ArgParser getParser() { if (parser == null && parameterNames != null && parameterNames.length >= posonlyargcount) { - parser = new ArgParser(name, scopeKind, methodKind, varArgsIndex >= 0, - varKeywordsIndex >= 0, posonlyargcount, kwonlyargcount, parameterNames, - regargcount); + parser = new ArgParser(name, scopeKind, methodKind, parameterNames, regargcount, + posonlyargcount, kwonlyargcount, varArgsIndex >= 0, varKeywordsIndex >= 0); parser.defaults(defaults).kwdefaults(kwdefaults); } return parser; } + /** + * Produce a method definition from this specification that + * references a method handle on the (single) defining method and + * the parser created from this specification. This is used in the + * construction of a module defined in Java (a {@link ModuleDef}). + * + * @param lookup authorisation to access methods + * @return corresponding method definition + * @throws InterpreterError on lookup prohibited + */ + MethodDef getMethodDef(Lookup lookup) throws InterpreterError { + assert methods.size() == 1; + Method m = methods.get(0); + MethodHandle mh; + try { + mh = lookup.unreflect(m); + } catch (IllegalAccessException e) { + throw cannotGetHandle(m, e); + } + return new MethodDef(getParser(), mh); + } + /** * Add a method implementation. (A test that the signature is * acceptable follows when we construct the {@link PyMethodDescr}.) @@ -1126,9 +1167,9 @@ static class MethodSpec extends CallableSpec { @Override PyMethodDescr asAttribute(PyType objclass, Lookup lookup) throws InterpreterError { - ArgParser ap = new ArgParser(name, scopeKind, MethodKind.INSTANCE, varArgsIndex >= 0, - varKeywordsIndex >= 0, posonlyargcount, kwonlyargcount, parameterNames, - regargcount); + ArgParser ap = new ArgParser(name, scopeKind, MethodKind.INSTANCE, parameterNames, + regargcount, posonlyargcount, kwonlyargcount, varArgsIndex >= 0, + varKeywordsIndex >= 0); ap.defaults(defaults).kwdefaults(kwdefaults); // Methods have self + this many args: diff --git a/core/src/main/java/org/python/core/JavaModule.java b/core/src/main/java/org/python/core/JavaModule.java new file mode 100644 index 000000000..dcdef00c2 --- /dev/null +++ b/core/src/main/java/org/python/core/JavaModule.java @@ -0,0 +1,23 @@ +// Copyright (c)2022 Jython Developers. +// Licensed to PSF under a contributor agreement. +package org.python.core; + +/** Common mechanisms for all Python modules defined in Java. */ +public abstract class JavaModule extends PyModule { + + final ModuleDef definition; + + /** + * Construct the base {@code JavaModule} and fill the module + * dictionary from the given module definition, which is normally + * created during static initialisation of the concrete class + * defining the module. + * + * @param definition of the module + */ + protected JavaModule(ModuleDef definition) { + super(definition.name); + this.definition = definition; + definition.addMembers(this); + } +} diff --git a/core/src/main/java/org/python/core/ModuleDef.java b/core/src/main/java/org/python/core/ModuleDef.java new file mode 100644 index 000000000..9e3518ea6 --- /dev/null +++ b/core/src/main/java/org/python/core/ModuleDef.java @@ -0,0 +1,157 @@ +// Copyright (c)2022 Jython Developers. +// Licensed to PSF under a contributor agreement. +package org.python.core; + +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles.Lookup; + +/** + * A {@code ModuleDef} is a definition from which instances of a module + * may be made. It stands in relation to the Java classes that define + * Python modules, somewhat in the way a Python {@code type} object + * stands in relation to the Java classes that define Python objects. + *

+ * What we most often encounter as "a module", a Python source file, is + * actually just a definition from which a module object may be made. + * This happens once in each interpreter where the module is + * imported. A distinct object, with mutable state, represents that + * module in each interpreter. There must therefore be a factory object + * that has access to the definition of the module, but is able to + * instantiate it (equivalent to executing the body of a module defined + * in Python). A {@code ModuleDef} is that factory. + *

+ * This initialisation cannot be identified with the static + * initialisation of the Java class, since that cannot be repeated, but + * must happen per instance. It is useful, however, to have an + * intermediate cache of the results of processing the defining Java + * class once statically initialised. + */ +public class ModuleDef { + // Compare CPython PyModuleDef + + /** Name of the module. */ + final String name; + + /** The Java class defining instances of the module. */ + final Class definingClass; + + /** + * Definitions for the members that appear in the dictionary of + * instances of the module named. Instances receive members by copy, + * by binding to the module instance (descriptors), or by reference + * (if immutable). + */ + private final MethodDef[] methods; + + /** + * Create a definition for the module, largely by introspection on + * the class and by forming {@code MethodHandle}s on discovered + * attributes. + * + * @param name of the module (e.g. "sys" or "math") + * @param lookup authorises access to the defining class. + */ + ModuleDef(String name, Lookup lookup) { + this.name = name; + this.definingClass = lookup.lookupClass(); + ModuleExposer exposer = Exposer.exposeModule(definingClass); + this.methods = exposer.getMethodDefs(lookup); + // XXX ... and for fields. + // XXX ... and for types defined in the module maybe? :o + } + + /** + * Get the method definitions. This method is provided for test use + * only. It isn't safe as for public use. + * + * @return the method definitions + */ + MethodDef[] getMethods() { return methods; } + + /** + * Add members defined here to the dictionary of a module instance. + * + * @param module to populate + */ + void addMembers(JavaModule module) { + PyDict d = module.dict; + for (MethodDef md : methods) { + // Create function by binding to the module + PyJavaFunction func = PyJavaFunction.fromParser( + md.argParser, md.handle, module, this.name); + d.put(md.argParser.name, func); + } + } + + /** + * A {@code MethodDef} describes a built-in function or method as it + * is declared in a Java module. It holds an argument parser and a + * handle for calling the method. + *

+ * Recall that a module definition may have multiple instances. The + * {@code MethodDef} represents the method between the definition of + * the module (exposure as a {@link ModuleDef}) and the creation of + * actual {@link JavaModule} instances. + *

+ * When a method is declared in Java as an instance method of the + * module, the {@code MethodDef} that describes it discounts the + * {@code self} argument. The {@link PyJavaFunction} created from it + * binds the module instance that is its target, so that it is is + * correct for a call to that {@code PyJavaFunction}. This is + * consistent with CPython. + */ + // Compare CPython struct PyMethodDef + static class MethodDef { + + /* + * The object here is only superficially similar to the CPython + * PyMethodDef: it is not used as a member of descriptors or + * methods; extension writers do not declare instances of them. + * Instead, we reify the argument information from the + * declaration in Java, and associated annotations. In CPython, + * this knowledge is present at run-time in the structure of the + * code generated by Argument Clinic, incompletely in the flags + * of the PyMethodDef, and textually in the signature that + * begins the documentation string. We do it by holding an + * ArgParser. + */ + + /** + * An argument parser constructed with this {@code MethodDef} + * from the description of the signature. Full information on + * the signature is available from this structure, and it is + * available to parse the arguments to a standard + * {@code (Object[], String[])} call. (In simple sub-classes it + * is only used to generate error messages once simple checks + * fail.) + */ + final ArgParser argParser; + + /** + * A handle to the implementation of the function or method. + * This is generated by reflecting the same object that + * {@link #argParser} describes. + */ + // CPython PyMethodDef: ml_meth + final MethodHandle handle; + + /** + * Create a {@link MethodDef} of the given kind from the + * {@link ArgParser} provided. + * + * @param argParser parser defining the method + * @param meth method handle prepared by sub-class + */ + MethodDef(ArgParser argParser, MethodHandle meth) { + this.argParser = argParser; + assert meth != null; + this.handle = meth; + } + + @Override + public String toString() { + return String.format("%s[%s]", getClass().getSimpleName(), + argParser); + } + } +} diff --git a/core/src/main/java/org/python/core/ModuleExposer.java b/core/src/main/java/org/python/core/ModuleExposer.java new file mode 100644 index 000000000..277f3b586 --- /dev/null +++ b/core/src/main/java/org/python/core/ModuleExposer.java @@ -0,0 +1,88 @@ +// Copyright (c)2022 Jython Developers. +// Licensed to PSF under a contributor agreement. +package org.python.core; + +import java.lang.invoke.MethodHandles.Lookup; +import java.lang.reflect.Method; + +import org.python.core.Exposed.PythonMethod; +import org.python.core.Exposed.PythonStaticMethod; +import org.python.base.InterpreterError; +import org.python.core.ModuleDef.MethodDef; + +/** + * A {@code ModuleExposer} provides access to the attributes of a module + * defined in Java (a built-in or extension module). These are primarily + * the {@link MethodDef}s derived from annotated methods in the defining + * class. It is normally obtained by a call to + * {@link Exposer#exposeModule(Class)}. + */ +class ModuleExposer extends Exposer { + + /** + * Construct the {@code ModuleExposer} instance for a particular + * module. + */ + ModuleExposer() {} + + /** + * Build the result from the defining class. + * + * @param definingClass to scan for definitions + */ + void expose(Class definingClass) { + // Scan the defining class for definitions + scanJavaMethods(definingClass); + // XXX ... and for fields. + // XXX ... and for types defined in the module maybe? :o + } + + @Override + ScopeKind kind() { return ScopeKind.MODULE; } + + /** + * From the methods discovered by introspection of the class, return + * an array of {@link MethodDef}s. This array will normally be part + * of a {@link ModuleDef} from which the dictionary of each instance + * of the module will be created. + * + * A {@link MethodDef} relies on {@code MethodHandle}, so a lookup + * object must be provided with the necessary access to the defining + * class. + * + * @param lookup authorisation to access methods + * @return method definitions + * @throws InterpreterError on lookup prohibited + */ + MethodDef[] getMethodDefs(Lookup lookup) throws InterpreterError { + MethodDef[] a = new MethodDef[methodSpecs.size()]; + int i = 0; + for (CallableSpec ms : methodSpecs) { + a[i++] = ms.getMethodDef(lookup); + } + return a; + } + + /** + * For a Python module defined in Java, add to {@link specs}, the + * methods found in the given defining class and annotated for + * exposure. + * + * @param definingClass to introspect for definitions + * @throws InterpreterError on duplicates or unsupported types + */ + @Override + void scanJavaMethods(Class definingClass) + throws InterpreterError { + + // Collect exposed functions (Java methods) + for (Method m : definingClass.getDeclaredMethods()) { + PythonMethod a = + m.getDeclaredAnnotation(PythonMethod.class); + if (a != null) { addMethodSpec(m, a); } + PythonStaticMethod sm = + m.getDeclaredAnnotation(PythonStaticMethod.class); + if (sm != null) { addStaticMethodSpec(m, sm); } + } + } +} diff --git a/core/src/test/java/org/python/core/ArgParserTest.java b/core/src/test/java/org/python/core/ArgParserTest.java index b9bbd985f..fe0a00534 100644 --- a/core/src/test/java/org/python/core/ArgParserTest.java +++ b/core/src/test/java/org/python/core/ArgParserTest.java @@ -1,9 +1,13 @@ +// Copyright (c)2022 Jython Developers. +// Licensed to PSF under a contributor agreement. package org.python.core; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; +import java.util.List; + import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; @@ -208,58 +212,105 @@ void throws_when_arg_missing() { } @Nested - @DisplayName("Example 1 from the Javadoc") - class FromJavadoc1 extends Standard { - - ArgParser ap = // - new ArgParser("func", "aa", "kk", // - 4, 3, // - "a", "b", "c", "d", "e", "f", "g", "h", "i")// - .defaults(3, 4, 5, 6) // - .kwdefaults(77, null, 99); - private String SIG = "func(a, b, c=3, d=4, /, e=5, f=6, *aa, g=77, h, i=99, **kk)"; + @DisplayName("A parser for a positional collector") + class PositionalCollector extends Standard { + + ArgParser ap = ArgParser.fromSignature("func", "*aa"); @Override @Test void has_expected_fields() { assertEquals("func", ap.name); - assertEquals(11, ap.argnames.length); - assertEquals(6, ap.argcount); - assertEquals(4, ap.posonlyargcount); - assertEquals(3, ap.kwonlyargcount); - assertEquals(9, ap.regargcount); - assertEquals(9, ap.varArgsIndex); - assertEquals(10, ap.varKeywordsIndex); + assertEquals(1, ap.argnames.length); + assertEquals(0, ap.argcount); + assertEquals(0, ap.posonlyargcount); + assertEquals(0, ap.kwonlyargcount); + assertEquals(0, ap.regargcount); + assertEquals(0, ap.varArgsIndex); + assertEquals(-1, ap.varKeywordsIndex); } @Override @Test void parses_classic_args() { - PyTuple args = Py.tuple(10, 20, 30); + PyTuple args = Py.tuple(1, 2, 3); PyDict kwargs = Py.dict(); - kwargs.put("g", 70); - kwargs.put("h", 80); - PyTuple expectedTuple = PyTuple.EMPTY; - PyDict expectedDict = Py.dict(); - Object[] expected = - new Object[] {10, 20, 30, 4, 5, 6, 70, 80, 99, expectedTuple, expectedDict}; + Object[] frame = ap.parse(args, kwargs); + assertEquals(1, frame.length); + assertEquals(List.of(1, 2, 3), frame[0]); + } + + @Test + void throws_on_keyword() { + PyTuple args = Py.tuple(1); + PyDict kwargs = Py.dict(); + kwargs.put("c", 3); + assertThrows(TypeError.class, () -> ap.parse(args, kwargs)); + } + + @Override + @Test + void has_expected_toString() { assertEquals("func(*aa)", ap.toString()); } + } + + @Nested + @DisplayName("A parser for a keyword collector") + class KeywordCollector extends Standard { + + ArgParser ap = ArgParser.fromSignature("func", "**kk"); + + @Override + @Test + void has_expected_fields() { + assertEquals("func", ap.name); + assertEquals(1, ap.argnames.length); + assertEquals(0, ap.argcount); + assertEquals(0, ap.posonlyargcount); + assertEquals(0, ap.kwonlyargcount); + assertEquals(0, ap.regargcount); + assertEquals(-1, ap.varArgsIndex); + assertEquals(0, ap.varKeywordsIndex); + } + + @Override + @Test + void parses_classic_args() { + PyTuple args = Py.tuple(); + PyDict kwargs = Py.dict(); + kwargs.put("b", 2); + kwargs.put("c", 3); + kwargs.put("a", 1); Object[] frame = ap.parse(args, kwargs); - assertArrayEquals(expected, frame); + assertEquals(1, frame.length); + PyDict kk = (PyDict)frame[0]; + assertEquals(1, kk.get("a")); + assertEquals(2, kk.get("b")); + assertEquals(3, kk.get("c")); + } + + @Test + void throws_on_positional() { + PyTuple args = Py.tuple(1); + PyDict kwargs = Py.dict(); + kwargs.put("b", 2); + kwargs.put("c", 3); + kwargs.put("a", 1); + assertThrows(TypeError.class, () -> ap.parse(args, kwargs)); } @Override @Test - void has_expected_toString() { assertEquals(SIG, ap.toString()); } + void has_expected_toString() { assertEquals("func(**kk)", ap.toString()); } } @Nested - @DisplayName("Example 2 from the Javadoc") - class FromJavadoc2 extends Standard { + @DisplayName("Example from the Javadoc") + class FromJavadoc extends Standard { String[] names = {"a", "b", "c", "d", "e", "f", "g", "h", "i", "aa", "kk"}; - ArgParser ap = new ArgParser("func", true, true, 4, 3, names, names.length - 2) // + ArgParser ap = new ArgParser("func", names, names.length - 2, 4, 3, true, true) // .defaults(3, 4, 5, 6) // .kwdefaults(77, null, 99); private String SIG = "func(a, b, c=3, d=4, /, e=5, f=6, *aa, g=77, h, i=99, **kk)"; diff --git a/core/src/test/java/org/python/core/ModuleExposerTest.java b/core/src/test/java/org/python/core/ModuleExposerTest.java new file mode 100644 index 000000000..f9d46b8d9 --- /dev/null +++ b/core/src/test/java/org/python/core/ModuleExposerTest.java @@ -0,0 +1,220 @@ +// Copyright (c)2022 Jython Developers. +// Licensed to PSF under a contributor agreement. +package org.python.core; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodHandles.Lookup; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; + +import org.python.core.Exposed.PositionalOnly; +import org.python.core.Exposed.PythonMethod; +import org.python.core.Exposed.PythonStaticMethod; +import org.python.core.ModuleDef.MethodDef; + +/** + * Test that a Python module defined in Java, using the scheme of + * annotations defined in {@link Exposed}, can be processed correctly by + * a {@link Exposer} to a {@link ModuleDef}. This tests a large part of + * the exposure mechanism. + *

+ * The class used in the test {@link FakeModule} is not actually a + * {@link PyModule}, but we go through the actions of the + * {@link ModuleExposer} so we can examine the intermediate results. + */ +@DisplayName("For a module exposed from a Java definition") +class ModuleExposerTest extends UnitTestSupport { + + /** + * This class is not actually a Python module definition, but is + * annotated as if it were. We will test whether the + * {@link MethodDef}s are created as expected. We'll also act on it + * to produce a dictionary as if it were a real module. + */ + static class FakeModule { + + static final Lookup LOOKUP = MethodHandles.lookup(); + + // Signature: () + @PythonStaticMethod + static void f0() {} + + // Signature: ($module, /) + @PythonMethod + void m0() {} + + // Signature: (a) + @PythonStaticMethod + static PyTuple f1(double a) {return Py.tuple(a);} + + // Signature: ($module, a, /) + @PythonMethod + @SuppressWarnings("static-method") + PyTuple m1(double a) {return Py.tuple(a);} + + // Signature: (a, b, c, /) + @PythonStaticMethod + static PyTuple f3(int a, String b, Object c) { + return Py.tuple(a, b, c); + } + + // Signature: ($module, a, b, c, /) + @PythonMethod + @SuppressWarnings("static-method") + PyTuple m3(int a, String b, Object c) { + return Py.tuple(a, b, c); + } + + // Signature: (/, a, b, c) + @PythonStaticMethod(positionalOnly = false) + static PyTuple f3pk(int a, String b, Object c) { + return Py.tuple(a, b, c); + } + + // Signature: ($module, /, a, b, c) + @PythonMethod(positionalOnly = false) + @SuppressWarnings("static-method") + PyTuple m3pk(int a, String b, Object c) { + return Py.tuple(a, b, c); + } + + // Signature: (a, b, /, c) + @PythonStaticMethod + static PyTuple f3p2(int a, @PositionalOnly String b, Object c) { + return Py.tuple(a, b, c); + } + + // Signature: ($module, a, b, /, c) + @PythonMethod + @SuppressWarnings("static-method") + PyTuple m3p2(int a, @PositionalOnly String b, Object c) { + return Py.tuple(a, b, c); + } + } + + @Nested + @DisplayName("calling the Exposer") + class TestExposer { + + @Test + @DisplayName("produces a ModuleExposer") + void getExposer() { + ModuleExposer exposer = + Exposer.exposeModule(FakeModule.class); + assertNotNull(exposer); + } + + @Test + @DisplayName("finds the expected methods") + void getMethodDefs() { + ModuleExposer exposer = + Exposer.exposeModule(FakeModule.class); + MethodDef[] mdArray = + exposer.getMethodDefs(FakeModule.LOOKUP); + checkMethodDefArray(mdArray); + } + } + + @Nested + @DisplayName("constructing a ModuleDef") + class TestDefinition { + + @Test + @DisplayName("produces a MethodDef array") + void createMethodDef() { + ModuleDef def = new ModuleDef("example", FakeModule.LOOKUP); + checkMethodDefArray(def.getMethods()); + } + } + + @Nested + @DisplayName("a module instance") + class TestInstance { + + @Test + @DisplayName("has expected method signatures") + void hasMethods() { + /* + * As FakeModule is not a PyModule, we must work a bit + * harder to take care of things normally automatic. Make a + * ModuleDef to hold the MethodDefs from the Exposer. + */ + ModuleDef def = new ModuleDef("example", FakeModule.LOOKUP); + // An instance of the "module" to bind in PyJavaMethods + FakeModule fake = new FakeModule(); + // A map to stand in for the module dictionary to hold them + Map dict = new HashMap<>(); + // Which we now fill ... + for (MethodDef md : def.getMethods()) { + ArgParser ap = md.argParser; + MethodHandle mh = md.handle; + PyJavaFunction m = + PyJavaFunction.fromParser(ap, mh, fake, def.name); + dict.put(md.argParser.name, m); + } + // And here we check what's in it + checkMethodSignatures(dict); + } + } + + private static void checkMethodDefArray(MethodDef[] defs) { + assertNotNull(defs); + + Map mds = new TreeMap<>(); + for (MethodDef def : defs) { mds.put(def.argParser.name, def); } + + Set expected = new TreeSet<>(); + expected.addAll(List.of( // + "f0", "f1", "f3", "f3pk", "f3p2", // + "m0", "m1", "m3", "m3pk", "m3p2")); + + assertEquals(expected, mds.keySet(), "contains expected names"); + } + + private static void + checkMethodSignatures(Map dict) { + assertNotNull(dict); + + checkSignature(dict, "f0()"); + checkSignature(dict, "m0($module, /)"); + checkSignature(dict, "f1(a, /)"); + checkSignature(dict, "m1($module, a, /)"); + checkSignature(dict, "f3(a, b, c, /)"); + checkSignature(dict, "m3($module, a, b, c, /)"); + checkSignature(dict, "f3pk(a, b, c)"); + checkSignature(dict, "m3pk($module, /, a, b, c)"); + checkSignature(dict, "f3p2(a, b, /, c)"); + checkSignature(dict, "m3p2($module, a, b, /, c)"); + } + + /** + * Check that a method with the expected signature is in the + * dictionary. + * + * @param dict dictionary + * @param spec signature + */ + private static void checkSignature(Map dict, + String spec) { + int k = spec.indexOf('('); + assertTrue(k > 0); + String name = spec.substring(0, k); + String expect = spec.substring(k); + PyJavaFunction pjm = (PyJavaFunction)dict.get(name); + assertEquals(expect, pjm.argParser.textSignature()); + } + +} From c4ba9fe0be2d195bcea0437c48ab199a6977a4b1 Mon Sep 17 00:00:00 2001 From: Jeff Allen Date: Sat, 31 Dec 2022 10:52:40 +0000 Subject: [PATCH 3/6] Test calling functions with a range of signatures As a result we find and fix a bug and some inefficiencies in PyJavaFunction calls. --- .../java/org/python/core/PyJavaFunction.java | 53 +- .../python/core/ModuleExposerMethodTest.java | 683 ++++++++++++++++++ .../org/python/core/ModuleExposerTest.java | 4 +- 3 files changed, 706 insertions(+), 34 deletions(-) create mode 100644 core/src/test/java/org/python/core/ModuleExposerMethodTest.java diff --git a/core/src/main/java/org/python/core/PyJavaFunction.java b/core/src/main/java/org/python/core/PyJavaFunction.java index d3083994c..6de09cda8 100644 --- a/core/src/main/java/org/python/core/PyJavaFunction.java +++ b/core/src/main/java/org/python/core/PyJavaFunction.java @@ -180,40 +180,13 @@ protected Object __repr__() throws Throwable { Object __call__(Object[] args, String[] names) throws TypeError, Throwable { try { - if (names != null && names.length != 0) { - return call(args, names); - } else { - int n = args.length; - switch (n) { - // case 0 (an error) handled by default clause - case 1: - return call(args[0]); - case 2: - return call(args[0], args[1]); - case 3: - return call(args[0], args[1], args[2]); - case 4: - return call(args[0], args[1], args[2], args[3]); - default: - return call(args); - } - } + // It is *not* worth unpacking the array here + return call(args, names); } catch (ArgumentError ae) { throw typeError(ae, args, names); } } - /* - * A simplified __call__ used in the narrative. To use, rename this - * to __call__, rename the real __call__ to something else, and - * force fromParser() and from() always to select General as the - * implementation type. - */ - Object simple__call__(Object[] args, String[] names) throws TypeError, Throwable { - Object[] frame = argParser.parse(args, names); - return handle.invokeExact(frame); - } - // exposed methods ----------------------------------------------- /** @return name of the function or method */ @@ -341,7 +314,23 @@ public Object call(Object[] args, String[] names) throws TypeError, Throwable { public Object call(Object[] args) throws TypeError, Throwable { // Make sure we find out if this is missing throw new InterpreterError( - "Sub-classes of AbstractPositional " + "must define call(Object[])"); + "Sub-classes of AbstractPositional must define call(Object[])"); + } + + // Save some indirection by specialising to positional + @Override + Object __call__(Object[] args, String[] names) + throws TypeError, Throwable { + try { + if (names == null || names.length == 0) { + // It is *not* worth unpacking the array here + return call(args); + } else { + throw new ArgumentError(Mode.NOKWARGS); + } + } catch (ArgumentError ae) { + throw typeError(ae, args, names); + } } } @@ -499,7 +488,7 @@ public Object call(Object[] a) throws ArgumentError, TypeError, Throwable { int n = a.length, k; if (n == 3) { // Number of arguments matches number of parameters - return handle.invokeExact(a[0], a[1], a[3]); + return handle.invokeExact(a[0], a[1], a[2]); } else if ((k = n - min) >= 0) { if (n == 2) { return handle.invokeExact(a[0], a[1], d[k]); @@ -515,7 +504,7 @@ public Object call(Object[] a) throws ArgumentError, TypeError, Throwable { @Override public Object call() throws Throwable { - if (min == 0) { return handle.invokeExact(d[0], d[1], d[3]); } + if (min == 0) { return handle.invokeExact(d[0], d[1], d[2]); } throw new ArgumentError(min, max); } diff --git a/core/src/test/java/org/python/core/ModuleExposerMethodTest.java b/core/src/test/java/org/python/core/ModuleExposerMethodTest.java new file mode 100644 index 000000000..cccbe3da5 --- /dev/null +++ b/core/src/test/java/org/python/core/ModuleExposerMethodTest.java @@ -0,0 +1,683 @@ +// Copyright (c)2022 Jython Developers. +// Licensed to PSF under a contributor agreement. +package org.python.core; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.lang.invoke.MethodHandles; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; + +import org.python.core.Exposed.PositionalOnly; +import org.python.core.Exposed.PythonMethod; +import org.python.core.Exposed.PythonStaticMethod; +import org.python.base.MethodKind; + +/** + * Test that functions exposed by a Python module defined in + * Java, using the scheme of annotations defined in {@link Exposed}, + * result in {@link PyJavaFunction} objects with characteristics + * that correspond to the definition. + *

+ * The first test in each case is to examine the fields in the + * parser that attaches to the {@link ModuleDef.MethodDef}. Then we + * call the function using the {@code __call__} special method, and + * using our "Java call" signatures. + *

+ * There is a nested test suite for each signature pattern. + */ +@DisplayName("A method exposed by a module") +class ModuleExposerMethodTest { + + /** + * Nested test classes implement these as standard. A base class + * here is just a way to describe the tests once that we repeat in + * each nested case. + */ + abstract static class Standard { + + // Working variables for the tests + /** The module we create. */ + PyModule module = new ExampleModule(); + /** The function to examine or call. */ + PyJavaFunction func; + /** The parser in the function we examine. */ + ArgParser ap; + /** The expected result of calling the function */ + Object[] exp; + + /** + * A parser attached to the function object should have field values + * that correctly reflect the signature and annotations in the + * defining class. + */ + abstract void has_expected_fields(); + + /** + * Call the function using the {@code __call__} special method with + * arguments correct for the function's specification. The function + * should obtain the correct result (and not throw). + * + * @throws Throwable unexpectedly + */ + abstract void supports__call__() throws Throwable; + + /** + * Call the function using the {@code __call__} special method with + * arguments correct for the function's specification, and + * explicitly zero or more keywords. The function should obtain the + * correct result (and not throw). + * + * @throws Throwable unexpectedly + */ + abstract void supports_keywords() throws Throwable; + + /** + * Call the function using the {@code __call__} special method and + * an unexpected keyword: where none is expected, for a positional + * argument, or simply an unacceptable name. The function should + * throw {@link TypeError}. + * + * @throws Throwable unexpectedly + */ + abstract void raises_TypeError_on_unexpected_keyword() throws Throwable; + + /** + * Call the function using the Java call interface with arguments + * correct for the function's specification. The function should + * obtain the correct result (and not throw). + * + * @throws Throwable unexpectedly + */ + abstract void supports_java_call() throws Throwable; + + /** + * Check that the fields of the parser match expectations for a + * method with no collector parameters and a certain number of + * positional-only parameters. + * + * @param kind static or instance + * @param name of method + * @param count of parameters + * @param posonlycount count of positional-only parameters + */ + void no_collector(MethodKind kind, String name, int count, int posonlycount) { + assertEquals(name, ap.name); + assertEquals(kind, ap.methodKind); + assertEquals(count, ap.argnames.length); + assertEquals(count, ap.argcount); + assertEquals(posonlycount, ap.posonlyargcount); + assertEquals(0, ap.kwonlyargcount); + assertEquals(count, ap.regargcount); + assertEquals(-1, ap.varArgsIndex); + assertEquals(-1, ap.varKeywordsIndex); + } + + /** + * Check that the fields of the parser match expectations for a + * static method with no collector parameters and a certain number + * of positional-only parameters. + * + * @param name of method + * @param count of parameters + * @param posonly count of positional-only parameters + */ + void no_collector_static(String name, int count, int posonly) { + no_collector(MethodKind.STATIC, name, count, posonly); + } + + /** + * Check that the fields of the parser match expectations for a + * instance method with no collector parameters and a certain number + * of positional-only parameters. + * + * @param name of method + * @param count of parameters + * @param posonly count of positional-only parameters + */ + void no_collector_instance(String name, int count, int posonly) { + no_collector(MethodKind.INSTANCE, name, count, posonly); + } + + /** + * Check the result of a call against {@link #exp}. The reference + * rtesult is the same throughout a given sub-class test. + * + * @param result of call + */ + void check_result(PyTuple result) { assertArrayEquals(exp, result.value); } + } + + /** + * A Python module definition that exhibits a range of method + * signatures explored in the tests. + */ + static class ExampleModule extends JavaModule { + + static final ModuleDef DEF = new ModuleDef("example", MethodHandles.lookup()); + + ExampleModule() { super(DEF); } + + /** + * See {@link StaticNoParams}: no parameters are allowed. + */ + @PythonStaticMethod + static void f0() {} + + /** + * See {@link NoParams}: no parameters are allowed. + */ + @PythonMethod + void m0() {} + + /** + * See {@link StaticOneParam}: the parameter is positional-only as a + * result of the default exposure. + * + * @param a positional arg + * @return the arg (tuple) + */ + @PythonStaticMethod + static PyTuple f1(double a) { return Py.tuple(a); } + + /** + * See {@link OneParam}: the parameter is positional-only as a + * result of the default exposure. + * + * @param a positional arg + * @return the arg (tuple) + */ + @PythonMethod + PyTuple m1(double a) { return Py.tuple(this, a); } + + /** + * See {@link StaticDefaultPositionalParams}: the parameters are + * positional-only as a result of the default exposure. + * + * @param a positional arg + * @param b positional arg + * @param c positional arg + * @return the args + */ + @PythonStaticMethod + static PyTuple f3(int a, String b, Object c) { return Py.tuple(a, b, c); } + + /** + * See {@link DefaultPositionalParams}: the parameters are + * positional-only as a result of the default exposure. + * + * @param a positional arg + * @param b positional arg + * @param c positional arg + * @return the args + */ + @PythonMethod + PyTuple m3(int a, String b, Object c) { return Py.tuple(this, a, b, c); } + + /** + * See {@link StaticPositionalOrKeywordParams}: the parameters are + * positional-or-keyword but none are positional-only. + * + * @param a positional-or-keyword arg + * @param b positional-or-keyword arg + * @param c positional-or-keyword arg + * @return the args + */ + @PythonStaticMethod(positionalOnly = false) + static PyTuple f3pk(int a, String b, Object c) { return Py.tuple(a, b, c); } + + /** + * See {@link PositionalOrKeywordParams}: the parameters are + * positional-or-keyword but none are positional-only. + * + * @param a positional-or-keyword arg + * @param b positional-or-keyword arg + * @param c positional-or-keyword arg + * @return the args + */ + @PythonMethod(positionalOnly = false) + PyTuple m3pk(int a, String b, Object c) { return Py.tuple(this, a, b, c); } + + /** + * See {@link SomePositionalOnlyParams}: two parameters are + * positional-only as a result of an annotation. + * + * @param a positional arg + * @param b positional arg + * @param c positional-or-keyword arg + * @return the args + */ + @PythonStaticMethod + static PyTuple f3p2(int a, @PositionalOnly String b, Object c) { return Py.tuple(a, b, c); } + + /** + * See {@link StaticSomePositionalOnlyParams}: two parameters are + * positional-only as a result of an annotation. + * + * @param a positional arg + * @param b positional arg + * @param c positional-or-keyword arg + * @return the args + */ + @PythonMethod + PyTuple m3p2(int a, @PositionalOnly String b, Object c) { return Py.tuple(this, a, b, c); } + } + + /** {@link ExampleModule#m0()} accepts no arguments. */ + @Nested + @DisplayName("with no parameters") + class NoParams extends Standard { + + @BeforeEach + void setup() throws AttributeError, Throwable { + // func = module.m0 + func = (PyJavaFunction)Abstract.getAttr(module, "m0"); + ap = func.argParser; + } + + @Override + @Test + void has_expected_fields() { no_collector_instance("m0", 0, 0); } + + @Override + @Test + void supports__call__() throws Throwable { + // We call func() + Object[] args = {}; + + // The method is declared void (which means return None) + Object r = func.__call__(args, null); + assertEquals(Py.None, r); + } + + /** Keywords must be empty. */ + @Override + @Test + void supports_keywords() throws Throwable { + // We call func() + Object[] args = {}; + String[] names = {}; + + // The method is declared void (which means return None) + Object r = func.__call__(args, names); + assertEquals(Py.None, r); + } + + @Override + @Test + void raises_TypeError_on_unexpected_keyword() { + // We call func(c=3) + Object[] args = {3}; + String[] names = {"c"}; // Nothing expected + + assertThrows(TypeError.class, () -> func.__call__(args, names)); + } + + @Override + @Test + void supports_java_call() throws Throwable { + // We call func() + // The method is declared void (which means return None) + Object r = func.call(); + assertEquals(Py.None, r); + } + } + + /** {@link ExampleModule#f0()} accepts no arguments. */ + @Nested + @DisplayName("static, with no parameters") + class StaticNoParams extends NoParams { + + @Override + @BeforeEach + void setup() throws AttributeError, Throwable { + // func = module.f0 + func = (PyJavaFunction)Abstract.getAttr(module, "f0"); + ap = func.argParser; + } + + @Override + @Test + void has_expected_fields() { no_collector_static("f0", 0, 0); } + } + + /** + * {@link ExampleModule#m1(double)} accepts one argument that + * must be given by position. + */ + @Nested + @DisplayName("with one positional-only parameter") + class OneParam extends Standard { + + @BeforeEach + void setup() throws AttributeError, Throwable { + // func = module.m1 + func = (PyJavaFunction)Abstract.getAttr(module, "m1"); + ap = func.argParser; + exp = new Object[] {module, 42.0}; + } + + @Override + @Test + void has_expected_fields() { no_collector_instance("m1", 1, 1); } + + @Override + @Test + void supports__call__() throws Throwable { + // We call func(42.0) + Object[] args = {42.0}; + // The method reports its arguments as a tuple + PyTuple r = (PyTuple)func.__call__(args, null); + check_result(r); + } + + @Override + @Test + void supports_keywords() throws Throwable { + // We call func(42.0) + Object[] args = {42.0}; + String[] names = {}; + // The method reports its arguments as a tuple + PyTuple r = (PyTuple)func.__call__(args, names); + check_result(r); + } + + @Override + @Test + void raises_TypeError_on_unexpected_keyword() { + // We call func(o, a=42.0) + Object o = new ExampleModule(); + Object[] args = {o, 42.0}; + String[] names = {"a"}; + + assertThrows(TypeError.class, () -> func.__call__(args, names)); + } + + @Override + @Test + void supports_java_call() throws Throwable { + // We call func(42.0) + PyTuple r = (PyTuple)func.call(42.0); + check_result(r); + } + } + + /** + * {@link ExampleModule#f1(double)} accepts one argument that + * must be given by position. + */ + @Nested + @DisplayName("static, with one positional-only parameter") + class StaticOneParam extends OneParam { + + @Override + @BeforeEach + void setup() throws AttributeError, Throwable { + // func = module.f1 + func = (PyJavaFunction)Abstract.getAttr(module, "f1"); + ap = func.argParser; + exp = new Object[] {42.0}; + } + + @Override + @Test + void has_expected_fields() { no_collector_static("f1", 1, 1); } + } + + /** + * {@link ExampleModule#m3(int, String, Object)} accepts 3 arguments + * that must be given by position. + */ + @Nested + @DisplayName("with positional-only parameters by default") + class DefaultPositionalParams extends Standard { + + @BeforeEach + void setup() throws AttributeError, Throwable { + // func = module.m3 + func = (PyJavaFunction)Abstract.getAttr(module, "m3"); + ap = func.argParser; + exp = new Object[] {module, 1, "2", 3}; + } + + @Override + @Test + void has_expected_fields() { no_collector_instance("m3", 3, 3); } + + @Override + @Test + void supports__call__() throws Throwable { + // We call func(1, '2', 3) + Object[] args = {1, "2", 3}; + // The method reports its arguments as a tuple + PyTuple r = (PyTuple)func.__call__(args, null); + check_result(r); + } + + @Override + @Test + void supports_keywords() throws Throwable { + // We call func(1, '2', 3) + Object[] args = {1, "2", 3}; + String[] names = {}; + // The method reports its arguments as a tuple + PyTuple r = (PyTuple)func.__call__(args, names); + check_result(r); + } + + @Override + @Test + void raises_TypeError_on_unexpected_keyword() { + // We call func(o, 1, '2', c=3) + Object o = new ExampleModule(); + Object[] args = {o, 1, "2", 3}; + String[] names = {"c"}; + + assertThrows(TypeError.class, () -> func.__call__(args, names)); + } + + @Override + @Test + void supports_java_call() throws Throwable { + // We call func(1, '2', 3) + PyTuple r = (PyTuple)func.call(1, "2", 3); + check_result(r); + } + } + + /** + * {@link ExampleModule#f3(int, String, Object)} accepts 3 arguments + * that must be given by position. + */ + @Nested + @DisplayName("static, with positional-only parameters by default") + class StaticDefaultPositionalParams extends DefaultPositionalParams { + + @Override + @BeforeEach + void setup() throws AttributeError, Throwable { + // func = module.f3 + func = (PyJavaFunction)Abstract.getAttr(module, "f3"); + ap = func.argParser; + exp = new Object[] {1, "2", 3}; + } + + @Override + @Test + void has_expected_fields() { no_collector_static("f3", 3, 3); } + } + + /** + * {@link ExampleModule#m3pk(int, String, Object)} accepts 3 + * arguments that may be given by position or keyword. + */ + @Nested + @DisplayName("with positional-or-keyword parameters") + class PositionalOrKeywordParams extends Standard { + + @BeforeEach + void setup() throws AttributeError, Throwable { + // func = module.m3pk + func = (PyJavaFunction)Abstract.getAttr(module, "m3pk"); + ap = func.argParser; + exp = new Object[] {module, 1, "2", 3}; + } + + @Override + @Test + void has_expected_fields() { no_collector_instance("m3pk", 3, 0); } + + @Override + @Test + void supports__call__() throws Throwable { + // We call func(1, '2', 3) + Object[] args = {1, "2", 3}; + String[] names = {}; + PyTuple r = (PyTuple)func.__call__(args, names); + check_result(r); + } + + /** Supply second and third arguments by keyword. */ + @Override + @Test + void supports_keywords() throws Throwable { + // We call func(1, c=3, b='2') + Object[] args = {1, 3, "2"}; + String[] names = {"c", "b"}; + PyTuple r = (PyTuple)func.__call__(args, names); + check_result(r); + } + + /** Get the wrong keyword. */ + @Override + @Test + void raises_TypeError_on_unexpected_keyword() throws Throwable { + // We call func(1, c=3, b='2', x=4) + Object[] args = {1, 3, "2", 4}; + String[] names = {"c", "b", /* unknown */"x"}; + assertThrows(TypeError.class, () -> func.__call__(args, names)); + } + + @Override + @Test + void supports_java_call() throws Throwable { + PyTuple r = (PyTuple)func.call(1, "2", 3); + check_result(r); + } + } + + /** + * {@link ExampleModule#f3pk(int, String, Object)} accepts 3 + * arguments that may be given by position or keyword. + */ + @Nested + @DisplayName("static, with positional-or-keyword parameters") + class StaticPositionalOrKeywordParams extends PositionalOrKeywordParams { + + @Override + @BeforeEach + void setup() throws AttributeError, Throwable { + // func = module.f3pk + func = (PyJavaFunction)Abstract.getAttr(module, "f3pk"); + ap = func.argParser; + exp = new Object[] {1, "2", 3}; + } + + @Override + @Test + void has_expected_fields() { no_collector_static("f3pk", 3, 0); } + + } + + /** + * {@link ExampleModule#m3p2(int, String, Object)} accepts 3 + * arguments, two of which may be given by position only, and the + * last by either position or keyword. + */ + @Nested + @DisplayName("with two positional-only parameters") + class SomePositionalOnlyParams extends Standard { + + @BeforeEach + void setup() throws AttributeError, Throwable { + // func = module.m3p2 + func = (PyJavaFunction)Abstract.getAttr(module, "m3p2"); + ap = func.argParser; + exp = new Object[] {module, 1, "2", 3}; + } + + @Override + @Test + void has_expected_fields() { no_collector_instance("m3p2", 3, 2); } + + @Override + @Test + void supports__call__() throws Throwable { + // We call func(1, '2', 3) + Object[] args = {1, "2", 3}; + String[] names = {}; + + // The method just parrots its arguments as a tuple + PyTuple r = (PyTuple)func.__call__(args, names); + check_result(r); + } + + /** Supply third argument by keyword. */ + @Override + @Test + void supports_keywords() throws Throwable { + // We call func(1, '2', c=3) + Object[] args = {1, "2", 3}; + String[] names = {"c"}; + + // The method reports its arguments as a tuple + PyTuple r = (PyTuple)func.__call__(args, names); + check_result(r); + } + + @Override + @Test + void raises_TypeError_on_unexpected_keyword() throws Throwable { + // We call func(1, c=3, b='2') + Object[] args = {1, 3, "2"}; + String[] names = {"c", /* positional */"b"}; + assertThrows(TypeError.class, () -> func.__call__(args, names)); + } + + @Override + @Test + void supports_java_call() throws Throwable { + // The method reports its arguments as a tuple + PyTuple r = (PyTuple)func.call(1, "2", 3); + check_result(r); + } + } + + /** + * {@link ExampleModule#f3p2(int, String, Object)} accepts 3 + * arguments, two of which may be given by position only, and the + * last by either position or keyword. + */ + @Nested + @DisplayName("static, with two positional-only parameters") + class StaticSomePositionalOnlyParams extends SomePositionalOnlyParams { + + @Override + @BeforeEach + void setup() throws AttributeError, Throwable { + // func = module.f3p2 + func = (PyJavaFunction)Abstract.getAttr(module, "f3p2"); + ap = func.argParser; + exp = new Object[] {1, "2", 3}; + } + + @Override + @Test + void has_expected_fields() { no_collector_static("f3p2", 3, 2); } + } +} diff --git a/core/src/test/java/org/python/core/ModuleExposerTest.java b/core/src/test/java/org/python/core/ModuleExposerTest.java index f9d46b8d9..dc1713965 100644 --- a/core/src/test/java/org/python/core/ModuleExposerTest.java +++ b/core/src/test/java/org/python/core/ModuleExposerTest.java @@ -213,8 +213,8 @@ private static void checkSignature(Map dict, assertTrue(k > 0); String name = spec.substring(0, k); String expect = spec.substring(k); - PyJavaFunction pjm = (PyJavaFunction)dict.get(name); - assertEquals(expect, pjm.argParser.textSignature()); + PyJavaFunction pjf = (PyJavaFunction)dict.get(name); + assertEquals(expect, pjf.argParser.textSignature()); } } From 27f79d7d0bb4220ebd4e2a7a517af4edcd6a9c87 Mon Sep 17 00:00:00 2001 From: Jeff Allen Date: Mon, 2 Jan 2023 10:19:15 +0000 Subject: [PATCH 4/6] PEP 489-ish extension module initialisation Separate extension module initialisation into create and exec phases, roughly as PEP 489. --- .../java/org/python/core/Interpreter.java | 4 +- .../main/java/org/python/core/JavaModule.java | 29 ++++++++++--- .../main/java/org/python/core/PyModule.java | 43 +++++++++++++++---- .../python/core/ModuleExposerMethodTest.java | 19 ++++---- 4 files changed, 72 insertions(+), 23 deletions(-) diff --git a/core/src/main/java/org/python/core/Interpreter.java b/core/src/main/java/org/python/core/Interpreter.java index 1fe0bb078..1febac47b 100644 --- a/core/src/main/java/org/python/core/Interpreter.java +++ b/core/src/main/java/org/python/core/Interpreter.java @@ -1,3 +1,5 @@ +// Copyright (c)2023 Jython Developers. +// Licensed to PSF under a contributor agreement. package org.python.core; import org.python.base.InterpreterError; @@ -30,7 +32,7 @@ class Interpreter { /** Create a new {@code Interpreter}. */ Interpreter() { // builtinsModule = new BuiltinsModule(); - // builtinsModule.init(); + // builtinsModule.exec(); // addModule(builtinsModule); builtinsModule = null; } diff --git a/core/src/main/java/org/python/core/JavaModule.java b/core/src/main/java/org/python/core/JavaModule.java index dcdef00c2..14d5dda04 100644 --- a/core/src/main/java/org/python/core/JavaModule.java +++ b/core/src/main/java/org/python/core/JavaModule.java @@ -1,4 +1,4 @@ -// Copyright (c)2022 Jython Developers. +// Copyright (c)2023 Jython Developers. // Licensed to PSF under a contributor agreement. package org.python.core; @@ -8,16 +8,35 @@ public abstract class JavaModule extends PyModule { final ModuleDef definition; /** - * Construct the base {@code JavaModule} and fill the module - * dictionary from the given module definition, which is normally - * created during static initialisation of the concrete class - * defining the module. + * Construct the base {@code JavaModule}, saving the module + * definition, which is normally created during static + * initialisation of the concrete class defining the module. In + * terms of PEP 489 phases, the constructor performs the + * {@code Py_mod_create}. We defer filling the module dictionary + * from the definition and other sources until {@link #exec()} is + * called. * * @param definition of the module */ protected JavaModule(ModuleDef definition) { super(definition.name); this.definition = definition; + } + + /** + * {@inheritDoc} + *

+ * In the case of a {@code JavaModule}, the base implementation + * mines the method definitions from the {@link #definition}. The + * module should extend this method, that is call + * {@code super.exec()} to add boilerplate and the methods, then add + * other definitions (typically constants) to the module namespace + * with {@link #add(String, Object) #add(String, Object)}. In terms + * of PEP 489 phases, this is the {@code Py_mod_exec} phase. + */ + @Override + void exec() { + super.exec(); definition.addMembers(this); } } diff --git a/core/src/main/java/org/python/core/PyModule.java b/core/src/main/java/org/python/core/PyModule.java index b22c7f274..b660f51d5 100644 --- a/core/src/main/java/org/python/core/PyModule.java +++ b/core/src/main/java/org/python/core/PyModule.java @@ -1,13 +1,10 @@ +// Copyright (c)2023 Jython Developers. +// Licensed to PSF under a contributor agreement. package org.python.core; import java.lang.invoke.MethodHandles; -import java.util.Map; -/** - * The Python {@code module} object. - *

- * Stop-gap implementation to satisfy use elsewhere in the project. - */ +/** The Python {@code module} object. */ public class PyModule implements CraftedPyObject, DictPyObject { /** The type of Python object this class implements. */ @@ -16,10 +13,10 @@ public class PyModule implements CraftedPyObject, DictPyObject { protected final PyType type; - /** Name of this module. **/ + /** Name of this module. Not {@code null}. **/ final String name; - /** Dictionary (globals) of this module. **/ + /** Dictionary (globals) of this module. Not {@code null}. **/ final PyDict dict; /** @@ -42,12 +39,40 @@ public class PyModule implements CraftedPyObject, DictPyObject { */ PyModule(String name) { this(TYPE, name); } + /** + * Initialise the module instance. The main action will be to add + * entries to {@link #dict}. These become the members (globals) of + * the module. + */ + void exec() {} + @Override public PyType getType() { return type; } + /** + * The global dictionary of a module instance. This is always a + * Python {@code dict} and never {@code null}. + * + * @return The globals of this module + */ @Override - public Map getDict() { return dict; } + public PyDict getDict() { return dict; } @Override public String toString() { return String.format("", name); } + + /** + * Add a type by name to the dictionary. + * + * @param t the type + */ + void add(PyType t) { dict.put(t.getName(), t); } + + /** + * Add an object by name to the module dictionary. + * + * @param name to use as key + * @param o value for key + */ + void add(String name, Object o) { dict.put(name, o); } } diff --git a/core/src/test/java/org/python/core/ModuleExposerMethodTest.java b/core/src/test/java/org/python/core/ModuleExposerMethodTest.java index cccbe3da5..db73c9fbc 100644 --- a/core/src/test/java/org/python/core/ModuleExposerMethodTest.java +++ b/core/src/test/java/org/python/core/ModuleExposerMethodTest.java @@ -1,4 +1,4 @@ -// Copyright (c)2022 Jython Developers. +// Copyright (c)2023 Jython Developers. // Licensed to PSF under a contributor agreement. package org.python.core; @@ -43,7 +43,7 @@ abstract static class Standard { // Working variables for the tests /** The module we create. */ - PyModule module = new ExampleModule(); + final PyModule module; /** The function to examine or call. */ PyJavaFunction func; /** The parser in the function we examine. */ @@ -51,6 +51,11 @@ abstract static class Standard { /** The expected result of calling the function */ Object[] exp; + Standard() { + this.module = new ExampleModule(); + this.module.exec(); + } + /** * A parser attached to the function object should have field values * that correctly reflect the signature and annotations in the @@ -390,9 +395,8 @@ void supports_keywords() throws Throwable { @Override @Test void raises_TypeError_on_unexpected_keyword() { - // We call func(o, a=42.0) - Object o = new ExampleModule(); - Object[] args = {o, 42.0}; + // We call func(42.0, a=5) + Object[] args = {42.0, 5}; String[] names = {"a"}; assertThrows(TypeError.class, () -> func.__call__(args, names)); @@ -473,9 +477,8 @@ void supports_keywords() throws Throwable { @Override @Test void raises_TypeError_on_unexpected_keyword() { - // We call func(o, 1, '2', c=3) - Object o = new ExampleModule(); - Object[] args = {o, 1, "2", 3}; + // We call func(1, '2', c=3) + Object[] args = {1, "2", 3}; String[] names = {"c"}; assertThrows(TypeError.class, () -> func.__call__(args, names)); From 2b12cad79beefd91351222d49d299db6ace81741 Mon Sep 17 00:00:00 2001 From: Jeff Allen Date: Mon, 2 Jan 2023 14:24:06 +0000 Subject: [PATCH 5/6] Begin a builtins module We implement a few built-in functions and object references, and a unit test of the functions. The Interpreter now makes itself an instance of the module so that it is available to the frame. --- .../java/org/python/core/BuiltinsModule.java | 185 ++++++++++++++++++ .../java/org/python/core/Interpreter.java | 5 +- .../main/java/org/python/core/PyUnicode.java | 4 +- .../org/python/core/BuiltinsModuleTest.java | 112 +++++++++++ 4 files changed, 301 insertions(+), 5 deletions(-) create mode 100644 core/src/main/java/org/python/core/BuiltinsModule.java create mode 100644 core/src/test/java/org/python/core/BuiltinsModuleTest.java diff --git a/core/src/main/java/org/python/core/BuiltinsModule.java b/core/src/main/java/org/python/core/BuiltinsModule.java new file mode 100644 index 000000000..87dc896fa --- /dev/null +++ b/core/src/main/java/org/python/core/BuiltinsModule.java @@ -0,0 +1,185 @@ +// Copyright (c)2023 Jython Developers. +// Licensed to PSF under a contributor agreement. +package org.python.core; + +import java.lang.invoke.MethodHandles; +import java.util.Iterator; + +import org.python.core.Exposed.Default; +import org.python.core.Exposed.DocString; +import org.python.core.Exposed.KeywordOnly; +import org.python.core.Exposed.Name; +import org.python.core.Exposed.PositionalCollector; +import org.python.core.Exposed.PythonStaticMethod; + +/** + * The {@code builtins} module is definitely called "builtins". + *

+ * Although it is fully a module, the {@link BuiltinsModule} lives + * in the {@code core} package because it needs privileged access to + * the core implementation that extension modules do not. + */ +class BuiltinsModule extends JavaModule { + + private static final ModuleDef DEFINITION = new ModuleDef("builtins", MethodHandles.lookup()); + + /** Construct an instance of the {@code builtins} module. */ + BuiltinsModule() { + super(DEFINITION); + + // This list is taken from CPython bltinmodule.c + add("None", Py.None); + // add("Ellipsis", Py.Ellipsis); + add("NotImplemented", Py.NotImplemented); + add("False", Py.False); + add("True", Py.True); + add("bool", PyBool.TYPE); + // add("memoryview", PyMemoryView.TYPE); + // add("bytearray", PyByteArray.TYPE); + add("bytes", PyBytes.TYPE); + // add("classmethod", PyClassMethod.TYPE); + // add("complex", PyComplex.TYPE); + add("dict", PyDict.TYPE); + // add("enumerate", PyEnum.TYPE); + // add("filter", PyFilter.TYPE); + add("float", PyFloat.TYPE); + // add("frozenset", PyFrozenSet.TYPE); + // add("property", PyProperty.TYPE); + add("int", PyLong.TYPE); + add("list", PyList.TYPE); + // add("map", PyMap.TYPE); + add("object", PyBaseObject.TYPE); + // add("range", PyRange.TYPE); + // add("reversed", PyReversed.TYPE); + // add("set", PySet.TYPE); + add("slice", PySlice.TYPE); + // add("staticmethod", PyStaticMethod.TYPE); + add("str", PyUnicode.TYPE); + // add("super", PySuper.TYPE); + add("tuple", PyTuple.TYPE); + add("type", PyType.TYPE); + // add("zip", PyZip.TYPE); + } + + @PythonStaticMethod + @DocString("Return the absolute value of the argument.") + static Object abs(Object x) throws Throwable { return PyNumber.absolute(x); } + + @PythonStaticMethod + @DocString("Return the number of items in a container.") + static Object len(Object v) throws Throwable { return PySequence.size(v); } + + /** + * Implementation of {@code max()}. + * + * @param arg1 a first argument or iterable of arguments + * @param args contains other positional arguments + * @param key function + * @param dflt to return when iterable is empty + * @return {@code max} result or {@code dflt} + * @throws Throwable from calling {@code key} or comparison + */ + @PythonStaticMethod(positionalOnly = false) + @DocString("Return the largest item in an iterable" + + " or the largest of two or more arguments.") + // Simplified version of max() + static Object max(Object arg1, @KeywordOnly @Default("None") Object key, + @Name("default") @Default("None") Object dflt, @PositionalCollector PyTuple args) + throws Throwable { + // @PositionalCollector has to be last. + return minmax(arg1, args, key, dflt, Comparison.GT); + } + + /** + * Implementation of {@code min()}. + * + * @param arg1 a first argument or iterable of arguments + * @param args contains other positional arguments + * @param key function + * @param dflt to return when iterable is empty + * @return {@code min} result or {@code dflt} + * @throws Throwable from calling {@code key} or comparison + */ + @PythonStaticMethod(positionalOnly = false) + @DocString("Return the smallest item in an iterable" + + " or the smallest of two or more arguments.") + // Simplified version of min() + static Object min(Object arg1, @KeywordOnly @Default("None") Object key, + @Name("default") @Default("None") Object dflt, @PositionalCollector PyTuple args) + throws Throwable { + // @PositionalCollector has to be last. + return minmax(arg1, args, key, dflt, Comparison.LT); + } + + /** + * Implementation of both + * {@link #min(Object, Object, Object, PyTuple) min()} and + * {@link #max(Object, Object, Object, PyTuple) max()}. + * + * @param arg1 a first argument or iterable of arguments + * @param args contains other positional arguments + * + * @param key function + * @param dflt to return when iterable is empty + * @param op {@code LT} for {@code min} and {@code GT} for + * {@code max}. + * @return min or max result as appropriate + * @throws Throwable from calling {@code op} or {@code key} + */ + // Compare CPython min_max in Python/bltinmodule.c + private static Object minmax(Object arg1, PyTuple args, Object key, Object dflt, Comparison op) + throws Throwable { + + int n = args.size(); + Object result; + Iterator others; + assert key != null; + + if (n > 0) { + /* + * Positional mode: arg1 is the first value, args contains the other + * values to compare + */ + result = key == Py.None ? arg1 : Callables.callFunction(key, arg1); + others = args.iterator(); + if (dflt != Py.None) { + String name = op == Comparison.LT ? "min" : "max"; + throw new TypeError(DEFAULT_WITHOUT_ITERABLE, name); + } + + } else { + // Single iterable argument of the values to compare + result = null; + // XXX define PySequence.iterable like PyMapping.map? + others = PySequence.fastList(arg1, null).iterator(); + } + + // Now we can get on with the comparison + while (others.hasNext()) { + Object item = others.next(); + if (key != Py.None) { item = Callables.callFunction(key, item); } + if (result == null) { + result = item; + } else if (Abstract.richCompareBool(item, result, op)) { result = item; } + } + + // result may be null if the single iterable argument is empty + if (result != null) { + return result; + } else if (dflt != Py.None) { + assert dflt != null; + return dflt; + } else { + String name = op == Comparison.LT ? "min" : "max"; + throw new ValueError("%s() arg is an empty sequence", name); + } + } + + private static final String DEFAULT_WITHOUT_ITERABLE = + "Cannot specify a default for %s() with multiple positional arguments"; + + @PythonStaticMethod + @DocString("Return the canonical string representation of the object.\n" + + "For many object types, including most builtins, eval(repr(obj)) == obj.") + static Object repr(Object obj) throws Throwable { return Abstract.repr(obj); } +} diff --git a/core/src/main/java/org/python/core/Interpreter.java b/core/src/main/java/org/python/core/Interpreter.java index 1febac47b..a4f58b17d 100644 --- a/core/src/main/java/org/python/core/Interpreter.java +++ b/core/src/main/java/org/python/core/Interpreter.java @@ -31,10 +31,9 @@ class Interpreter { /** Create a new {@code Interpreter}. */ Interpreter() { - // builtinsModule = new BuiltinsModule(); - // builtinsModule.exec(); + builtinsModule = new BuiltinsModule(); + builtinsModule.exec(); // addModule(builtinsModule); - builtinsModule = null; } void addModule(PyModule m) { diff --git a/core/src/main/java/org/python/core/PyUnicode.java b/core/src/main/java/org/python/core/PyUnicode.java index 84fdcf6cc..0751d4aeb 100644 --- a/core/src/main/java/org/python/core/PyUnicode.java +++ b/core/src/main/java/org/python/core/PyUnicode.java @@ -1,4 +1,4 @@ -// Copyright (c)2021 Jython Developers. +// Copyright (c)2023 Jython Developers. // Licensed to PSF under a contributor agreement. package org.python.core; @@ -289,7 +289,7 @@ final static PyObject new(PyNewWrapper new_, boolean init, PyType subtype, private static Object __repr__(Object self) { try { // XXX make encode_UnicodeEscape (if needed) take a delegate - return "u" + encode_UnicodeEscape(convertToString(self), true); + return encode_UnicodeEscape(convertToString(self), true); } catch (NoConversion nc) { throw Abstract.impossibleArgumentError("str", self); } diff --git a/core/src/test/java/org/python/core/BuiltinsModuleTest.java b/core/src/test/java/org/python/core/BuiltinsModuleTest.java new file mode 100644 index 000000000..188417495 --- /dev/null +++ b/core/src/test/java/org/python/core/BuiltinsModuleTest.java @@ -0,0 +1,112 @@ +package org.python.core; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNotSame; +import static org.junit.jupiter.api.Assertions.assertSame; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; + +/** + * This is a test of instantiating and using the {@code builtins} + * module, which has a special place in the Python interpreter as the + * name space. Many built-in types and functions are named there for use + * by the Python interpreter and it is effectively implicitly imported. + */ +@DisplayName("The builtins module") +class BuiltinsModuleTest extends UnitTestSupport { + + static final String FILE = "BuiltinsModuleTest.java"; + + @Test + @DisplayName("exists on an interepreter") + @SuppressWarnings("static-method") + void existsOnInterpreter() { + Interpreter interp = new Interpreter(); + PyModule builtins = interp.builtinsModule; + assertNotNull(builtins); + } + + @Test + @DisplayName("has independent instances") + @SuppressWarnings("static-method") + void canBeInstantiated() { + Interpreter interp1 = new Interpreter(); + Interpreter interp2 = new Interpreter(); + // Look up an arbitrary function in each interpreter + PyJavaFunction abs1 = (PyJavaFunction)interp1.getBuiltin("abs"); + assertSame(abs1.self, interp1.builtinsModule); + PyJavaFunction abs2 = (PyJavaFunction)interp2.getBuiltin("abs"); + assertSame(abs2.self, interp2.builtinsModule); + // Each module provides distinct function objects + assertNotSame(abs1, abs2); + // builtins module instances are distinct + assertNotSame(interp1.builtinsModule, interp2.builtinsModule); + } + + @Nested + @DisplayName("provides expected function ...") + class TestFunctions { + Interpreter interp; + PyDict globals; + /* BuiltinsModule? */ PyModule builtins; + + @BeforeEach + void setup() { + interp = new Interpreter(); + globals = Py.dict(); + builtins = interp.builtinsModule; + } + + + @Test + @DisplayName("abs") + void testAbs() throws Throwable { + Object f = Abstract.getAttr(builtins, "abs"); + Object r = Callables.callFunction(f, -5.0); + assertEquals(5.0, r); + } + + + @Test + @DisplayName("len") + void testLen() throws Throwable { + Object f = Abstract.getAttr(builtins, "len"); + Object r = Callables.callFunction(f, "hello"); + assertEquals(5, r); + } + + @Test + @DisplayName("max") + void testMax() throws Throwable { + Object f = Abstract.getAttr(builtins, "max"); + Object r = Callables.callFunction(f, 4, 4.2, 5.0, 6); + assertEquals(6, r); + r = Callables.callFunction(f, Py.tuple(4, 4.2, 5.0, 6)); + assertEquals(6, r); + } + + @Test + @DisplayName("min") + void testMin() throws Throwable { + Object f = Abstract.getAttr(builtins, "min"); + Object r = Callables.callFunction(f, 4, 5.0, 6, 4.2); + assertEquals(4, r); + r = Callables.callFunction(f, Py.tuple(4, 5.0, 6, 4.2)); + assertEquals(4, r); + } + + @Test + @DisplayName("repr") + void testRepr() throws Throwable { + Object f = Abstract.getAttr(builtins, "repr"); + assertEquals("123", Callables.callFunction(f, 123)); + assertEquals("'spam'", Callables.callFunction(f, "spam")); + // XXX implement None.__repr__ + // assertEquals("None", Callables.callFunction(f, Py.None)); + } + } +} From 7a167a961d1c95f1e47b2bcc12cb17484efa68f0 Mon Sep 17 00:00:00 2001 From: Jeff Allen Date: Mon, 2 Jan 2023 14:36:29 +0000 Subject: [PATCH 6/6] Show access to built-ins from byte code With a few adjustments to the interpreter frame and the function object, we show that we can access the contents of the builtins module as we interpret CPython byte code. --- .../java/org/python/core/CPython38Frame.java | 27 ++++++++++++++++-- .../main/java/org/python/core/PyFrame.java | 2 ++ .../java/org/python/core/PyJavaFunction.java | 10 ++++--- .../src/main/java/org/python/core/PyType.java | 2 ++ .../org/python/core/CPython38CodeTest.java | 9 ++++-- .../src/test/pythonExample/builtins_module.py | 28 +++++++++++++++++++ 6 files changed, 69 insertions(+), 9 deletions(-) create mode 100644 core/src/test/pythonExample/builtins_module.py diff --git a/core/src/main/java/org/python/core/CPython38Frame.java b/core/src/main/java/org/python/core/CPython38Frame.java index 143365abb..10c591155 100644 --- a/core/src/main/java/org/python/core/CPython38Frame.java +++ b/core/src/main/java/org/python/core/CPython38Frame.java @@ -1,4 +1,4 @@ -// Copyright (c)2022 Jython Developers. +// Copyright (c)2023 Jython Developers. // Licensed to PSF under a contributor agreement. package org.python.core; @@ -155,11 +155,21 @@ Object eval() { try { locals.put(name, s[--sp]); } catch (NullPointerException npe) { - throw new SystemError("no locals found when storing '%s'", name); + throw noLocals("storing", name); } oparg = 0; break; + case Opcode.DELETE_NAME: + name = names[oparg | opword & 0xff]; + oparg = 0; + try { + locals.remove(name); + } catch (NullPointerException npe) { + throw noLocals("deleting", name); + } + break; + case Opcode.BUILD_MAP: // k1 | v1 | ... | kN | vN | -> | map | // -------------------------^sp -------^sp @@ -355,7 +365,7 @@ Object eval() { * regular method in it. {@code CALL_METHOD} will detect and use * this optimised form if the first element is not {@code null}. * - * @param obj of whichg the callable is an attribute + * @param obj of which the callable is an attribute * @param name of callable attribute * @param offset in stack at which to place results * @throws AttributeError ifthe named attribute does not exist @@ -463,4 +473,15 @@ private void getMethod(Object obj, String name, int offset) throws AttributeErro // All the look-ups and descriptors came to nothing :( throw Abstract.noAttributeError(obj, name); } + + /** + * Generate error to throw when we cannot access locals. + * + * @param action "loading", "storing" or "deleting" + * @param name variable name + * @return + */ + private static SystemError noLocals(String action, String name) { + return new SystemError("no locals found when %s '%s'", name); + } } diff --git a/core/src/main/java/org/python/core/PyFrame.java b/core/src/main/java/org/python/core/PyFrame.java index 250904e8f..a6966e817 100644 --- a/core/src/main/java/org/python/core/PyFrame.java +++ b/core/src/main/java/org/python/core/PyFrame.java @@ -172,6 +172,8 @@ protected PyFrame(Interpreter interpreter, C code, PyDict globals, Object locals */ this.locals = PyMapping.map(locals); } + // Fix up the builtins module dictionary (simplified) + this.builtins = interpreter.builtinsModule.getDict(); } /** diff --git a/core/src/main/java/org/python/core/PyJavaFunction.java b/core/src/main/java/org/python/core/PyJavaFunction.java index 6de09cda8..08e2a93e4 100644 --- a/core/src/main/java/org/python/core/PyJavaFunction.java +++ b/core/src/main/java/org/python/core/PyJavaFunction.java @@ -6,9 +6,11 @@ import java.lang.invoke.MethodHandles; import java.util.List; -import org.python.core.ArgumentError.Mode; import org.python.base.InterpreterError; import org.python.base.MethodKind; +import org.python.core.ArgumentError.Mode; +import org.python.core.Exposed.Getter; +import org.python.core.Exposed.Member; /** * The Python {@code builtin_function_or_method} object. Java @@ -35,6 +37,7 @@ public abstract class PyJavaFunction implements CraftedPyObject, FastCall { * ({@code object} or {@code type}). A function obtained from a * module may be a method bound to an instance of that module. */ + @Member("__self__") final Object self; /** @@ -191,7 +194,7 @@ Object __call__(Object[] args, String[] names) throws TypeError, Throwable { /** @return name of the function or method */ // Compare CPython meth_get__name__ in methodobject.c - // @Exposed.Getter + @Getter String __name__() { return argParser.name; } // plumbing ------------------------------------------------------ @@ -319,8 +322,7 @@ public Object call(Object[] args) throws TypeError, Throwable { // Save some indirection by specialising to positional @Override - Object __call__(Object[] args, String[] names) - throws TypeError, Throwable { + Object __call__(Object[] args, String[] names) throws TypeError, Throwable { try { if (names == null || names.length == 0) { // It is *not* worth unpacking the array here diff --git a/core/src/main/java/org/python/core/PyType.java b/core/src/main/java/org/python/core/PyType.java index 1e0ac018c..a1ca89af5 100644 --- a/core/src/main/java/org/python/core/PyType.java +++ b/core/src/main/java/org/python/core/PyType.java @@ -16,6 +16,7 @@ import java.util.Map; import org.python.base.InterpreterError; +import org.python.core.Exposed.Getter; import org.python.core.Slot.Signature; /** @@ -624,6 +625,7 @@ protected void updateAfterSetAttr(String name) { * * @return name of this type */ + @Getter("__name__") public String getName() { return name; } /** diff --git a/core/src/test/java/org/python/core/CPython38CodeTest.java b/core/src/test/java/org/python/core/CPython38CodeTest.java index 32063b249..52a20a621 100644 --- a/core/src/test/java/org/python/core/CPython38CodeTest.java +++ b/core/src/test/java/org/python/core/CPython38CodeTest.java @@ -1,3 +1,5 @@ +// Copyright (c)2023 Jython Developers. +// Licensed to PSF under a contributor agreement. package org.python.core; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -117,11 +119,14 @@ void co_consts() { @SuppressWarnings("static-method") @DisplayName("We can execute ...") @ParameterizedTest(name = "{0}.py") - @ValueSource(strings = {"load_store_name", "unary_op", "binary_op", "call_method_builtin"}) + @ValueSource(strings = {"load_store_name", "unary_op", "binary_op", "call_method_builtin", + "builtins_module"}) void executeSimple(String name) { CPython38Code code = readCode(name); + Interpreter interp = new Interpreter(); PyDict globals = new PyDict(); - code.createFrame(null, globals, globals).eval(); + PyFrame f = code.createFrame(interp, globals, globals); + f.eval(); assertExpectedVariables(readResultDict(name), globals); } diff --git a/core/src/test/pythonExample/builtins_module.py b/core/src/test/pythonExample/builtins_module.py new file mode 100644 index 000000000..043601492 --- /dev/null +++ b/core/src/test/pythonExample/builtins_module.py @@ -0,0 +1,28 @@ +# builtins_module.py +# +# The focus of this test is the way the interpreter resolves names +# in the builtins dictionary (after local and global namespaces). +# This happens in opcodes LOAD_NAME and LOAD_GLOBAL. + +# Access sample objects from the builtins module implicitly +# Opcode is LOAD_NAME + +int_name = int.__name__ +max_name = max.__name__ + +# Call functions to prove we can +# Opcode is LOAD_NAME +ai = abs(-42) +af = abs(-41.9) + + +# Sometimes __builtins__ is not the builtins module. Find it with: +bi = max.__self__ + +# Check explicit attribute access to the (real) builtins module +bi_int_name = bi.int.__name__ +bi_max_name = bi.max.__name__ + + +# Not marshallable +del bi