SONARPY-788 Fix parsing error on annotated assignment for Python 3.8 (SonarSource#856)

guillaume-dequenne · web-flow · commit 4363355dd26c · 2020-08-27T14:16:39.000+02:00
diff --git a/python-frontend/src/main/java/org/sonar/python/api/PythonGrammar.java b/python-frontend/src/main/java/org/sonar/python/api/PythonGrammar.java
@@ -186,8 +186,8 @@ public static void grammar(LexerfulGrammarBuilder b) {
         b.sequence(AUGASSIGN, b.firstOf(YIELD_EXPR, TESTLIST)),
         b.zeroOrMore("=", b.firstOf(YIELD_EXPR, TESTLIST_STAR_EXPR))));
 
-    // https://github.com/python/cpython/blob/v3.6.4/Grammar/Grammar#L43
-    b.rule(ANNASSIGN).is(":", TEST, b.optional( "=", TEST));
+    // https://github.com/python/cpython/blob/v3.8.0/Grammar/Grammar#L87
+    b.rule(ANNASSIGN).is(":", TEST, b.optional( "=", b.firstOf(YIELD_EXPR, TESTLIST_STAR_EXPR)));
 
     b.rule(TESTLIST_STAR_EXPR).is(b.firstOf(TEST, STAR_EXPR), b.zeroOrMore(",", b.firstOf(TEST, STAR_EXPR)), b.optional(","));
     b.rule(AUGASSIGN).is(b.firstOf("+=", "-=", "*=", "/=", "//=", "%=", "**=", ">>=", "<<=", "&=", "^=", "|=", "@="));
diff --git a/python-frontend/src/main/java/org/sonar/python/tree/PythonTreeMaker.java b/python-frontend/src/main/java/org/sonar/python/tree/PythonTreeMaker.java
@@ -878,6 +878,9 @@ public Expression expression(AstNode astNode) {
     if (astNode.is(PythonGrammar.ELLIPSIS)) {
       return new EllipsisExpressionImpl(toPyToken(astNode.getTokens()));
     }
+    if (astNode.is(PythonGrammar.TESTLIST_STAR_EXPR)) {
+      return exprListOrTestList(astNode);
+    }
     throw new IllegalStateException("Expression " + astNode.getType() + " not correctly translated to strongly typed AST");
   }
 
diff --git a/python-frontend/src/test/java/org/sonar/python/tree/PythonTreeMakerTest.java b/python-frontend/src/test/java/org/sonar/python/tree/PythonTreeMakerTest.java
@@ -1169,6 +1169,23 @@ public void annotated_assignment() {
     assertThat(((Name) annAssign.annotation().expression()).name()).isEqualTo("string");
     assertThat(annAssign.assignedValue()).isNull();
     assertThat(annAssign.equalToken()).isNull();
+
+    astNode = p.parse("y: int = yield _bar(x)");
+    statementWithSeparator = new StatementWithSeparator(astNode, null);
+    annAssign = treeMaker.annotatedAssignment(statementWithSeparator);
+    assertThat(annAssign.variable().is(Kind.NAME)).isTrue();
+    assertThat(((Name) annAssign.variable()).name()).isEqualTo("y");
+    assertThat(annAssign.assignedValue().is(Kind.YIELD_EXPR)).isTrue();
+    assertThat(((YieldExpression) annAssign.assignedValue()).expressions().size()).isEqualTo(1);
+    assertThat(((YieldExpression) annAssign.assignedValue()).expressions().get(0).is(Kind.CALL_EXPR)).isTrue();
+
+    astNode = p.parse("y: Tuple = a, b, c");
+    statementWithSeparator = new StatementWithSeparator(astNode, null);
+    annAssign = treeMaker.annotatedAssignment(statementWithSeparator);
+    assertThat(annAssign.variable().is(Kind.NAME)).isTrue();
+    assertThat(((Name) annAssign.variable()).name()).isEqualTo("y");
+    assertThat(annAssign.assignedValue().is(Kind.TUPLE)).isTrue();
+    assertThat(((Tuple) annAssign.assignedValue()).elements().size()).isEqualTo(3);
   }
 
   @Test
diff --git a/python-frontend/src/test/resources/parser/own/annassign_yield_expr.py b/python-frontend/src/test/resources/parser/own/annassign_yield_expr.py
@@ -0,0 +1,8 @@
+def _bar(current):
+    return current
+
+def _foo():
+    x: int = 0
+    y: int = yield _bar(x)
+    y: int = yield a, a
+    yield x + (y if y else 0)

Original file line number	Diff line number	Diff line change
`@@ -878,6 +878,9 @@ public Expression expression(AstNode astNode) {`
`878`	`878`	`if (astNode.is(PythonGrammar.ELLIPSIS)) {`
`879`	`879`	`return new EllipsisExpressionImpl(toPyToken(astNode.getTokens()));`
`880`	`880`	`}`
	`881`	`+ if (astNode.is(PythonGrammar.TESTLIST_STAR_EXPR)) {`
	`882`	`+ return exprListOrTestList(astNode);`
	`883`	`+ }`
`881`	`884`	`throw new IllegalStateException("Expression " + astNode.getType() + " not correctly translated to strongly typed AST");`
`882`	`885`	`}`
`883`	`886`