Skip to content

Commit a05916d

Browse files
guillaume-dequenneguillaume-dequenne
authored
SONARPY-630 Support equal specifiers in f-string (Python 3.8) (SonarSource#640)
1 parent 9aaef8a commit a05916d

13 files changed

Lines changed: 202 additions & 28 deletions

File tree

‎python-checks/src/main/java/org/sonar/python/checks/ConfusingWalrusCheck.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
import org.sonar.plugins.python.api.tree.AssignmentExpression;
3131
import org.sonar.plugins.python.api.tree.BaseTreeVisitor;
3232
import org.sonar.plugins.python.api.tree.Expression;
33+
import org.sonar.plugins.python.api.tree.FormattedExpression;
3334
import org.sonar.plugins.python.api.tree.Parameter;
3435
import org.sonar.plugins.python.api.tree.RegularArgument;
3536
import org.sonar.plugins.python.api.tree.StringElement;
@@ -48,8 +49,8 @@ public void initialize(Context context) {
4849

4950
context.registerSyntaxNodeConsumer(Tree.Kind.STRING_ELEMENT, ctx -> {
5051
StringElement stringElement = (StringElement) ctx.syntaxNode();
51-
for (Expression expression : stringElement.interpolatedExpressions()) {
52-
checkNestedWalrus(ctx, expression, String.format(MOVE_MESSAGE, "interpolated expression"));
52+
for (FormattedExpression formattedExpression : stringElement.formattedExpressions()) {
53+
checkNestedWalrus(ctx, formattedExpression.expression(), String.format(MOVE_MESSAGE, "interpolated expression"));
5354
}
5455
});
5556

‎python-frontend/src/main/java/org/sonar/plugins/python/api/tree/BaseTreeVisitor.java‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -317,7 +317,7 @@ public void visitStringLiteral(StringLiteral pyStringLiteralTree) {
317317
@Override
318318
public void visitStringElement(StringElement tree) {
319319
if (tree.isInterpolated()) {
320-
scan(tree.interpolatedExpressions());
320+
scan(tree.formattedExpressions());
321321
}
322322
}
323323

@@ -461,4 +461,9 @@ public void visitDecorator(Decorator decorator) {
461461
public void visitToken(Token token) {
462462
// noop
463463
}
464+
465+
@Override
466+
public void visitFormattedExpression(FormattedExpression formattedExpression) {
467+
scan(formattedExpression.expression());
468+
}
464469
}

‎python-frontend/src/main/java/org/sonar/plugins/python/api/tree/FormattedExpression.java‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
/*
2+
* SonarQube Python Plugin
3+
* Copyright (C) 2011-2020 SonarSource SA
4+
* mailto:info AT sonarsource DOT com
5+
*
6+
* This program is free software; you can redistribute it and/or
7+
* modify it under the terms of the GNU Lesser General Public
8+
* License as published by the Free Software Foundation; either
9+
* version 3 of the License, or (at your option) any later version.
10+
*
11+
* This program is distributed in the hope that it will be useful,
12+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
13+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14+
* Lesser General Public License for more details.
15+
*
16+
* You should have received a copy of the GNU Lesser General Public License
17+
* along with this program; if not, write to the Free Software Foundation,
18+
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
19+
*/
20+
package org.sonar.plugins.python.api.tree;
21+
22+
import javax.annotation.CheckForNull;
23+
24+
public interface FormattedExpression extends Tree {
25+
26+
Expression expression();
27+
28+
/**
29+
* @return Optional equal specifier introduced in Python 3.8
30+
*/
31+
@CheckForNull
32+
Token equalToken();
33+
}

‎python-frontend/src/main/java/org/sonar/plugins/python/api/tree/StringElement.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,5 +42,15 @@ public interface StringElement extends Tree {
4242

4343
boolean isInterpolated();
4444

45+
/**
46+
* @deprecated Use {@link #formattedExpressions()} instead.
47+
*/
48+
@Deprecated
4549
List<Expression> interpolatedExpressions();
50+
51+
/**
52+
* @return Formatted expressions of an f-string.
53+
* Empty list if the string element is not an f-string.
54+
*/
55+
List<FormattedExpression> formattedExpressions();
4656
}

‎python-frontend/src/main/java/org/sonar/plugins/python/api/tree/Tree.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -134,6 +134,8 @@ enum Kind {
134134

135135
STRING_ELEMENT(StringElement.class),
136136

137+
FORMATTED_EXPRESSION(FormattedExpression.class),
138+
137139
TRY_STMT(TryStatement.class),
138140

139141
PARAMETER(Parameter.class),

‎python-frontend/src/main/java/org/sonar/plugins/python/api/tree/TreeVisitor.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -168,4 +168,6 @@ public interface TreeVisitor {
168168
void visitDecorator(Decorator decorator);
169169

170170
void visitToken(Token token);
171+
172+
void visitFormattedExpression(FormattedExpression formattedExpression);
171173
}

‎python-frontend/src/main/java/org/sonar/python/api/PythonGrammar.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,7 @@ public enum PythonGrammar implements GrammarRuleKey {
8282
OR_EXPR,
8383

8484
NAMED_EXPR_TEST,
85+
FORMATTED_EXPR,
8586

8687
COMPARISON,
8788
COMP_OPERATOR,
@@ -198,6 +199,7 @@ public static void grammar(LexerfulGrammarBuilder b) {
198199

199200
b.rule(STAR_EXPR).is("*", EXPR);
200201
b.rule(EXPR).is(XOR_EXPR, b.zeroOrMore("|", XOR_EXPR));
202+
b.rule(FORMATTED_EXPR).is(b.sequence(EXPR, b.optional(PythonPunctuator.ASSIGN)));
201203

202204
b.rule(FACTOR).is(b.firstOf(
203205
b.sequence(b.firstOf("+", "-", "~"), FACTOR),

‎python-frontend/src/main/java/org/sonar/python/tree/FormattedExpressionImpl.java‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
/*
2+
* SonarQube Python Plugin
3+
* Copyright (C) 2011-2020 SonarSource SA
4+
* mailto:info AT sonarsource DOT com
5+
*
6+
* This program is free software; you can redistribute it and/or
7+
* modify it under the terms of the GNU Lesser General Public
8+
* License as published by the Free Software Foundation; either
9+
* version 3 of the License, or (at your option) any later version.
10+
*
11+
* This program is distributed in the hope that it will be useful,
12+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
13+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14+
* Lesser General Public License for more details.
15+
*
16+
* You should have received a copy of the GNU Lesser General Public License
17+
* along with this program; if not, write to the Free Software Foundation,
18+
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
19+
*/
20+
package org.sonar.python.tree;
21+
22+
import java.util.List;
23+
import java.util.Objects;
24+
import java.util.stream.Collectors;
25+
import java.util.stream.Stream;
26+
import javax.annotation.Nullable;
27+
import org.sonar.plugins.python.api.tree.Expression;
28+
import org.sonar.plugins.python.api.tree.FormattedExpression;
29+
import org.sonar.plugins.python.api.tree.Token;
30+
import org.sonar.plugins.python.api.tree.Tree;
31+
import org.sonar.plugins.python.api.tree.TreeVisitor;
32+
33+
public class FormattedExpressionImpl extends PyTree implements FormattedExpression {
34+
35+
private final Expression expression;
36+
private final Token equalToken;
37+
38+
public FormattedExpressionImpl(Expression expression, @Nullable Token equalToken) {
39+
this.expression = expression;
40+
this.equalToken = equalToken;
41+
}
42+
43+
@Override
44+
public Expression expression() {
45+
return this.expression;
46+
}
47+
48+
@Override
49+
public Token equalToken() {
50+
return this.equalToken;
51+
}
52+
53+
@Override
54+
List<Tree> computeChildren() {
55+
return Stream.of(expression, equalToken).filter(Objects::nonNull).collect(Collectors.toList());
56+
}
57+
58+
@Override
59+
public void accept(TreeVisitor visitor) {
60+
visitor.visitFormattedExpression(this);
61+
}
62+
63+
@Override
64+
public Kind getKind() {
65+
return Kind.FORMATTED_EXPRESSION;
66+
}
67+
}

‎python-frontend/src/main/java/org/sonar/python/tree/PythonTreeMaker.java‎

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@
6060
import org.sonar.plugins.python.api.tree.FileInput;
6161
import org.sonar.plugins.python.api.tree.FinallyClause;
6262
import org.sonar.plugins.python.api.tree.ForStatement;
63+
import org.sonar.plugins.python.api.tree.FormattedExpression;
6364
import org.sonar.plugins.python.api.tree.FunctionDef;
6465
import org.sonar.plugins.python.api.tree.GlobalStatement;
6566
import org.sonar.plugins.python.api.tree.IfStatement;
@@ -1280,28 +1281,31 @@ private Expression stringLiteral(AstNode astNode) {
12801281
.map(node -> new StringElementImpl(toPyToken(node.getToken()))).collect(Collectors.toList());
12811282
stringElements.stream()
12821283
.filter(StringElement::isInterpolated)
1283-
.forEach(se -> interpolatedExpressions(se).forEach(se::addInterpolatedExpression));
1284+
.forEach(se -> interpolatedExpressions(se).forEach(se::addFormattedExpression));
12841285
List<StringElement> stringElems = new ArrayList<>(stringElements);
12851286
return new StringLiteralImpl(stringElems);
12861287
}
12871288

1288-
private List<Expression> interpolatedExpressions(StringElementImpl se) {
1289+
private List<FormattedExpression> interpolatedExpressions(StringElementImpl se) {
12891290
String literalValue = se.trimmedQuotesValue();
12901291
Token token = se.firstToken();
12911292
int startOfLiteral = token.value().indexOf(literalValue);
12921293
LineOffsetCounter lineOffsetCounter = new LineOffsetCounter(literalValue);
1293-
List<Expression> res = new ArrayList<>();
1294-
parser.setRootRule(parser.getGrammar().rule(PythonGrammar.EXPR));
1294+
List<FormattedExpression> res = new ArrayList<>();
1295+
parser.setRootRule(parser.getGrammar().rule(PythonGrammar.FORMATTED_EXPR));
12951296
// get escaped interpolation
12961297
Matcher matcher = INTERPOLATED_EXPR_START_PATTERN.matcher(literalValue);
12971298
int index = 0;
12981299
while (matcher.find(index)) {
12991300
int end = matcher.end();
13001301
AstNode parse = parser.parse(literalValue.substring(end));
1301-
Expression exp = expression(parse);
1302-
setParents(exp);
1302+
Expression exp = expression(parse.getFirstChild(PythonGrammar.EXPR));
1303+
AstNode equalNode = parse.getFirstChild(PythonPunctuator.ASSIGN);
1304+
Token equalToken = equalNode == null ? null : toPyToken(equalNode.getToken());
1305+
FormattedExpression formattedExpression = new FormattedExpressionImpl(exp, equalToken);
1306+
setParents(formattedExpression);
13031307
updateTokensLineAndColumn(token, startOfLiteral, lineOffsetCounter, exp, end);
1304-
res.add(exp);
1308+
res.add(formattedExpression);
13051309
com.sonar.sslr.api.Token lastToken = parse.getLastToken();
13061310
index = end + lastToken.getColumn() + lastToken.getValue().length();
13071311
}
@@ -1325,7 +1329,7 @@ private static void updateTokensLineAndColumn(Token token, int startOfLiteral, L
13251329
}
13261330

13271331
private static void adjustNestedInterpolations(StringElement se) {
1328-
se.interpolatedExpressions().forEach(e -> TreeUtils.tokens(e).forEach(t -> {
1332+
se.formattedExpressions().stream().map(FormattedExpression::expression).forEach(e -> TreeUtils.tokens(e).forEach(t -> {
13291333
int newline = t.line() - 1 + se.firstToken().line();
13301334
int col = t.line() == 1 ? (t.column() + se.firstToken().column()) : t.column();
13311335
((TokenImpl) t).setLineColumn(newline, col);

‎python-frontend/src/main/java/org/sonar/python/tree/StringElementImpl.java‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,9 @@
2222
import java.util.ArrayList;
2323
import java.util.Collections;
2424
import java.util.List;
25+
import java.util.stream.Collectors;
2526
import org.sonar.plugins.python.api.tree.Expression;
27+
import org.sonar.plugins.python.api.tree.FormattedExpression;
2628
import org.sonar.plugins.python.api.tree.StringElement;
2729
import org.sonar.plugins.python.api.tree.Token;
2830
import org.sonar.plugins.python.api.tree.Tree;
@@ -32,7 +34,7 @@ public class StringElementImpl extends PyTree implements StringElement {
3234

3335
private final String value;
3436
private final Token token;
35-
private List<Expression> interpolatedExpressions = new ArrayList<>();
37+
private List<FormattedExpression> formattedExpressions = new ArrayList<>();
3638

3739
public StringElementImpl(Token token) {
3840
value = token.value();
@@ -87,11 +89,16 @@ public boolean isInterpolated() {
8789

8890
@Override
8991
public List<Expression> interpolatedExpressions() {
90-
return interpolatedExpressions;
92+
return formattedExpressions.stream().map(FormattedExpression::expression).collect(Collectors.toList());
9193
}
9294

93-
void addInterpolatedExpression(Expression expression) {
94-
interpolatedExpressions.add(expression);
95+
@Override
96+
public List<FormattedExpression> formattedExpressions() {
97+
return formattedExpressions;
98+
}
99+
100+
void addFormattedExpression(FormattedExpression formattedExpression) {
101+
formattedExpressions.add(formattedExpression);
95102
}
96103

97104

0 commit comments

Comments
 (0)