Skip to content

Commit b9a1ed8

Browse files
andreaguarinopynicolas
andreaguarino
authored and
pynicolas
committed
SONARPY-340 Syntax highlighting based on PyCharm frontend
1 parent 117ee1b commit b9a1ed8

9 files changed

Lines changed: 323 additions & 100 deletions

File tree

‎python-frontend/src/main/java/org/sonar/python/frontend/PythonKeyword.java‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
/*
2+
* SonarQube Python Plugin
3+
* Copyright (C) 2011-2019 SonarSource SA
4+
* mailto:info AT sonarsource DOT com
5+
*
6+
* This program is free software; you can redistribute it and/or
7+
* modify it under the terms of the GNU Lesser General Public
8+
* License as published by the Free Software Foundation; either
9+
* version 3 of the License, or (at your option) any later version.
10+
*
11+
* This program is distributed in the hope that it will be useful,
12+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
13+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14+
* Lesser General Public License for more details.
15+
*
16+
* You should have received a copy of the GNU Lesser General Public License
17+
* along with this program; if not, write to the Free Software Foundation,
18+
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
19+
*/
20+
package org.sonar.python.frontend;
21+
22+
import com.jetbrains.python.PyTokenTypes;
23+
import com.jetbrains.python.psi.PyElementType;
24+
import java.util.Arrays;
25+
import java.util.HashSet;
26+
import java.util.Set;
27+
28+
public class PythonKeyword {
29+
30+
private PythonKeyword() {
31+
// empty constructor
32+
}
33+
34+
private static final Set<PyElementType> KEYWORDS_ELEMENT_TYPES = new HashSet<>(Arrays.asList(
35+
PyTokenTypes.AND_KEYWORD,
36+
PyTokenTypes.AS_KEYWORD,
37+
PyTokenTypes.ASSERT_KEYWORD,
38+
PyTokenTypes.BREAK_KEYWORD,
39+
PyTokenTypes.CLASS_KEYWORD,
40+
PyTokenTypes.CONTINUE_KEYWORD,
41+
PyTokenTypes.DEF_KEYWORD,
42+
PyTokenTypes.DEL_KEYWORD,
43+
PyTokenTypes.ELIF_KEYWORD,
44+
PyTokenTypes.ELSE_KEYWORD,
45+
PyTokenTypes.EXCEPT_KEYWORD,
46+
PyTokenTypes.EXEC_KEYWORD,
47+
PyTokenTypes.FINALLY_KEYWORD,
48+
PyTokenTypes.FOR_KEYWORD,
49+
PyTokenTypes.FROM_KEYWORD,
50+
PyTokenTypes.GLOBAL_KEYWORD,
51+
PyTokenTypes.IF_KEYWORD,
52+
PyTokenTypes.IMPORT_KEYWORD,
53+
PyTokenTypes.IN_KEYWORD,
54+
PyTokenTypes.IS_KEYWORD,
55+
PyTokenTypes.LAMBDA_KEYWORD,
56+
PyTokenTypes.NOT_KEYWORD,
57+
PyTokenTypes.OR_KEYWORD,
58+
PyTokenTypes.PASS_KEYWORD,
59+
PyTokenTypes.PRINT_KEYWORD,
60+
PyTokenTypes.RAISE_KEYWORD,
61+
PyTokenTypes.RETURN_KEYWORD,
62+
PyTokenTypes.TRY_KEYWORD,
63+
PyTokenTypes.WITH_KEYWORD,
64+
PyTokenTypes.WHILE_KEYWORD,
65+
PyTokenTypes.YIELD_KEYWORD,
66+
PyTokenTypes.NONE_KEYWORD,
67+
PyTokenTypes.TRUE_KEYWORD,
68+
PyTokenTypes.FALSE_KEYWORD,
69+
PyTokenTypes.NONLOCAL_KEYWORD,
70+
PyTokenTypes.DEBUG_KEYWORD,
71+
PyTokenTypes.ASYNC_KEYWORD,
72+
PyTokenTypes.AWAIT_KEYWORD));
73+
74+
public static boolean isKeyword(PyElementType elementType) {
75+
return KEYWORDS_ELEMENT_TYPES.contains(elementType);
76+
}
77+
}

‎python-frontend/src/main/java/org/sonar/python/frontend/PythonTokenLocation.java‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
/*
2+
* SonarQube Python Plugin
3+
* Copyright (C) 2011-2019 SonarSource SA
4+
* mailto:info AT sonarsource DOT com
5+
*
6+
* This program is free software; you can redistribute it and/or
7+
* modify it under the terms of the GNU Lesser General Public
8+
* License as published by the Free Software Foundation; either
9+
* version 3 of the License, or (at your option) any later version.
10+
*
11+
* This program is distributed in the hope that it will be useful,
12+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
13+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14+
* Lesser General Public License for more details.
15+
*
16+
* You should have received a copy of the GNU Lesser General Public License
17+
* along with this program; if not, write to the Free Software Foundation,
18+
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
19+
*/
20+
package org.sonar.python.frontend;
21+
22+
import com.intellij.openapi.editor.Document;
23+
import com.intellij.psi.PsiElement;
24+
import org.jetbrains.annotations.NotNull;
25+
26+
public class PythonTokenLocation {
27+
private final int startLine;
28+
private final int startLineOffset;
29+
private final int endLine;
30+
private final int endLineOffset;
31+
32+
public PythonTokenLocation(@NotNull PsiElement element) {
33+
Document psiDocument = element.getContainingFile().getViewProvider().getDocument();
34+
int startOffset = element.getTextRange().getStartOffset();
35+
startLine = psiDocument.getLineNumber(startOffset);
36+
int startLineNumberOffset = psiDocument.getLineStartOffset(startLine);
37+
startLineOffset = startOffset - startLineNumberOffset;
38+
int endOffset = element.getTextRange().getEndOffset();
39+
endLine = psiDocument.getLineNumber(endOffset);
40+
int endLineNumberOffset = psiDocument.getLineStartOffset(endLine);
41+
endLineOffset = endOffset - endLineNumberOffset;
42+
}
43+
44+
public int startLine() {
45+
return startLine + 1;
46+
}
47+
48+
public int startLineOffset() {
49+
return startLineOffset;
50+
}
51+
52+
public int endLine() {
53+
return endLine + 1;
54+
}
55+
56+
public int endLineOffset() {
57+
return endLineOffset;
58+
}
59+
}

‎python-frontend/src/test/java/org/sonar/python/frontend/PythonKeywordTest.java‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
/*
2+
* SonarQube Python Plugin
3+
* Copyright (C) 2011-2019 SonarSource SA
4+
* mailto:info AT sonarsource DOT com
5+
*
6+
* This program is free software; you can redistribute it and/or
7+
* modify it under the terms of the GNU Lesser General Public
8+
* License as published by the Free Software Foundation; either
9+
* version 3 of the License, or (at your option) any later version.
10+
*
11+
* This program is distributed in the hope that it will be useful,
12+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
13+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14+
* Lesser General Public License for more details.
15+
*
16+
* You should have received a copy of the GNU Lesser General Public License
17+
* along with this program; if not, write to the Free Software Foundation,
18+
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
19+
*/
20+
package org.sonar.python.frontend;
21+
22+
import com.jetbrains.python.PyTokenTypes;
23+
import org.junit.Test;
24+
25+
import static org.junit.Assert.*;
26+
import static org.assertj.core.api.Assertions.assertThat;
27+
28+
public class PythonKeywordTest {
29+
30+
@Test
31+
public void isKeyword() {
32+
assertThat(PythonKeyword.isKeyword(PyTokenTypes.DOCSTRING)).isFalse();
33+
assertThat(PythonKeyword.isKeyword(PyTokenTypes.IF_KEYWORD)).isTrue();
34+
}
35+
}

‎python-frontend/src/test/java/org/sonar/python/frontend/PythonTokenLocationTest.java‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
/*
2+
* SonarQube Python Plugin
3+
* Copyright (C) 2011-2019 SonarSource SA
4+
* mailto:info AT sonarsource DOT com
5+
*
6+
* This program is free software; you can redistribute it and/or
7+
* modify it under the terms of the GNU Lesser General Public
8+
* License as published by the Free Software Foundation; either
9+
* version 3 of the License, or (at your option) any later version.
10+
*
11+
* This program is distributed in the hope that it will be useful,
12+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
13+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14+
* Lesser General Public License for more details.
15+
*
16+
* You should have received a copy of the GNU Lesser General Public License
17+
* along with this program; if not, write to the Free Software Foundation,
18+
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
19+
*/
20+
package org.sonar.python.frontend;
21+
22+
import com.intellij.psi.PsiElement;
23+
import org.junit.Test;
24+
25+
import static org.assertj.core.api.Assertions.assertThat;
26+
27+
public class PythonTokenLocationTest {
28+
private PythonParser parser = new PythonParser();
29+
30+
@Test
31+
public void test_multiline() {
32+
PythonTokenLocation tokenLocation = new PythonTokenLocation(getTokens("'''first line\nsecond'''")[0]);
33+
assertOffsets(tokenLocation, 1, 0, 2, 9);
34+
}
35+
36+
@Test
37+
public void test_newline_token() {
38+
PythonTokenLocation tokenLocation = new PythonTokenLocation(getTokens("foo\n")[1]);
39+
assertOffsets(tokenLocation, 1, 3, 2, 0);
40+
}
41+
42+
@Test
43+
public void test_one_line() {
44+
PsiElement[] tokens = getTokens("'''first line'''");
45+
PythonTokenLocation tokenLocation = new PythonTokenLocation(tokens[0]);
46+
assertOffsets(tokenLocation, 1, 0, 1, 16);
47+
48+
tokenLocation = new PythonTokenLocation(getTokens("foo")[0]);
49+
assertOffsets(tokenLocation, 1, 0, 1, 3);
50+
}
51+
52+
@Test
53+
public void test_comment() {
54+
PythonTokenLocation commentLocation = new PythonTokenLocation(getTokens("#comment\n")[0]);
55+
assertOffsets(commentLocation, 1, 0, 1, 8);
56+
}
57+
58+
private static void assertOffsets(PythonTokenLocation tokenLocation, int startLine, int startLineOffset, int endLine, int endLineOffset) {
59+
assertThat(tokenLocation.startLine()).as("start line").isEqualTo(startLine);
60+
assertThat(tokenLocation.startLineOffset()).as("start line offset").isEqualTo(startLineOffset);
61+
assertThat(tokenLocation.endLine()).as("end line").isEqualTo(endLine);
62+
assertThat(tokenLocation.endLineOffset()).as("end line offset").isEqualTo(endLineOffset);
63+
}
64+
65+
private PsiElement[] getTokens(String toLex) {
66+
return parser.parse(toLex).getChildren();
67+
}
68+
}

‎python-frontend/test.sh‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
#!/bin/bash
2+
java -verbose:class -cp target/classes:target/test-classes:\
3+
lib/extensions.jar:\
4+
lib/openapi.jar:\
5+
lib/platform-api.jar:\
6+
lib/platform-impl.jar:\
7+
lib/pycharm.jar:\
8+
lib/pycharm-pydev.jar:\
9+
lib/resources_en.jar:\
10+
lib/util.jar:\
11+
lib/trove4j.jar:\
12+
lib/picocontainer-1.2.jar:\
13+
lib/kotlin-stdlib-1.3.11.jar:\
14+
lib/guava-25.1-jre.jar\
15+
org.sonar.python.frontend.MyTest

‎sonar-python-plugin/pom.xml‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,11 @@
2424
</scm>
2525

2626
<dependencies>
27+
<dependency>
28+
<groupId>org.sonarsource.python</groupId>
29+
<artifactId>python-frontend</artifactId>
30+
<version>${project.version}</version>
31+
</dependency>
2732
<dependency>
2833
<groupId>org.sonarsource.sonarqube</groupId>
2934
<artifactId>sonar-plugin-api</artifactId>
@@ -135,8 +140,8 @@
135140
<configuration>
136141
<rules>
137142
<requireFilesSize>
138-
<maxsize>2800000</maxsize>
139-
<minsize>2600000</minsize>
143+
<maxsize>65000000</maxsize>
144+
<minsize>62000000</minsize>
140145
<files>
141146
<file>${project.build.directory}/${project.build.finalName}.jar</file>
142147
</files>

0 commit comments

Comments
 (0)