Skip to content

Commit 4d72c3c

Browse files
alban-auzeillpynicolas
alban-auzeill
authored and
pynicolas
committed
SONARPY-227 SONARPY-246 Fix import of coverage results (SonarSource#127)
* SONARPY-227 Fix compatibility with Coverage.py * Fix from review
1 parent 4216962 commit 4d72c3c

7 files changed

Lines changed: 157 additions & 87 deletions

File tree

‎sonar-python-plugin/src/main/java/org/sonar/plugins/python/coverage/CoberturaParser.java‎

Lines changed: 41 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -36,45 +36,69 @@ public class CoberturaParser {
3636

3737
private static final Logger LOG = LoggerFactory.getLogger(CoberturaParser.class);
3838

39+
private int unresolvedFilenameCount;
40+
3941
public void parseReport(File xmlFile, SensorContext context, final Map<InputFile, NewCoverage> coverageData) throws XMLStreamException {
4042
LOG.info("Parsing report '{}'", xmlFile);
43+
unresolvedFilenameCount = 0;
4144

4245
StaxParser parser = new StaxParser(rootCursor -> {
46+
File baseDirectory = context.fileSystem().baseDir();
4347
try {
4448
rootCursor.advance();
4549
} catch (com.ctc.wstx.exc.WstxEOFException eofExc) {
4650
LOG.debug("Unexpected end of file is encountered", eofExc);
4751
throw new EmptyReportException();
4852
}
49-
collectPackageMeasures(rootCursor.descendantElementCursor("package"), context, coverageData);
53+
SMInputCursor cursor = rootCursor.childElementCursor();
54+
while (cursor.getNext() != null) {
55+
if ("sources".equals(cursor.getLocalName())) {
56+
baseDirectory = extractBaseDirectory(cursor, baseDirectory);
57+
} else if ("packages".equals(cursor.getLocalName())) {
58+
collectFileMeasures(cursor.descendantElementCursor("class"), context, coverageData, baseDirectory);
59+
}
60+
}
5061
});
5162
parser.parse(xmlFile);
63+
if (unresolvedFilenameCount > 1) {
64+
LOG.error("Cannot resolve {} file paths, ignoring coverage measures for those files", unresolvedFilenameCount);
65+
}
5266
}
5367

54-
private static void collectPackageMeasures(SMInputCursor pack, SensorContext context, Map<InputFile, NewCoverage> coverageData) throws XMLStreamException {
55-
while (pack.getNext() != null) {
56-
collectFileMeasures(pack.descendantElementCursor("class"), context, coverageData);
68+
private static File extractBaseDirectory(SMInputCursor sources, File defaultBaseDirectory) throws XMLStreamException {
69+
SMInputCursor source = sources.childElementCursor("source");
70+
while (source.getNext() != null) {
71+
String path = source.collectDescendantText();
72+
if (!StringUtils.isBlank(path)) {
73+
File baseDirectory = new File(path);
74+
if (baseDirectory.isDirectory()) {
75+
return baseDirectory;
76+
} else {
77+
LOG.warn("Invalid directory path in 'source' element: {}", path);
78+
}
79+
}
5780
}
81+
return defaultBaseDirectory;
5882
}
5983

60-
private static void collectFileMeasures(SMInputCursor classCursor, SensorContext context, Map<InputFile, NewCoverage> coverageData) throws XMLStreamException {
84+
private void collectFileMeasures(SMInputCursor classCursor, SensorContext context, Map<InputFile, NewCoverage> coverageData, File baseDirectory)
85+
throws XMLStreamException {
6186
while (classCursor.getNext() != null) {
62-
String fileName = classCursor.getAttrValue("filename");
63-
InputFile inputFile = context.fileSystem().inputFile(context.fileSystem().predicates().hasPath(fileName));
64-
87+
File file = new File(classCursor.getAttrValue("filename"));
88+
if (!file.isAbsolute()) {
89+
file = new File(baseDirectory, file.getPath());
90+
}
91+
InputFile inputFile = context.fileSystem().inputFile(context.fileSystem().predicates().hasAbsolutePath(file.getAbsolutePath()));
6592
if (inputFile != null) {
66-
NewCoverage coverage = coverageData.get(inputFile);
67-
if (coverage == null) {
68-
coverage = context.newCoverage().onFile(inputFile);
69-
coverageData.put(inputFile, coverage);
70-
}
93+
NewCoverage coverage = coverageData.computeIfAbsent(inputFile, f -> context.newCoverage().onFile(f));
7194
collectFileData(classCursor, coverage);
72-
7395
} else {
74-
LOG.debug("Cannot find the file '{}', ignoring coverage measures", fileName);
75-
classCursor.getNext();
96+
classCursor.advance();
97+
unresolvedFilenameCount++;
98+
if (unresolvedFilenameCount == 1) {
99+
LOG.error("Cannot find the file '{}' in the base directory '{}', ignoring coverage measures for this file", file.getPath(), baseDirectory.getPath());
100+
}
76101
}
77-
78102
}
79103
}
80104

‎sonar-python-plugin/src/main/java/org/sonar/plugins/python/coverage/PythonCoverageSensor.java‎

Lines changed: 29 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
package org.sonar.plugins.python.coverage;
2121

2222
import java.io.File;
23+
import java.util.ArrayList;
2324
import java.util.HashMap;
2425
import java.util.HashSet;
2526
import java.util.List;
@@ -33,7 +34,6 @@
3334
import org.sonar.api.batch.fs.FileSystem;
3435
import org.sonar.api.batch.fs.InputFile;
3536
import org.sonar.api.batch.sensor.SensorContext;
36-
import org.sonar.api.batch.sensor.coverage.CoverageType;
3737
import org.sonar.api.batch.sensor.coverage.NewCoverage;
3838
import org.sonar.api.config.Settings;
3939
import org.sonar.plugins.python.EmptyReportException;
@@ -53,105 +53,73 @@ public class PythonCoverageSensor {
5353
public static final String OVERALL_DEFAULT_REPORT_PATH = "coverage-reports/overall-coverage-*.xml";
5454
public static final String FORCE_ZERO_COVERAGE_KEY = "sonar.python.coverage.forceZeroCoverage";
5555

56-
private CoberturaParser parser = new CoberturaParser();
57-
5856
public void execute(SensorContext context, Map<InputFile, Set<Integer>> linesOfCode) {
5957
String baseDir = context.fileSystem().baseDir().getPath();
6058
Settings settings = context.settings();
6159

62-
LOG.info("Python unit test coverage");
63-
List<File> reports = getReports(settings, baseDir, REPORT_PATH_KEY, DEFAULT_REPORT_PATH);
64-
Map<InputFile, NewCoverage> coverageMeasures = parseReports(reports, context);
65-
HashSet<InputFile> filesCoveredByUT = new HashSet<>();
66-
saveMeasures(coverageMeasures, filesCoveredByUT, CoverageType.UNIT);
67-
68-
LOG.info("Python integration test coverage");
69-
List<File> itReports = getReports(settings, baseDir, IT_REPORT_PATH_KEY, IT_DEFAULT_REPORT_PATH);
70-
Map<InputFile, NewCoverage> itCoverageMeasures = parseReports(itReports, context);
71-
HashSet<InputFile> filesCoveredByIT = new HashSet<>();
72-
saveMeasures(itCoverageMeasures, filesCoveredByIT, CoverageType.IT);
73-
74-
LOG.info("Python overall test coverage");
75-
List<File> overallReports = getReports(settings, baseDir, OVERALL_REPORT_PATH_KEY, OVERALL_DEFAULT_REPORT_PATH);
76-
Map<InputFile, NewCoverage> overallCoverageMeasures = parseReports(overallReports, context);
77-
HashSet<InputFile> filesCoveredOverall = new HashSet<>();
78-
saveMeasures(overallCoverageMeasures, filesCoveredOverall, CoverageType.OVERALL);
79-
60+
HashSet<InputFile> filesCovered = new HashSet<>();
61+
List<File> reports = new ArrayList<>();
62+
reports.addAll(getReports(settings, baseDir, REPORT_PATH_KEY, DEFAULT_REPORT_PATH));
63+
reports.addAll(getReports(settings, baseDir, IT_REPORT_PATH_KEY, IT_DEFAULT_REPORT_PATH));
64+
reports.addAll(getReports(settings, baseDir, OVERALL_REPORT_PATH_KEY, OVERALL_DEFAULT_REPORT_PATH));
65+
if (!reports.isEmpty()) {
66+
LOG.info("Python test coverage");
67+
for (File report : reports) {
68+
Map<InputFile, NewCoverage> coverageMeasures = parseReport(report, context);
69+
saveMeasures(coverageMeasures, filesCovered);
70+
}
71+
}
8072
if (settings.getBoolean(FORCE_ZERO_COVERAGE_KEY)) {
8173
LOG.debug("Zeroing coverage information for untouched files");
82-
zeroMeasuresWithoutReports(context, filesCoveredByUT, filesCoveredByIT, filesCoveredOverall, linesOfCode);
74+
zeroMeasuresWithoutReports(context, filesCovered, linesOfCode);
8375
}
8476
}
8577

86-
private static void zeroMeasuresWithoutReports(
87-
SensorContext context,
88-
HashSet<InputFile> filesCoveredByUT,
89-
HashSet<InputFile> filesCoveredByIT,
90-
HashSet<InputFile> filesCoveredOverall,
91-
Map<InputFile, Set<Integer>> linesOfCode
92-
) {
78+
private static void zeroMeasuresWithoutReports(SensorContext context, HashSet<InputFile> filesCovered, Map<InputFile, Set<Integer>> linesOfCode) {
9379
FileSystem fileSystem = context.fileSystem();
9480
FilePredicates p = fileSystem.predicates();
9581
Iterable<InputFile> inputFiles = fileSystem.inputFiles(p.and(p.hasType(InputFile.Type.MAIN), p.hasLanguage(Python.KEY)));
9682
for (InputFile inputFile : inputFiles) {
97-
Set<Integer> linesOfCodeForFile = linesOfCode.get(inputFile);
98-
99-
if (!filesCoveredByUT.contains(inputFile)) {
100-
saveZeroValueForResource(inputFile, context, CoverageType.UNIT, linesOfCodeForFile);
101-
}
102-
103-
if (!filesCoveredByIT.contains(inputFile)) {
104-
saveZeroValueForResource(inputFile, context, CoverageType.IT, linesOfCodeForFile);
105-
}
106-
107-
if (!filesCoveredOverall.contains(inputFile)) {
108-
saveZeroValueForResource(inputFile, context, CoverageType.OVERALL, linesOfCodeForFile);
83+
if (!filesCovered.contains(inputFile)) {
84+
saveZeroValueForResource(inputFile, context, linesOfCode.get(inputFile));
10985
}
11086
}
11187
}
11288

113-
private static void saveZeroValueForResource(InputFile inputFile, SensorContext context, CoverageType ctype, @Nullable Set<Integer> linesOfCode) {
89+
private static void saveZeroValueForResource(InputFile inputFile, SensorContext context, @Nullable Set<Integer> linesOfCode) {
11490
if (linesOfCode != null) {
11591
if (LOG.isDebugEnabled()) {
116-
LOG.debug("Zeroing {} coverage measures for file '{}'", ctype, inputFile.relativePath());
92+
LOG.debug("Zeroing coverage measures for file '{}'", inputFile.relativePath());
11793
}
118-
11994
NewCoverage newCoverage = context.newCoverage()
120-
.onFile(inputFile)
121-
.ofType(ctype);
95+
.onFile(inputFile);
12296
linesOfCode.forEach((Integer line) -> newCoverage.lineHits(line, 0));
12397
newCoverage.save();
12498
}
12599
}
126100

127-
128-
private Map<InputFile, NewCoverage> parseReports(List<File> reports, SensorContext context) {
101+
private static Map<InputFile, NewCoverage> parseReport(File report, SensorContext context) {
129102
Map<InputFile, NewCoverage> coverageMeasures = new HashMap<>();
130-
for (File report : reports) {
131-
try {
132-
parser.parseReport(report, context, coverageMeasures);
133-
} catch (EmptyReportException e) {
134-
LOG.warn("The report '{}' seems to be empty, ignoring. '{}'", report, e);
135-
} catch (XMLStreamException e) {
136-
throw new IllegalStateException("Error parsing the report '" + report + "'", e);
137-
}
103+
try {
104+
CoberturaParser parser = new CoberturaParser();
105+
parser.parseReport(report, context, coverageMeasures);
106+
} catch (EmptyReportException e) {
107+
LOG.warn("The report '{}' seems to be empty, ignoring. '{}'", report, e);
108+
} catch (XMLStreamException e) {
109+
throw new IllegalStateException("Error parsing the report '" + report + "'", e);
138110
}
139111
return coverageMeasures;
140112
}
141113

142-
private static void saveMeasures(Map<InputFile, NewCoverage> coverageMeasures, HashSet<InputFile> coveredFiles, CoverageType coverageType) {
114+
private static void saveMeasures(Map<InputFile, NewCoverage> coverageMeasures, HashSet<InputFile> coveredFiles) {
143115
for (Map.Entry<InputFile, NewCoverage> entry : coverageMeasures.entrySet()) {
144116
InputFile inputFile = entry.getKey();
145117
coveredFiles.add(inputFile);
146-
147118
if (LOG.isDebugEnabled()) {
148119
LOG.debug("Saving coverage measures for file '{}'", inputFile.relativePath());
149120
}
150-
151121
entry.getValue()
152-
.ofType(coverageType)
153122
.save();
154-
155123
}
156124
}
157125
}

‎sonar-python-plugin/src/test/java/org/sonar/plugins/python/coverage/PythonCoverageSensorTest.java‎

Lines changed: 42 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,13 @@
2222
import com.google.common.base.Charsets;
2323
import com.google.common.collect.ImmutableSet;
2424
import java.io.File;
25+
import java.util.Arrays;
2526
import java.util.HashMap;
27+
import java.util.List;
2628
import java.util.Map;
2729
import java.util.Set;
30+
import java.util.stream.Collectors;
31+
import java.util.stream.IntStream;
2832
import org.junit.Before;
2933
import org.junit.Test;
3034
import org.sonar.api.batch.fs.InputFile;
@@ -39,10 +43,10 @@
3943

4044
public class PythonCoverageSensorTest {
4145

42-
4346
private final String FILE1_KEY = "moduleKey:sources/file1.py";
4447
private final String FILE2_KEY = "moduleKey:sources/file2.py";
4548
private final String FILE3_KEY = "moduleKey:sources/file3.py";
49+
private final String FILE4_KEY = "moduleKey:sources/file4.py";
4650
private SensorContextTester context;
4751
private Settings settings;
4852
private Map<InputFile, Set<Integer>> linesOfCode;
@@ -60,11 +64,13 @@ public void init() {
6064
InputFile inputFile1 = inputFile("sources/file1.py", Type.MAIN);
6165
InputFile inputFile2 = inputFile("sources/file2.py", Type.MAIN);
6266
InputFile inputFile3 = inputFile("sources/file3.py", Type.MAIN);
67+
InputFile inputFile4 = inputFile("sources/file4.py", Type.MAIN);
6368

6469
linesOfCode = new HashMap<>();
6570
linesOfCode.put(inputFile1, ImmutableSet.of(1, 4, 6));
6671
linesOfCode.put(inputFile2, ImmutableSet.of(1, 2, 3, 4, 5, 6));
6772
linesOfCode.put(inputFile3, ImmutableSet.of(1, 3));
73+
linesOfCode.put(inputFile4, ImmutableSet.of(6, 7, 8, 9, 10, 11, 13, 15, 16, 17));
6874
}
6975

7076
private InputFile inputFile(String relativePath, Type type) {
@@ -100,6 +106,7 @@ public void absolute_path() throws Exception {
100106

101107
@Test
102108
public void test_coverage() {
109+
settings.setProperty(PythonCoverageSensor.FORCE_ZERO_COVERAGE_KEY, "false");
103110
coverageSensor.execute(context, linesOfCode);
104111
Integer[] file1Expected = {1, null, null, 0, null, 0};
105112
Integer[] file2Expected = {1, 3, 1, 0, 1, 1};
@@ -108,13 +115,46 @@ public void test_coverage() {
108115
assertThat(context.lineHits(FILE1_KEY, line)).isEqualTo(file1Expected[line - 1]);
109116
assertThat(context.lineHits(FILE2_KEY, line)).isEqualTo(file2Expected[line - 1]);
110117
assertThat(context.lineHits(FILE3_KEY, line)).isNull();
118+
assertThat(context.lineHits(FILE4_KEY, line)).isNull();
111119
}
112120

113121
assertThat(context.conditions(FILE2_KEY, 2)).isNull();
114122
assertThat(context.conditions(FILE2_KEY, 3)).isEqualTo(2);
115123
assertThat(context.coveredConditions(FILE2_KEY, 3)).isEqualTo(1);
116124
}
117125

126+
@Test
127+
public void test_coverage_4_4_2() {
128+
settings.setProperty(PythonCoverageSensor.REPORT_PATH_KEY, "coverage.4.4.2.xml");
129+
settings.setProperty(PythonCoverageSensor.FORCE_ZERO_COVERAGE_KEY, "true");
130+
coverageSensor.execute(context, linesOfCode);
131+
List<Integer> actual = IntStream.range(1, 18).mapToObj(line -> context.lineHits(FILE4_KEY, line)).collect(Collectors.toList());
132+
assertThat(actual).isEqualTo(Arrays.asList(
133+
null, // line 1
134+
null,
135+
null,
136+
null,
137+
null,
138+
1, // line 6
139+
1, // line 7
140+
1, // line 8
141+
0, // line 9
142+
1, // line 10
143+
1, // line 11
144+
null,
145+
0, // line 13
146+
null,
147+
1, // line 15
148+
null, // Coverage.py does not consider line 16 and 17 as LOC, here it's null even when "linesOfCode" considers them as code
149+
null));
150+
151+
assertThat(context.conditions(FILE4_KEY, 7)).isNull();
152+
assertThat(context.conditions(FILE4_KEY, 8)).isEqualTo(2);
153+
assertThat(context.coveredConditions(FILE4_KEY, 8)).isEqualTo(1);
154+
assertThat(context.conditions(FILE4_KEY, 10)).isEqualTo(2);
155+
assertThat(context.coveredConditions(FILE4_KEY, 10)).isEqualTo(1);
156+
}
157+
118158
@Test
119159
public void test_unresolved_path() {
120160
settings.setProperty(PythonCoverageSensor.REPORT_PATH_KEY, "coverage_with_unresolved_path.xml");
@@ -166,6 +206,7 @@ public void should_fail_on_unexpected_eof() {
166206
public void should_do_nothing_on_empty_report() {
167207
settings.setProperty(PythonCoverageSensor.REPORT_PATH_KEY, "empty-coverage-result.xml");
168208
settings.setProperty(PythonCoverageSensor.IT_REPORT_PATH_KEY, "this-file-does-not-exist.xml");
209+
settings.setProperty(PythonCoverageSensor.FORCE_ZERO_COVERAGE_KEY, "false");
169210
coverageSensor.execute(context, linesOfCode);
170211

171212
assertThat(context.lineHits(FILE1_KEY, 1)).isNull();

‎sonar-python-plugin/src/test/resources/org/sonar/plugins/python/coverage-reports/coverage.4.4.2.xml‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
<?xml version="1.0" ?>
2+
<coverage branch-rate="0.5" branches-covered="2" branches-valid="4" complexity="0" line-rate="0.75" lines-covered="6" lines-valid="8" timestamp="1515594208483" version="4.4.2">
3+
<!-- Generated by coverage.py: https://coverage.readthedocs.io -->
4+
<!-- Based on https://raw.githubusercontent.com/cobertura/web/master/htdocs/xml/coverage-04.dtd -->
5+
<sources>
6+
<source>src/test/resources/org/sonar/plugins/python/coverage-reports/sources</source>
7+
</sources>
8+
<packages>
9+
<package branch-rate="0.5" complexity="0" line-rate="0.75" name=".">
10+
<classes>
11+
<class branch-rate="0.5" complexity="0" filename="file4.py" line-rate="0.75" name="file4.py">
12+
<methods/>
13+
<lines>
14+
<line hits="1" number="6"/>
15+
<line hits="1" number="7"/>
16+
<line branch="true" condition-coverage="50% (1/2)" hits="1" missing-branches="9" number="8"/>
17+
<line hits="0" number="9"/>
18+
<line branch="true" condition-coverage="50% (1/2)" hits="1" missing-branches="13" number="10"/>
19+
<line hits="1" number="11"/>
20+
<line hits="0" number="13"/>
21+
<line hits="1" number="15"/>
22+
</lines>
23+
</class>
24+
</classes>
25+
</package>
26+
</packages>
27+
</coverage>

‎sonar-python-plugin/src/test/resources/org/sonar/plugins/python/coverage-reports/coverage.xml‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,6 @@
22
<!DOCTYPE coverage
33
SYSTEM 'http://cobertura.sourceforge.net/xml/coverage-03.dtd'>
44
<coverage branch-rate="0.5" line-rate="0.27397260274" timestamp="1335184370" version="gcovr 2.5-prerelease (r2774)">
5-
<sources>
6-
<source>
7-
/home/wenns/src/sonar-plugins/cxx/src/samples/SampleProject
8-
</source>
9-
</sources>
105
<packages>
116
<package branch-rate="0.0" complexity="0.0" line-rate="0.0" name="sources.utils">
127
<classes>

‎sonar-python-plugin/src/test/resources/org/sonar/plugins/python/coverage-reports/coverage_with_unresolved_path.xml‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,7 @@
33
SYSTEM 'http://cobertura.sourceforge.net/xml/coverage-03.dtd'>
44
<coverage branch-rate="0.5" line-rate="0.27397260274" timestamp="1335184370" version="gcovr 2.5-prerelease (r2774)">
55
<sources>
6-
<source>
7-
/home/wenns/src/sonar-plugins/cxx/src/samples/SampleProject
8-
</source>
6+
<source>src/test/resources/org/sonar/plugins/python/coverage-reports</source>
97
</sources>
108
<packages>
119
<package branch-rate="0.0" complexity="0.0" line-rate="0.0" name="sources.utils">

0 commit comments

Comments
 (0)