Skip to content

Commit 809da40

Browse files
pongadpongad
authored
bigquery: properly detect statistics type (#2742)
Fixes #2357. If the job is incomplete, the statistics itself doesn't give us enough info about what kind of job we're seeing. However, the configuration should, since the request is required to populate it.
1 parent 85038af commit 809da40

4 files changed

Lines changed: 75 additions & 38 deletions

File tree

‎google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/JobInfo.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@
1919
import com.google.api.services.bigquery.model.Job;
2020
import com.google.common.base.Function;
2121
import com.google.common.base.MoreObjects;
22-
2322
import java.io.Serializable;
2423
import java.util.Objects;
2524

@@ -178,7 +177,7 @@ static final class BuilderImpl extends Builder {
178177
this.status = JobStatus.fromPb(jobPb.getStatus());
179178
}
180179
if (jobPb.getStatistics() != null) {
181-
this.statistics = JobStatistics.fromPb(jobPb.getStatistics());
180+
this.statistics = JobStatistics.fromPb(jobPb);
182181
}
183182
this.userEmail = jobPb.getUserEmail();
184183
this.configuration = JobConfiguration.fromPb(jobPb.getConfiguration());

‎google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/JobStatistics.java‎

Lines changed: 29 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -16,13 +16,13 @@
1616

1717
package com.google.cloud.bigquery;
1818

19+
import com.google.api.services.bigquery.model.JobConfiguration;
1920
import com.google.api.services.bigquery.model.JobStatistics2;
2021
import com.google.api.services.bigquery.model.JobStatistics3;
2122
import com.google.api.services.bigquery.model.JobStatistics4;
2223
import com.google.common.base.MoreObjects;
2324
import com.google.common.base.MoreObjects.ToStringHelper;
2425
import com.google.common.collect.Lists;
25-
2626
import java.io.Serializable;
2727
import java.util.List;
2828
import java.util.Objects;
@@ -104,7 +104,9 @@ private Builder() {}
104104

105105
private Builder(com.google.api.services.bigquery.model.JobStatistics statisticsPb) {
106106
super(statisticsPb);
107-
this.destinationUriFileCounts = statisticsPb.getExtract().getDestinationUriFileCounts();
107+
if (statisticsPb.getExtract() != null) {
108+
this.destinationUriFileCounts = statisticsPb.getExtract().getDestinationUriFileCounts();
109+
}
108110
}
109111

110112
Builder setDestinationUriFileCounts(List<Long> destinationUriFileCounts) {
@@ -192,10 +194,12 @@ private Builder() {}
192194

193195
private Builder(com.google.api.services.bigquery.model.JobStatistics statisticsPb) {
194196
super(statisticsPb);
195-
this.inputBytes = statisticsPb.getLoad().getInputFileBytes();
196-
this.inputFiles = statisticsPb.getLoad().getInputFiles();
197-
this.outputBytes = statisticsPb.getLoad().getOutputBytes();
198-
this.outputRows = statisticsPb.getLoad().getOutputRows();
197+
if (statisticsPb.getLoad() != null) {
198+
this.inputBytes = statisticsPb.getLoad().getInputFileBytes();
199+
this.inputFiles = statisticsPb.getLoad().getInputFiles();
200+
this.outputBytes = statisticsPb.getLoad().getOutputBytes();
201+
this.outputRows = statisticsPb.getLoad().getOutputRows();
202+
}
199203
}
200204

201205
Builder setInputBytes(Long inputBytes) {
@@ -332,13 +336,16 @@ private Builder() {}
332336

333337
private Builder(com.google.api.services.bigquery.model.JobStatistics statisticsPb) {
334338
super(statisticsPb);
335-
this.billingTier = statisticsPb.getQuery().getBillingTier();
336-
this.cacheHit = statisticsPb.getQuery().getCacheHit();
337-
this.totalBytesBilled = statisticsPb.getQuery().getTotalBytesBilled();
338-
this.totalBytesProcessed = statisticsPb.getQuery().getTotalBytesProcessed();
339-
if (statisticsPb.getQuery().getQueryPlan() != null) {
340-
this.queryPlan =
341-
Lists.transform(statisticsPb.getQuery().getQueryPlan(), QueryStage.FROM_PB_FUNCTION);
339+
if (statisticsPb.getQuery() != null) {
340+
this.billingTier = statisticsPb.getQuery().getBillingTier();
341+
this.cacheHit = statisticsPb.getQuery().getCacheHit();
342+
this.totalBytesBilled = statisticsPb.getQuery().getTotalBytesBilled();
343+
this.totalBytesProcessed = statisticsPb.getQuery().getTotalBytesProcessed();
344+
if (statisticsPb.getQuery().getQueryPlan() != null) {
345+
this.queryPlan =
346+
Lists.transform(
347+
statisticsPb.getQuery().getQueryPlan(), QueryStage.FROM_PB_FUNCTION);
348+
}
342349
}
343350
}
344351

@@ -577,16 +584,19 @@ com.google.api.services.bigquery.model.JobStatistics toPb() {
577584
}
578585

579586
@SuppressWarnings("unchecked")
580-
static <T extends JobStatistics> T fromPb(
581-
com.google.api.services.bigquery.model.JobStatistics statisticPb) {
582-
if (statisticPb.getLoad() != null) {
587+
static <T extends JobStatistics> T fromPb(com.google.api.services.bigquery.model.Job jobPb) {
588+
JobConfiguration jobConfigPb = jobPb.getConfiguration();
589+
com.google.api.services.bigquery.model.JobStatistics statisticPb = jobPb.getStatistics();
590+
if (jobConfigPb.getLoad() != null) {
583591
return (T) LoadStatistics.fromPb(statisticPb);
584-
} else if (statisticPb.getExtract() != null) {
592+
} else if (jobConfigPb.getExtract() != null) {
585593
return (T) ExtractStatistics.fromPb(statisticPb);
586-
} else if (statisticPb.getQuery() != null) {
594+
} else if (jobConfigPb.getQuery() != null) {
587595
return (T) QueryStatistics.fromPb(statisticPb);
588-
} else {
596+
} else if (jobConfigPb.getCopy() != null) {
589597
return (T) CopyStatistics.fromPb(statisticPb);
598+
} else {
599+
throw new IllegalArgumentException("unknown job configuration: " + jobConfigPb);
590600
}
591601
}
592602
}

‎google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/JobInfoTest.java‎

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -22,19 +22,17 @@
2222
import static org.junit.Assert.assertTrue;
2323

2424
import com.google.cloud.bigquery.JobInfo.CreateDisposition;
25-
import com.google.cloud.bigquery.JobInfo.WriteDisposition;
2625
import com.google.cloud.bigquery.JobInfo.SchemaUpdateOption;
26+
import com.google.cloud.bigquery.JobInfo.WriteDisposition;
2727
import com.google.cloud.bigquery.JobStatistics.CopyStatistics;
2828
import com.google.cloud.bigquery.JobStatistics.ExtractStatistics;
2929
import com.google.cloud.bigquery.JobStatistics.LoadStatistics;
3030
import com.google.cloud.bigquery.JobStatistics.QueryStatistics;
3131
import com.google.common.collect.ImmutableList;
3232
import com.google.common.collect.ImmutableMap;
33-
34-
import org.junit.Test;
35-
3633
import java.util.List;
3734
import java.util.Map;
35+
import org.junit.Test;
3836

3937
public class JobInfoTest {
4038

@@ -311,14 +309,14 @@ public void testToPbAndFromPb() {
311309
assertNull(COPY_JOB.toPb().getConfiguration().getExtract());
312310
assertNull(COPY_JOB.toPb().getConfiguration().getLoad());
313311
assertNull(COPY_JOB.toPb().getConfiguration().getQuery());
314-
assertEquals(COPY_JOB_STATISTICS, JobStatistics.fromPb(COPY_JOB.getStatistics().toPb()));
312+
assertEquals(COPY_JOB_STATISTICS, JobStatistics.fromPb(COPY_JOB.toPb()));
315313
compareJobInfo(COPY_JOB, JobInfo.fromPb(COPY_JOB.toPb()));
316314
assertTrue(JobInfo.fromPb(COPY_JOB.toPb()).getConfiguration() instanceof CopyJobConfiguration);
317315
assertNull(EXTRACT_JOB.toPb().getConfiguration().getCopy());
318316
assertNotNull(EXTRACT_JOB.toPb().getConfiguration().getExtract());
319317
assertNull(EXTRACT_JOB.toPb().getConfiguration().getLoad());
320318
assertNull(EXTRACT_JOB.toPb().getConfiguration().getQuery());
321-
assertEquals(EXTRACT_JOB_STATISTICS, JobStatistics.fromPb(EXTRACT_JOB.getStatistics().toPb()));
319+
assertEquals(EXTRACT_JOB_STATISTICS, JobStatistics.fromPb(EXTRACT_JOB.toPb()));
322320
compareJobInfo(EXTRACT_JOB, JobInfo.fromPb(EXTRACT_JOB.toPb()));
323321
assertTrue(
324322
JobInfo.fromPb(EXTRACT_JOB.toPb()).getConfiguration() instanceof ExtractJobConfiguration);
@@ -327,15 +325,15 @@ public void testToPbAndFromPb() {
327325
assertNull(LOAD_JOB.toPb().getConfiguration().getExtract());
328326
assertNotNull(LOAD_JOB.toPb().getConfiguration().getLoad());
329327
assertNull(LOAD_JOB.toPb().getConfiguration().getQuery());
330-
assertEquals(LOAD_JOB_STATISTICS, JobStatistics.fromPb(LOAD_JOB.getStatistics().toPb()));
328+
assertEquals(LOAD_JOB_STATISTICS, JobStatistics.fromPb(LOAD_JOB.toPb()));
331329
compareJobInfo(LOAD_JOB, JobInfo.fromPb(LOAD_JOB.toPb()));
332330
assertTrue(JobInfo.fromPb(LOAD_JOB.toPb()).getConfiguration() instanceof LoadJobConfiguration);
333331
assertTrue(JobInfo.fromPb(LOAD_JOB.toPb()).getStatistics() instanceof LoadStatistics);
334332
assertNull(QUERY_JOB.toPb().getConfiguration().getCopy());
335333
assertNull(QUERY_JOB.toPb().getConfiguration().getExtract());
336334
assertNull(QUERY_JOB.toPb().getConfiguration().getLoad());
337335
assertNotNull(QUERY_JOB.toPb().getConfiguration().getQuery());
338-
assertEquals(QUERY_JOB_STATISTICS, JobStatistics.fromPb(QUERY_JOB.getStatistics().toPb()));
336+
assertEquals(QUERY_JOB_STATISTICS, JobStatistics.fromPb(QUERY_JOB.toPb()));
339337
compareJobInfo(QUERY_JOB, JobInfo.fromPb(QUERY_JOB.toPb()));
340338
assertTrue(
341339
JobInfo.fromPb(QUERY_JOB.toPb()).getConfiguration() instanceof QueryJobConfiguration);

‎google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/JobStatisticsTest.java‎

Lines changed: 39 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -16,18 +16,17 @@
1616

1717
package com.google.cloud.bigquery;
1818

19+
import static com.google.common.truth.Truth.assertThat;
1920
import static org.junit.Assert.assertEquals;
2021

21-
import com.google.common.collect.ImmutableList;
2222
import com.google.cloud.bigquery.JobStatistics.CopyStatistics;
2323
import com.google.cloud.bigquery.JobStatistics.ExtractStatistics;
2424
import com.google.cloud.bigquery.JobStatistics.LoadStatistics;
2525
import com.google.cloud.bigquery.JobStatistics.QueryStatistics;
2626
import com.google.cloud.bigquery.QueryStage.QueryStep;
27-
28-
import org.junit.Test;
29-
27+
import com.google.common.collect.ImmutableList;
3028
import java.util.List;
29+
import org.junit.Test;
3130

3231
public class JobStatisticsTest {
3332

@@ -157,12 +156,43 @@ public void testToPbAndFromPb() {
157156
ExtractStatistics.fromPb(EXTRACT_STATISTICS.toPb()));
158157
compareLoadStatistics(LOAD_STATISTICS, LoadStatistics.fromPb(LOAD_STATISTICS.toPb()));
159158
compareQueryStatistics(QUERY_STATISTICS, QueryStatistics.fromPb(QUERY_STATISTICS.toPb()));
160-
compareStatistics(COPY_STATISTICS, JobStatistics.fromPb(COPY_STATISTICS.toPb()));
159+
compareStatistics(COPY_STATISTICS, CopyStatistics.fromPb(COPY_STATISTICS.toPb()));
160+
161+
compareLoadStatistics(
162+
LOAD_STATISTICS_INCOMPLETE, LoadStatistics.fromPb(LOAD_STATISTICS_INCOMPLETE.toPb()));
163+
compareQueryStatistics(
164+
QUERY_STATISTICS_INCOMPLETE, QueryStatistics.fromPb(QUERY_STATISTICS_INCOMPLETE.toPb()));
165+
}
161166

162-
compareLoadStatistics(LOAD_STATISTICS_INCOMPLETE,
163-
LoadStatistics.fromPb(LOAD_STATISTICS_INCOMPLETE.toPb()));
164-
compareQueryStatistics(QUERY_STATISTICS_INCOMPLETE,
165-
QueryStatistics.fromPb(QUERY_STATISTICS_INCOMPLETE.toPb()));
167+
@Test
168+
public void testIncomplete() {
169+
// https://github.com/GoogleCloudPlatform/google-cloud-java/issues/2357
170+
com.google.api.services.bigquery.model.Job job =
171+
new com.google.api.services.bigquery.model.Job()
172+
.setStatistics(
173+
new com.google.api.services.bigquery.model.JobStatistics()
174+
.setCreationTime(1234L)
175+
.setStartTime(5678L));
176+
177+
job.setConfiguration(
178+
new com.google.api.services.bigquery.model.JobConfiguration()
179+
.setCopy(new com.google.api.services.bigquery.model.JobConfigurationTableCopy()));
180+
assertThat(JobStatistics.fromPb(job)).isInstanceOf(CopyStatistics.class);
181+
182+
job.setConfiguration(
183+
new com.google.api.services.bigquery.model.JobConfiguration()
184+
.setLoad(new com.google.api.services.bigquery.model.JobConfigurationLoad()));
185+
assertThat(JobStatistics.fromPb(job)).isInstanceOf(LoadStatistics.class);
186+
187+
job.setConfiguration(
188+
new com.google.api.services.bigquery.model.JobConfiguration()
189+
.setExtract(new com.google.api.services.bigquery.model.JobConfigurationExtract()));
190+
assertThat(JobStatistics.fromPb(job)).isInstanceOf(ExtractStatistics.class);
191+
192+
job.setConfiguration(
193+
new com.google.api.services.bigquery.model.JobConfiguration()
194+
.setQuery(new com.google.api.services.bigquery.model.JobConfigurationQuery()));
195+
assertThat(JobStatistics.fromPb(job)).isInstanceOf(QueryStatistics.class);
166196
}
167197

168198
private void compareExtractStatistics(ExtractStatistics expected, ExtractStatistics value) {

0 commit comments

Comments
 (0)