Skip to content

Commit 18c5d0d

Browse files
markpet49markpet49
committed
OSM CLI download app
1 parent 2543c23 commit 18c5d0d

6 files changed

Lines changed: 289 additions & 9 deletions

File tree

‎nbproject/configs/BulkDownloadCli.properties‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
main.class=gov.nasa.worldwindx.examples.util.BulkDownloadCli

‎src/config/Earth/OpenStreetMap2.xml‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
~ Copyright 2006-2009, 2017, 2020 United States Government, as represented by the
4+
~ Administrator of the National Aeronautics and Space Administration.
5+
~ All rights reserved.
6+
~
7+
~ The NASA World Wind Java (WWJ) platform is licensed under the Apache License,
8+
~ Version 2.0 (the "License"); you may not use this file except in compliance
9+
~ with the License. You may obtain a copy of the License at
10+
~ http://www.apache.org/licenses/LICENSE-2.0
11+
~
12+
~ Unless required by applicable law or agreed to in writing, software distributed
13+
~ under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
14+
~ CONDITIONS OF ANY KIND, either express or implied. See the License for the
15+
~ specific language governing permissions and limitations under the License.
16+
~
17+
~ NASA World Wind Java (WWJ) also contains the following 3rd party Open Source
18+
~ software:
19+
~
20+
~ Jackson Parser – Licensed under Apache 2.0
21+
~ GDAL – Licensed under MIT
22+
~ JOGL – Licensed under Berkeley Software Distribution (BSD)
23+
~ Gluegen – Licensed under Berkeley Software Distribution (BSD)
24+
~
25+
~ A complete listing of 3rd Party software notices and licenses included in
26+
~ NASA World Wind Java (WWJ) can be found in the WorldWindJava-v2.2 3rd-party
27+
~ notices and licenses PDF found in code directory.
28+
-->
29+
30+
<!-- Open Street Map imagery -->
31+
<Layer version="1" layerType="TiledImageLayer">
32+
<DisplayName>Open Street Map</DisplayName>
33+
<Service serviceName="OGC:WMS" version="1.3">
34+
<GetCapabilitiesURL>https://worldwind47.arc.nasa.gov/mapcache</GetCapabilitiesURL>
35+
<GetMapURL>https://worldwind47.arc.nasa.gov/mapcache</GetMapURL>
36+
<LayerNames>osm</LayerNames>
37+
</Service>
38+
<DataDetailHint>0.40</DataDetailHint>
39+
<RetrievePropertiesFromService>true</RetrievePropertiesFromService>
40+
<!-- day month year hours:minutes:seconds timezone -->
41+
<LastUpdate>01 11 2022 22:52:00 GMT</LastUpdate>
42+
<DataCacheName>Earth/OpenStreetMap2</DataCacheName>
43+
<ImageFormat>image/jpeg</ImageFormat>
44+
<AvailableImageFormats>
45+
<ImageFormat>image/jpeg</ImageFormat>
46+
</AvailableImageFormats>
47+
<FormatSuffix>.jpg</FormatSuffix>
48+
<NumLevels count="16" numEmpty="0"/>
49+
<TileOrigin>
50+
<LatLon units="degrees" latitude="-90" longitude="-180"/>
51+
</TileOrigin>
52+
<LevelZeroTileDelta>
53+
<LatLon units="degrees" latitude="36" longitude="36"/>
54+
</LevelZeroTileDelta>
55+
<TileSize>
56+
<Dimension width="256" height="256"/>
57+
</TileSize>
58+
<Sector>
59+
<SouthWest>
60+
<LatLon units="degrees" latitude="-90" longitude="-180"/>
61+
</SouthWest>
62+
<NorthEast>
63+
<LatLon units="degrees" latitude="90" longitude="180"/>
64+
</NorthEast>
65+
</Sector>
66+
<ForceLevelZeroLoads>false</ForceLevelZeroLoads>
67+
<RetainLevelZeroTiles>false</RetainLevelZeroTiles>
68+
<UseTransparentTextures>true</UseTransparentTextures>
69+
<RetrievalTimeouts>
70+
<ReadTimeout>
71+
<Time units="milliseconds" value="20000"/>
72+
</ReadTimeout>
73+
</RetrievalTimeouts>
74+
</Layer>

‎src/gov/nasa/worldwind/layers/BasicTiledImageLayer.java‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,8 @@
5555
public class BasicTiledImageLayer extends TiledImageLayer implements BulkRetrievable
5656
{
5757
protected final Object fileLock = new Object();
58-
58+
protected long averageFileSize=BasicTiledImageLayerBulkDownloader.DEFAULT_AVERAGE_FILE_SIZE; // The average file size of an individual tile to use for size estimates.
59+
5960
// Layer resource properties.
6061
protected static final int RESOURCE_ID_OGC_CAPABILITIES = 1;
6162

@@ -523,12 +524,18 @@ public long getEstimatedMissingDataSize(Sector sector, double resolution, FileSt
523524

524525
BasicTiledImageLayerBulkDownloader downloader = new BasicTiledImageLayerBulkDownloader(this, sector, resolution,
525526
fileStore != null ? fileStore : this.getDataFileStore(), null);
526-
527+
downloader.setAverageFileSize(this.averageFileSize);
527528
return downloader.getEstimatedMissingDataSize();
528529
}
529530

530-
// *** Tile download ***
531-
// *** Tile download ***
531+
/**
532+
* Set the average size of a tile image to use in estimating download sizes.
533+
* @param value The average size of a tile image.
534+
*/
535+
public void setAverageFileSize(long value) {
536+
this.averageFileSize=value;
537+
}
538+
532539
// *** Tile download ***
533540

534541
protected void retrieveTexture(TextureTile tile, DownloadPostProcessor postProcessor)

‎src/gov/nasa/worldwind/layers/BasicTiledImageLayerBulkDownloader.java‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@ public class BasicTiledImageLayerBulkDownloader extends BulkRetrievalThread
5858
protected final BasicTiledImageLayer layer;
5959
protected final int level;
6060
protected ArrayList<TextureTile> missingTiles;
61+
protected long averageFileSize=DEFAULT_AVERAGE_FILE_SIZE;
6162

6263
/**
6364
* Constructs a downloader to retrieve imagery not currently available in the WorldWind file cache.
@@ -472,7 +473,7 @@ public boolean accept(File file)
472473
}
473474
}
474475

475-
Long averageTileSize = DEFAULT_AVERAGE_FILE_SIZE;
476+
Long averageTileSize = this.averageFileSize;
476477
if (count > 0 && size > 0)
477478
{
478479
averageTileSize = size / count;
@@ -481,6 +482,14 @@ public boolean accept(File file)
481482

482483
return averageTileSize;
483484
}
485+
486+
/**
487+
* Set the average size of a tile image to use in estimating download sizes.
488+
* @param value The average size of a tile image.
489+
*/
490+
public void setAverageFileSize(long value) {
491+
this.averageFileSize=value;
492+
}
484493

485494
protected long computeAverageTileSize(File dir)
486495
{

‎src/gov/nasa/worldwind/wms/WMSTiledImageLayer.java‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -196,18 +196,16 @@ public URLBuilder(AVList params)
196196
String version = params.getStringValue(AVKey.WMS_VERSION);
197197

198198
String coordSystemKey;
199-
String defaultCS;
199+
String defaultCS = "EPSG:4326";
200200
if (version == null || WWUtil.compareVersion(version, "1.3.0") >= 0)
201201
{
202202
this.wmsVersion = MAX_VERSION;
203203
coordSystemKey = "&crs=";
204-
defaultCS = "CRS:84"; // would like to do EPSG:4326 but that's incompatible with our old WMS server, see WWJ-474
205204
}
206205
else
207206
{
208207
this.wmsVersion = version;
209208
coordSystemKey = "&srs=";
210-
defaultCS = "EPSG:4326";
211209
}
212210

213211
String coordinateSystem = params.getStringValue(AVKey.COORDINATE_SYSTEM);
@@ -220,7 +218,6 @@ public URL getURL(Tile tile, String altImageFormat) throws MalformedURLException
220218
if (this.URLTemplate == null)
221219
{
222220
sb = new StringBuffer(WWXML.fixGetMapString(tile.getLevel().getService()));
223-
224221
if (!sb.toString().toLowerCase().contains("service=wms"))
225222
sb.append("service=WMS");
226223
sb.append("&request=GetMap");

‎src/gov/nasa/worldwindx/examples/util/BulkDownloadCli.java‎

Lines changed: 192 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,192 @@
1+
/*
2+
* Copyright 2006-2009, 2017, 2020 United States Government, as represented by the
3+
* Administrator of the National Aeronautics and Space Administration.
4+
* All rights reserved.
5+
*
6+
* The NASA World Wind Java (WWJ) platform is licensed under the Apache License,
7+
* Version 2.0 (the "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software distributed
12+
* under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
13+
* CONDITIONS OF ANY KIND, either express or implied. See the License for the
14+
* specific language governing permissions and limitations under the License.
15+
*
16+
* NASA World Wind Java (WWJ) also contains the following 3rd party Open Source
17+
* software:
18+
*
19+
* Jackson Parser – Licensed under Apache 2.0
20+
* GDAL – Licensed under MIT
21+
* JOGL – Licensed under Berkeley Software Distribution (BSD)
22+
* Gluegen – Licensed under Berkeley Software Distribution (BSD)
23+
*
24+
* A complete listing of 3rd Party software notices and licenses included in
25+
* NASA World Wind Java (WWJ) can be found in the WorldWindJava-v2.2 3rd-party
26+
* notices and licenses PDF found in code directory.
27+
*/
28+
package gov.nasa.worldwindx.examples.util;
29+
30+
import gov.nasa.worldwind.cache.BasicDataFileStore;
31+
import gov.nasa.worldwind.geom.LatLon;
32+
import gov.nasa.worldwind.geom.Sector;
33+
import gov.nasa.worldwind.globes.Earth;
34+
import gov.nasa.worldwind.globes.Globe;
35+
import gov.nasa.worldwind.layers.BasicLayerFactory;
36+
import gov.nasa.worldwind.layers.BasicTiledImageLayer;
37+
import gov.nasa.worldwind.layers.BasicTiledImageLayerBulkDownloader;
38+
import gov.nasa.worldwind.retrieve.Progress;
39+
40+
import java.io.File;
41+
import java.io.IOException;
42+
import java.nio.file.FileVisitResult;
43+
import java.nio.file.Files;
44+
import java.nio.file.Path;
45+
import java.nio.file.SimpleFileVisitor;
46+
import java.nio.file.attribute.BasicFileAttributes;
47+
import java.util.StringTokenizer;
48+
49+
public class BulkDownloadCli {
50+
51+
private static class FileUtils extends SimpleFileVisitor<Path> {
52+
53+
@Override
54+
public FileVisitResult visitFile(Path file, BasicFileAttributes attr) throws IOException {
55+
Files.delete(file);
56+
return FileVisitResult.CONTINUE;
57+
}
58+
59+
@Override
60+
public FileVisitResult postVisitDirectory(Path dir, IOException ex) throws IOException {
61+
Files.delete(dir);
62+
return FileVisitResult.CONTINUE;
63+
}
64+
65+
static void deleteDirectory(File f) throws IOException {
66+
Files.walkFileTree(Path.of(f.getAbsolutePath()), new FileUtils());
67+
}
68+
}
69+
70+
private static final String VERSION = "1.1";
71+
private static final long OSM_AVERAGE_TILE_SIZE = 15000;
72+
73+
public static String makeSizeDescription(long size) {
74+
double sizeInMegaBytes = size / 1024 / 1024;
75+
if (sizeInMegaBytes < 1024) {
76+
return String.format("%,.1f MB", sizeInMegaBytes);
77+
} else if (sizeInMegaBytes < 1024 * 1024) {
78+
return String.format("%,.1f GB", sizeInMegaBytes / 1024);
79+
}
80+
return String.format("%,.1f TB", sizeInMegaBytes / 1024 / 1024);
81+
}
82+
83+
public static void cliDownload(String[] args) {
84+
System.out.println("WorldWind Bulk Download Tool v" + VERSION);
85+
Globe globe = new Earth();
86+
String usage = "Usage: BulkDownload -sector [centerLat,centerLon,radius meters] -path [path for download] -estimate";
87+
String outputPath = null;
88+
Sector sector = null;
89+
boolean estimate = false;
90+
int i = 0;
91+
while (i < args.length) {
92+
switch (args[i]) {
93+
case "-help":
94+
System.out.println(usage);
95+
return;
96+
case "-sector":
97+
i++;
98+
StringTokenizer st = new StringTokenizer(args[i], ",");
99+
if (st.countTokens() != 3) {
100+
System.out.println("Error: Invalid sector specification.");
101+
System.out.println(usage);
102+
return;
103+
}
104+
double lat = Double.parseDouble(st.nextToken());
105+
double lon = Double.parseDouble(st.nextToken());
106+
double radius = Double.parseDouble(st.nextToken());
107+
sector = Sector.boundingSector(globe, LatLon.fromDegrees(lat, lon), radius);
108+
break;
109+
case "-path":
110+
i++;
111+
outputPath = args[i];
112+
break;
113+
case "-estimate":
114+
estimate = true;
115+
break;
116+
default:
117+
System.out.println("Unknown argument: " + args[i]);
118+
System.out.println(usage);
119+
break;
120+
}
121+
i++;
122+
}
123+
124+
if (sector == null) {
125+
System.out.println("Error: Sector not specified.");
126+
System.out.println(usage);
127+
return;
128+
}
129+
130+
System.out.println("Sector: " + sector);
131+
BasicLayerFactory factory = new BasicLayerFactory();
132+
BasicTiledImageLayer layer = (BasicTiledImageLayer) factory.createFromConfigSource("config/Earth/OpenStreetMap2.xml", null);
133+
layer.setAverageFileSize(OSM_AVERAGE_TILE_SIZE);
134+
if (estimate) {
135+
System.out.println(layer.getName() + " estimated download size: " + makeSizeDescription(layer.getEstimatedMissingDataSize(sector, 0)));
136+
return;
137+
}
138+
139+
if (outputPath == null) {
140+
System.out.println("Error: Output path not specified.");
141+
System.out.println(usage);
142+
return;
143+
}
144+
145+
try {
146+
String earthPath = outputPath + "/Earth";
147+
String osmPath = earthPath + "/OpenStreetMap2";
148+
File previousTilePath = new File(earthPath);
149+
File previousOsmPath = new File(osmPath);
150+
if (previousTilePath.exists() && previousTilePath.isDirectory() && previousOsmPath.exists()) {
151+
FileUtils.deleteDirectory(previousTilePath);
152+
}
153+
154+
BasicDataFileStore cache = new BasicDataFileStore(new File(outputPath));
155+
BasicTiledImageLayerBulkDownloader downloadThread = (BasicTiledImageLayerBulkDownloader) layer.makeLocal(sector, 0, cache, null);
156+
downloadThread.setAverageFileSize(OSM_AVERAGE_TILE_SIZE);
157+
Progress progress = downloadThread.getProgress();
158+
System.out.println("Downloading " + layer.getName() + " Tiles");
159+
int lastPercent = -1;
160+
long lastCurrentSize = -1;
161+
while (downloadThread.isAlive()) {
162+
int percent = 0;
163+
if (progress.getTotalCount() > 0) {
164+
percent = (int) ((float) progress.getCurrentCount() / progress.getTotalCount() * 100f);
165+
}
166+
if (lastPercent != percent || lastCurrentSize != progress.getCurrentSize()) {
167+
lastPercent = percent;
168+
lastCurrentSize = progress.getCurrentSize();
169+
String text = percent + "% ";
170+
text += " (" + makeSizeDescription(lastCurrentSize)
171+
+ " / " + makeSizeDescription(progress.getTotalSize())
172+
+ ")";
173+
System.out.println(text);
174+
}
175+
Thread.sleep(1000);
176+
}
177+
int nRemoveLevels = 9;
178+
for (i = 0; i < nRemoveLevels; i++) {
179+
File levelPath = new File(osmPath + "/" + i);
180+
if (levelPath.exists() && levelPath.isDirectory()) {
181+
FileUtils.deleteDirectory(levelPath);
182+
}
183+
}
184+
} catch (Exception ex) {
185+
ex.printStackTrace();
186+
}
187+
}
188+
189+
public static void main(String[] args) {
190+
cliDownload(args);
191+
}
192+
}

0 commit comments

Comments
 (0)