Skip to content

Pick up tilestats fixes and other changes from Tippecanoe #43

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
e-n-f merged 1 commit into from
Aug 23, 2017
Merged

Pick up tilestats fixes and other changes from Tippecanoe #43

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion tile.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1531,7 +1531,7 @@ int main(int argc, char **argv) {

std::map<std::string, layermap_entry> lm = merge_layermaps(layermaps);

mbtiles_write_metadata(outdb, NULL, outfile, 0, zooms - 1, minlat, minlon, maxlat, maxlon, midlat, midlon, false, "", lm, !bitmap, outfile);
mbtiles_write_metadata(outdb, NULL, outfile, 0, zooms - 1, minlat, minlon, maxlat, maxlon, midlat, midlon, false, "", lm, !bitmap, outfile, true);

write_meta(zoom_max, outdb);

Expand Down
148 changes: 81 additions & 67 deletions tippecanoe/mbtiles.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,26 +85,21 @@ void mbtiles_write_tile(sqlite3 *outdb, int z, int tx, int ty, const char *data,
}
}

static void quote(std::string *buf, const char *s) {
char tmp[strlen(s) * 8 + 1];
char *out = tmp;

for (; *s != '\0'; s++) {
unsigned char ch = (unsigned char) *s;
static void quote(std::string &buf, std::string const &s) {
for (size_t i = 0; i < s.size(); i++) {
unsigned char ch = s[i];

if (ch == '\\' || ch == '\"') {
*out++ = '\\';
*out++ = ch;
buf.push_back('\\');
buf.push_back(ch);
} else if (ch < ' ') {
sprintf(out, "\\u%04x", ch);
out = out + strlen(out);
char tmp[7];
sprintf(tmp, "\\u%04x", ch);
buf.append(std::string(tmp));
} else {
*out++ = ch;
buf.push_back(ch);
}
}

*out = '\0';
buf->append(tmp, strlen(tmp));
}

void aprintf(std::string *buf, const char *format, ...) {
Expand Down Expand Up @@ -142,7 +137,7 @@ bool type_and_string::operator!=(const type_and_string &o) const {
return false;
}

std::string tilestats(std::map<std::string, layermap_entry> const &layermap1) {
std::string tilestats(std::map<std::string, layermap_entry> const &layermap1, size_t elements) {
// Consolidate layers/attributes whose names are truncated
std::vector<std::map<std::string, layermap_entry>> lmv;
lmv.push_back(layermap1);
Expand All @@ -166,7 +161,7 @@ std::string tilestats(std::map<std::string, layermap_entry> const &layermap1) {
out.append("\t\t{\n");

out.append("\t\t\t\"layer\": \"");
quote(&out, layer.first.c_str());
quote(out, layer.first.c_str());
out.append("\",\n");

out.append("\t\t\t\"count\": ");
Expand All @@ -181,7 +176,7 @@ std::string tilestats(std::map<std::string, layermap_entry> const &layermap1) {
}

out.append("\t\t\t\"geometry\": \"");
quote(&out, geomtype.c_str());
quote(out, geomtype.c_str());
out.append("\",\n");

size_t attrib_count = layer.second.file_keys.size();
Expand All @@ -197,7 +192,7 @@ std::string tilestats(std::map<std::string, layermap_entry> const &layermap1) {

size_t attrs = 0;
for (auto attribute : layer.second.file_keys) {
if (attrs == 100) {
if (attrs == elements) {
break;
}
if (attrs != 0) {
Expand All @@ -208,7 +203,7 @@ std::string tilestats(std::map<std::string, layermap_entry> const &layermap1) {
out.append("\t\t\t\t{\n");

out.append("\t\t\t\t\t\"attribute\": \"");
quote(&out, attribute.first.c_str());
quote(out, attribute.first.c_str());
out.append("\",\n");

size_t val_count = attribute.second.sample_values.size();
Expand Down Expand Up @@ -238,30 +233,36 @@ std::string tilestats(std::map<std::string, layermap_entry> const &layermap1) {
}

out.append("\t\t\t\t\t\"type\": \"");
quote(&out, type_str.c_str());
quote(out, type_str.c_str());
out.append("\",\n");

out.append("\t\t\t\t\t\"values\": [\n");

size_t vals = 0;
for (auto value : attribute.second.sample_values) {
if (vals == 100) {
if (vals == elements) {
break;
}
if (vals != 0) {
out.append(",\n");
}
vals++;

if (value.type == mvt_double || value.type == mvt_bool) {
if (vals != 0) {
out.append(",\n");
}
vals++;

out.append("\t\t\t\t\t\t");
out.append(value.string);
} else {
std::string trunc = truncate16(value.string, 256);

if (trunc.size() == value.string.size()) {
if (vals != 0) {
out.append(",\n");
}
vals++;

out.append("\t\t\t\t\t\t\"");
quote(&out, value.string.c_str());
quote(out, value.string.c_str());
out.append("\"");
}
}
Expand Down Expand Up @@ -304,7 +305,7 @@ std::string tilestats(std::map<std::string, layermap_entry> const &layermap1) {
return out2;
}

void mbtiles_write_metadata(sqlite3 *outdb, const char *outdir, const char *fname, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double midlat, double midlon, int forcetable, const char *attribution, std::map<std::string, layermap_entry> const &layermap, bool vector, const char *description) {
void mbtiles_write_metadata(sqlite3 *outdb, const char *outdir, const char *fname, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double midlat, double midlon, int forcetable, const char *attribution, std::map<std::string, layermap_entry> const &layermap, bool vector, const char *description, bool do_tilestats) {
char *sql, *err;

sqlite3 *db = outdb;
Expand Down Expand Up @@ -412,57 +413,66 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *outdir, const char *fnam
sqlite3_free(sql);

if (vector) {
std::string buf("{");
aprintf(&buf, "\"vector_layers\": [ ");
size_t elements = 100;
std::string buf;

std::vector<std::string> lnames;
for (auto ai = layermap.begin(); ai != layermap.end(); ++ai) {
lnames.push_back(ai->first);
}
{
buf = "{";
aprintf(&buf, "\"vector_layers\": [ ");

for (size_t i = 0; i < lnames.size(); i++) {
if (i != 0) {
aprintf(&buf, ", ");
std::vector<std::string> lnames;
for (auto ai = layermap.begin(); ai != layermap.end(); ++ai) {
lnames.push_back(ai->first);
}

auto fk = layermap.find(lnames[i]);
aprintf(&buf, "{ \"id\": \"");
quote(&buf, lnames[i].c_str());
aprintf(&buf, "\", \"description\": \"\", \"minzoom\": %d, \"maxzoom\": %d, \"fields\": {", fk->second.minzoom, fk->second.maxzoom);

bool first = true;
for (auto j = fk->second.file_keys.begin(); j != fk->second.file_keys.end(); ++j) {
if (first) {
first = false;
} else {
for (size_t i = 0; i < lnames.size(); i++) {
if (i != 0) {
aprintf(&buf, ", ");
}

aprintf(&buf, "\"");
quote(&buf, j->first.c_str());
auto fk = layermap.find(lnames[i]);
aprintf(&buf, "{ \"id\": \"");
quote(buf, lnames[i]);
aprintf(&buf, "\", \"description\": \"\", \"minzoom\": %d, \"maxzoom\": %d, \"fields\": {", fk->second.minzoom, fk->second.maxzoom);

bool first = true;
for (auto j = fk->second.file_keys.begin(); j != fk->second.file_keys.end(); ++j) {
if (first) {
first = false;
} else {
aprintf(&buf, ", ");
}

aprintf(&buf, "\"");
quote(buf, j->first.c_str());

int type = 0;
for (auto s : j->second.sample_values) {
type |= (1 << s.type);
}
int type = 0;
for (auto s : j->second.sample_values) {
type |= (1 << s.type);
}

if (type == (1 << mvt_double)) {
aprintf(&buf, "\": \"Number\"");
} else if (type == (1 << mvt_bool)) {
aprintf(&buf, "\": \"Boolean\"");
} else if (type == (1 << mvt_string)) {
aprintf(&buf, "\": \"String\"");
} else {
aprintf(&buf, "\": \"Mixed\"");
if (type == (1 << mvt_double)) {
aprintf(&buf, "\": \"Number\"");
} else if (type == (1 << mvt_bool)) {
aprintf(&buf, "\": \"Boolean\"");
} else if (type == (1 << mvt_string)) {
aprintf(&buf, "\": \"String\"");
} else {
aprintf(&buf, "\": \"Mixed\"");
}
}

aprintf(&buf, "} }");
}

aprintf(&buf, "} }");
}
aprintf(&buf, " ]");

if (do_tilestats && elements > 0) {
aprintf(&buf, ",\"tilestats\": %s", tilestats(layermap, elements).c_str());
}

aprintf(&buf, " ],");
aprintf(&buf, "\"tilestats\": %s", tilestats(layermap).c_str());
aprintf(&buf, "}");
aprintf(&buf, "}");
}

sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('json', %Q);", buf.c_str());
if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) {
Expand Down Expand Up @@ -490,8 +500,8 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *outdir, const char *fnam
while (sqlite3_step(stmt) == SQLITE_ROW) {
std::string key, value;

quote(&key, (const char *) sqlite3_column_text(stmt, 0));
quote(&value, (const char *) sqlite3_column_text(stmt, 1));
quote(key, (const char *) sqlite3_column_text(stmt, 0));
quote(value, (const char *) sqlite3_column_text(stmt, 1));

if (!first) {
fprintf(fp, ",\n");
Expand Down Expand Up @@ -536,6 +546,10 @@ std::map<std::string, layermap_entry> merge_layermaps(std::vector<std::map<std::

for (size_t i = 0; i < maps.size(); i++) {
for (auto map = maps[i].begin(); map != maps[i].end(); ++map) {
if (map->second.points + map->second.lines + map->second.polygons == 0) {
continue;
}

std::string layername = map->first;
if (trunc) {
layername = truncate16(layername, 256);
Expand Down
2 changes: 1 addition & 1 deletion tippecanoe/mbtiles.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ sqlite3 *mbtiles_open(char *dbname, char **argv, int forcetable);

void mbtiles_write_tile(sqlite3 *outdb, int z, int tx, int ty, const char *data, int size);

void mbtiles_write_metadata(sqlite3 *outdb, const char *outdir, const char *fname, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double midlat, double midlon, int forcetable, const char *attribution, std::map<std::string, layermap_entry> const &layermap, bool vector, const char *description);
void mbtiles_write_metadata(sqlite3 *outdb, const char *outdir, const char *fname, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double midlat, double midlon, int forcetable, const char *attribution, std::map<std::string, layermap_entry> const &layermap, bool vector, const char *description, bool do_tilestats);

void mbtiles_close(sqlite3 *outdb, const char *pgm);

Expand Down
68 changes: 68 additions & 0 deletions tippecanoe/mvt.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@
#include <vector>
#include <map>
#include <zlib.h>
#include <errno.h>
#include <limits.h>
#include "mvt.hpp"
#include "protozero/varint.hpp"
#include "protozero/pbf_reader.hpp"
Expand Down Expand Up @@ -416,3 +418,69 @@ void mvt_layer::tag(mvt_feature &feature, std::string key, mvt_value value) {
feature.tags.push_back(ko);
feature.tags.push_back(vo);
}

static int is_integer(const char *s, long long *v) {
errno = 0;
char *endptr;

*v = strtoll(s, &endptr, 0);
if (*v == 0 && errno != 0) {
return 0;
}
if ((*v == LLONG_MIN || *v == LLONG_MAX) && (errno == ERANGE)) {
return 0;
}
if (*endptr != '\0') {
// Special case: If it is an integer followed by .0000 or similar,
// it is still an integer

if (*endptr != '.') {
return 0;
}
endptr++;
for (; *endptr != '\0'; endptr++) {
if (*endptr != '0') {
return 0;
}
}

return 1;
}

return 1;
}

mvt_value stringified_to_mvt_value(int type, const char *s) {
mvt_value tv;

if (type == mvt_double) {
long long v;
if (is_integer(s, &v)) {
if (v >= 0) {
tv.type = mvt_int;
tv.numeric_value.int_value = v;
} else {
tv.type = mvt_sint;
tv.numeric_value.sint_value = v;
}
} else {
double d = atof(s);

if (d == (float) d) {
tv.type = mvt_float;
tv.numeric_value.float_value = d;
} else {
tv.type = mvt_double;
tv.numeric_value.double_value = d;
}
}
} else if (type == mvt_bool) {
tv.type = mvt_bool;
tv.numeric_value.bool_value = (s[0] == 't');
} else {
tv.type = mvt_string;
tv.string_value = s;
}

return tv;
}
7 changes: 4 additions & 3 deletions tippecanoe/mvt.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,8 @@ enum mvt_operation {
};

struct mvt_geometry {
int x;
int y;
long long x;
long long y;
int /* mvt_operation */ op;

mvt_geometry(int op, long long x, long long y);
Expand Down Expand Up @@ -86,7 +86,7 @@ struct mvt_layer {
std::vector<mvt_feature> features;
std::vector<std::string> keys;
std::vector<mvt_value> values;
int extent;
long long extent;

// Add a key-value pair to a feature, using this layer's constant pool
void tag(mvt_feature &feature, std::string key, mvt_value value);
Expand All @@ -108,4 +108,5 @@ int decompress(std::string const &input, std::string &output);
int compress(std::string const &input, std::string &output);
int dezig(unsigned n);

mvt_value stringified_to_mvt_value(int type, const char *s);
#endif