Skip to content

Use stdio instead of mapped memory to read the count file when tiling #31

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
e-n-f merged 2 commits into from
Apr 26, 2017
Merged

Use stdio instead of mapped memory to read the count file when tiling #31

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
012fad2
Use stdio instead of mapped memory to read the count file when tiling
e-n-f Apr 25, 2017
238f004
Fix memory leak when densities in a PNG tile are all 0
e-n-f Apr 26, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
88 changes: 64 additions & 24 deletions tile.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ struct tiler {
long long midx, midy;
long long atmid;

unsigned char *map;
FILE *fp;
size_t minzoom;
size_t zooms;
size_t detail;
Expand Down Expand Up @@ -202,28 +202,32 @@ void make_tile(sqlite3 *outdb, tile &tile, int z, int detail, long long zoom_max
}

if (bitmap) {
png_structp png_ptr;
png_infop info_ptr;
bool anything = false;

unsigned char *rows[1U << detail];
for (size_t y = 0; y < 1U << detail; y++) {
rows[y] = new unsigned char[1U << detail];

for (size_t x = 0; x < (1U << detail); x++) {
long long density = normalized[y * (1 << detail) + x];
rows[y][x] = density;

if (density > 0) {
anything = true;
}
}
}

if (!anything) {
return;
}

unsigned char *rows[1U << detail];
for (size_t y = 0; y < 1U << detail; y++) {
rows[y] = new unsigned char[1U << detail];

for (size_t x = 0; x < (1U << detail); x++) {
long long density = normalized[y * (1 << detail) + x];
rows[y][x] = density;
}
}

png_structp png_ptr;
png_infop info_ptr;

png_ptr = png_create_write_struct(PNG_LIBPNG_VER_STRING, NULL, fail, fail);
if (png_ptr == NULL) {
fprintf(stderr, "PNG failure (write struct)\n");
Expand Down Expand Up @@ -260,6 +264,8 @@ void make_tile(sqlite3 *outdb, tile &tile, int z, int detail, long long zoom_max
png_set_rows(png_ptr, info_ptr, rows);
png_set_write_fn(png_ptr, &compressed, string_append, NULL);
png_write_png(png_ptr, info_ptr, 0, NULL);
png_write_end(png_ptr, info_ptr);
png_destroy_info_struct(png_ptr, &info_ptr);
png_destroy_write_struct(&png_ptr, &info_ptr);
}

Expand Down Expand Up @@ -399,17 +405,48 @@ void *run_tile(void *p) {
return NULL;
}

unsigned long long first = read64(t->map + HEADER_LEN + t->start * RECORD_BYTES);
unsigned long long last = read64(t->map + HEADER_LEN + (t->end - 1) * RECORD_BYTES);
unsigned char firstbuf[RECORD_BYTES];
unsigned char lastbuf[RECORD_BYTES];

if (fseeko(t->fp, t->start * RECORD_BYTES + HEADER_LEN, SEEK_SET) != 0) {
perror("fseeko");
exit(EXIT_FAILURE);
}
if (fread(firstbuf, RECORD_BYTES, 1, t->fp) != 1) {
perror("fread");
exit(EXIT_FAILURE);
}

if (fseeko(t->fp, (t->end - 1) * RECORD_BYTES + HEADER_LEN, SEEK_SET) != 0) {
perror("fseeko");
exit(EXIT_FAILURE);
}
if (fread(lastbuf, RECORD_BYTES, 1, t->fp) != 1) {
perror("fread");
exit(EXIT_FAILURE);
}

unsigned long long first = read64(firstbuf);
unsigned long long last = read64(lastbuf);

long long seq = 0;
long long percent = -1;
long long max = 0;

if (fseeko(t->fp, t->start * RECORD_BYTES + HEADER_LEN, SEEK_SET) != 0) {
perror("fseeko");
exit(EXIT_FAILURE);
}

unsigned long long oindex = 0;
for (size_t i = t->start; i < t->end; i++) {
unsigned long long index = read64(t->map + HEADER_LEN + i * RECORD_BYTES);
unsigned long long count = read32(t->map + HEADER_LEN + i * RECORD_BYTES + INDEX_BYTES);
unsigned char buf[RECORD_BYTES];
if (fread(buf, RECORD_BYTES, 1, t->fp) != 1) {
perror("fread");
exit(EXIT_FAILURE);
}
unsigned long long index = read64(buf);
unsigned long long count = read32(buf + INDEX_BYTES);
seq++;

if (oindex > index) {
Expand Down Expand Up @@ -1290,18 +1327,21 @@ int main(int argc, char **argv) {
exit(EXIT_FAILURE);
}

int fd = open(argv[optind], O_RDONLY);
if (fd < 0) {
perror(argv[optind]);
exit(EXIT_FAILURE);
FILE *fps[cpus];
for (size_t j = 0; j < cpus; j++) {
fps[j] = fopen(argv[optind], "rb");
if (fps[j] == NULL) {
perror(argv[optind]);
exit(EXIT_FAILURE);
}
}
unsigned char *map = (unsigned char *) mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
if (map == MAP_FAILED) {
perror("mmap");

char buf[HEADER_LEN];
if (fread(buf, HEADER_LEN, 1, fps[0]) != 1) {
perror("fread file header");
exit(EXIT_FAILURE);
}

if (memcmp(map, header_text, HEADER_LEN) != 0) {
if (memcmp(buf, header_text, HEADER_LEN) != 0) {
fprintf(stderr, "%s: not a tile-count file\n", argv[optind]);
exit(EXIT_FAILURE);
}
Expand Down Expand Up @@ -1335,7 +1375,7 @@ int main(int argc, char **argv) {

size_t records = (st.st_size - HEADER_LEN) / RECORD_BYTES;
for (size_t j = 0; j < cpus; j++) {
tilers[j].map = map;
tilers[j].fp = fps[j];
tilers[j].start = j * records / cpus;
if (j > 0) {
tilers[j - 1].end = tilers[j].start;
Expand Down