/******************************************************************************

* src/gnuplot.cpp

*

* Process embedded SQL plot instructions in Gnuplot files.

*

******************************************************************************

* Copyright (C) 2013-2016 Timo Bingmann <[email protected]>

*

* This program is free software: you can redistribute it and/or modify it

* under the terms of the GNU General Public License as published by the Free

* Software Foundation, either version 3 of the License, or (at your option)

* any later version.

*

* This program is distributed in the hope that it will be useful, but WITHOUT

* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for

* more details.

*

* You should have received a copy of the GNU General Public License along with

* this program. If not, see <http://www.gnu.org/licenses/>.

*****************************************************************************/

#include <cassert>

#include <cctype>

#include <cerrno>

#include <cstdio>

#include <cstdlib>

#include <cstring>

#include <fstream>

#include <iomanip>

#include <iostream>

#include <map>

#include <sstream>

#include <vector>

#include <boost/regex.hpp>

#include "common.h"

#include "strtools.h"

#include "sql.h"

#include "textlines.h"

#include "importdata.h"

class SpGnuplot

{

public:

//! processed line data

TextLines& m_lines;

//! comment character

static const char comment_char = '#';

// *** current gnuplot datafile ***

std::ostream* m_datafile;

std::string m_datafilename;

unsigned int m_dataindex;

//! scan for next comment line with given prefix

inline ssize_t

scan_lines_for_comment(size_t ln, const std::string& cprefix)

{

return m_lines.scan_for_comment<comment_char>(ln, cprefix);

}

//! Process # SQL commands

void sql(size_t ln, size_t indent, const std::string& cmdline);

//! Process # IMPORT-DATA commands

int importdata(size_t ln, size_t indent, const std::string& cmdline);

//! Process # CONNECT commands

bool connect(size_t ln, size_t indent, const std::string& cmdline);

//! Struct to rewrite Gnuplot "plot" directives with new datafile/index

//! pairs

struct Dataset

{

unsigned int index;

std::string title;

std::string type;

};

//! Helper to rewrite Gnuplot "plot" directives with new datafile/index

//! pairs

void plot_rewrite(size_t ln, size_t indent,

const std::vector<Dataset>& datasets,

const char* plot_type);

//! Process # PLOT commands

void plot(size_t ln, size_t indent, const std::string& cmdline);

//! Process # MULTIPLOT commands

void multiplot(size_t ln, size_t indent, const std::string& cmdline);

//! Process # MACRO commands

void macro(size_t ln, size_t indent, const std::string& cmdline);

//! Process TextLines

int process();

//! Process Textlines

SpGnuplot(const std::string& filename, TextLines& lines);

};

//! Process # SQL commands

void SpGnuplot::sql(size_t /* ln */, size_t /* indent */, const std::string& cmdline)

{

SqlQuery sql = g_db->query(cmdline);

OUT("SQL command successful.");

}

//! Process # IMPORT-DATA commands

int SpGnuplot::importdata(size_t /* ln */, size_t /* indent */, const std::string& cmdline)

{

// split argument at whitespaces

std::vector<std::string> args = split_ws(cmdline);

// convert std::strings to char*

char* argv[args.size() + 1];

for (size_t i = 0; i < args.size(); ++i)

argv[i] = (char*)args[i].c_str();

argv[args.size()] = NULL;

return ImportData(true).main(args.size(), argv);

}

//! Process # CONNECT commands

bool SpGnuplot::connect(size_t /* ln */, size_t /* indent */, const std::string& cmdline)

{

return g_db_connect(cmdline);

}

//! Helper to rewrite Gnuplot "plot" directives with new datafile/index pairs

void SpGnuplot::plot_rewrite(size_t ln, size_t indent,

const std::vector<Dataset>& datasets,

const char* plot_type)

{

std::ostringstream oss;

// check whether line contains an "plot" command

static const boost::regex re_plot("[[:blank:]]*plot.*\\\\[[:blank:]]*");

if (ln >= m_lines.size() ||

!boost::regex_match(m_lines[ln], re_plot))

{

// no "plot" command: construct default version from scratch

if (datasets.size())

oss << "plot";

for (size_t i = 0; i < datasets.size(); ++i)

{

if (i != 0) oss << ',';

oss << " \\" << std::endl

<< " '" << m_datafilename << "' index " << datasets[i].index;

if (datasets[i].title.size())

oss << " title \"" << datasets[i].title << '"';

oss << " with " << datasets[i].type;

}

if (datasets.size())

oss << std::endl;

m_lines.replace(ln, ln, indent, oss.str(), plot_type);

return;

}

// scan following lines for plot descriptions

static const boost::regex re_line("[[:blank:]]*'[^']+' index [0-9]+( title \"[^\"]*\")?( .*?)(, \\\\)?[[:blank:]]*");

boost::smatch rm;

if (datasets.size())

oss << "plot";

size_t eln = ln+1;

size_t entry = 0; // dataset entry

while (eln < m_lines.size() &&

boost::regex_match(m_lines[eln], rm, re_line))

{

++eln;

// copy properties to new plot line

if (entry < datasets.size())

{

if (entry != 0) oss << ',';

oss << " \\" << std::endl

<< " '" << m_datafilename << "' index " << datasets[entry].index;

// if dataset contains a title, add it

if (datasets[entry].title.size())

oss << " title \"" << datasets[entry].title << '"';

// output extended properties

oss << rm[2];

++entry;

// break if no \ was found at the end

if (rm[3].length() == 0) break;

}

else

{

// gobble additional plot line

}

}

// append missing plot descriptions

while (entry < datasets.size())

{

if (entry != 0) oss << ',';

oss << " \\" << std::endl

<< " '" << m_datafilename << "' index " << datasets[entry].index;

if (datasets[entry].title.size())

oss << " title \"" << datasets[entry].title << '"';

oss << " with " << datasets[entry].type;

++entry;

}

if (datasets.size())

oss << std::endl;

m_lines.replace(ln, eln, indent, oss.str(), plot_type);

}

//! Process # PLOT commands

void SpGnuplot::plot(size_t ln, size_t indent, const std::string& cmdline)

{

SqlQuery sql = g_db->query(cmdline);

// write a header to the datafile containing the query

std::ostream& df = *m_datafile;

df << std::string(80, '#') << std::endl

<< "# PLOT " << cmdline << std::endl

<< '#' << std::endl;

// write result data rows

while (sql->step())

{

for (unsigned int col = 0; col < sql->num_cols(); ++col)

{

if (col != 0) df << '\t';

df << sql->text(col);

}

df << std::endl;

}

// append plot line to gnuplot

std::vector<Dataset> datasets(1);

datasets[0].index = m_dataindex;

datasets[0].type = "linespoints";

// finish index in datafile

df << std::endl << std::endl;

++m_dataindex;

plot_rewrite(ln, indent, datasets, "PLOT");

}

//! Process # MULTIPLOT commands

void SpGnuplot::multiplot(size_t ln, size_t indent, const std::string& cmdline)

{

// extract MULTIPLOT columns

static const boost::regex

re_multiplot("MULTIPLOT\\(([^)]+)\\) (.+)");

boost::smatch rm_multiplot;

if (!boost::regex_match(cmdline, rm_multiplot, re_multiplot))

OUT_THROW("MULTIPLOT() requires group column list.");

std::string multiplot = rm_multiplot[1].str();

std::string query = rm_multiplot[2].str();

query = replace_all(query, "MULTIPLOT", multiplot);

std::vector<std::string> groupfields = split(multiplot, ',');

std::for_each(groupfields.begin(), groupfields.end(), trim_inplace_ws);

// execute query

SqlQuery sql = g_db->query(query);

std::vector<Dataset> datasets;

// read column names

sql->read_colmap();

// check for existing x and y columns.

if (!sql->exist_col("x"))

OUT_THROW("MULTIPLOT failed: result contains no 'x' column.");

if (!sql->exist_col("y"))

OUT_THROW("MULTIPLOT failed: result contains no 'y' column.");

unsigned int col_x = sql->find_col("x"), col_y = sql->find_col("y");

int col_xmin = !sql->exist_col("xmin") ? -1 : sql->find_col("xmin");

int col_xmax = !sql->exist_col("xmax") ? -1 : sql->find_col("xmax");

int col_ymin = !sql->exist_col("ymin") ? -1 : sql->find_col("ymin");

int col_ymax = !sql->exist_col("ymax") ? -1 : sql->find_col("ymax");

bool have_xerrorbars = col_xmin >= 0 && col_xmax >= 0;

bool have_yerrorbars = col_ymin >= 0 && col_ymax >= 0;

// check existance of group fields and save ids

std::vector<int> groupcols;

for (std::vector<std::string>::const_iterator gi = groupfields.begin();

gi != groupfields.end(); ++gi)

{

if (!sql->exist_col(*gi))

{

OUT_THROW("MULTIPLOT failed: result contains no '" << *gi <<

"' column, which is a MULTIPLOT group field.");

}

groupcols.push_back(sql->find_col(*gi));

}

// write a header to the datafile containing the query

std::ostream& df = *m_datafile;

df << std::string(80, '#') << std::endl

<< "# " << cmdline << std::endl

<< '#' << std::endl;

// collect coordinates groups

{

std::vector<std::string> lastgroup;

size_t rows = 0;

while (sql->step())

{

// collect groupfields for this row

std::vector<std::string> rowgroup (groupcols.size());

for (size_t i = 0; i < groupcols.size(); ++i)

rowgroup[i] = sql->text(groupcols[i]);

if (sql->current_row() == 0 || lastgroup != rowgroup)

{

// group fields mismatch (or first row) -> start new group

if (sql->current_row() != 0) {

df << std::endl << std::endl;

++m_dataindex;

}

lastgroup = rowgroup;

// store group's legend string

std::ostringstream os;

for (size_t i = 0; i < groupcols.size(); ++i) {

if (i != 0) os << ',';

os << groupfields[i] << '=' << rowgroup[i];

}

datasets.push_back(Dataset());

datasets.back().index = m_dataindex;

datasets.back().title = os.str();

datasets.back().type = "linespoints";

if (have_xerrorbars && have_yerrorbars)

datasets.back().type = "xyerrorbars";

else if (have_xerrorbars)

datasets.back().type = "xerrorbars";

else if (have_yerrorbars)

datasets.back().type = "yerrorbars";

df << "# index " << m_dataindex << ' ' << os.str() << std::endl;

}

// group fields match with last row -> append coordinates.

df << sql->text(col_x)

<< '\t' << sql->text(col_y);

if (have_xerrorbars) {

df << '\t' << sql->text(col_xmin)

<< '\t' << sql->text(col_xmax);

}

if (have_yerrorbars) {

df << '\t' << sql->text(col_ymin)

<< '\t' << sql->text(col_ymax);

}

df << std::endl;

++rows;

}

if (rows == 0)

df << "- # (no data rows)" << std::endl;

// finish last plot

df << std::endl << std::endl;

++m_dataindex;

}

plot_rewrite(ln, indent, datasets, "MULTIPLOT");

}

static inline

std::string maybe_quote(const std::string& str)

{

if (str_is_double(str)) // fully parsed

return str;

else // needs quoting

return '\'' + str + '\'';

}

//! Process # MACRO commands

void SpGnuplot::macro(size_t ln, size_t indent, const std::string& cmdline)

{

SqlQuery sql = g_db->query(cmdline);

sql->step();

// write each column as macro value

std::ostringstream oss;

for (unsigned int col = 0; col < sql->num_cols(); ++col)

{

oss << sql->col_name(col) << " = "

<< maybe_quote( sql->text(col) ) << std::endl;

}

// scan following lines for macro defintions

static const boost::regex re_macro("[^=]+ = .*");

size_t eln = ln;

while (eln < m_lines.size() &&

boost::regex_match(m_lines[eln], re_macro))

{

++eln;

}

m_lines.replace(ln, eln, indent, oss.str(), "MACRO");

}

//! process line-based file in place

int SpGnuplot::process()

{

bool active_range = gopt_ranges.size() ? false : true;

// iterate over all lines

for (size_t ln = 0; ln < m_lines.size();)

{

// collect command from comment lines

std::string cmd;

size_t indent;

if (!m_lines.collect_comment<comment_char>(ln, cmd, indent))

continue;

// extract first word

std::string::size_type space_pos =

cmd.find_first_not_of("ABCDEFGHIJKLMNOPQRSTUVWXYZ-_");

std::string first_word = cmd.substr(0, space_pos);

if (first_word == "RANGE")

{

// extract second word

std::string::size_type non_space_pos =

cmd.find_first_of("ABCDEFGHIJKLMNOPQRSTUVWXYZ-_", space_pos);

std::string::size_type space2_pos =

cmd.find_first_not_of("ABCDEFGHIJKLMNOPQRSTUVWXYZ-_", non_space_pos);

std::string::size_type non_space2_pos =

cmd.find_first_not_of(" ", space2_pos);

std::string second_word = cmd.substr(

non_space_pos, space2_pos - non_space_pos);

std::string rest_word = cmd.substr(non_space2_pos);

if (second_word == "BEGIN")

{

if (std::find(gopt_ranges.begin(), gopt_ranges.end(),

rest_word) != gopt_ranges.end())

{

OUT(ln << "# " << cmd);

active_range = true;

}

}

else if (second_word == "END")

{

if (std::find(gopt_ranges.begin(), gopt_ranges.end(),

rest_word) != gopt_ranges.end())

{

OUT(ln << "# " << cmd);

active_range = false;

}

}

else

{

OUT("? maybe unknown keywords " << first_word << " " << second_word);

}

}

else if (!active_range)

{

// skip keywords in non-active ranges

}

else if (first_word == "SQL")

{

OUT(ln << "# " << cmd);

sql(ln, indent, cmd.substr(space_pos+1));

}

else if (first_word == "IMPORT-DATA")

{

OUT(ln << "# " << cmd);

if (importdata(ln, indent, cmd) != EXIT_SUCCESS)

return EXIT_FAILURE;

}

else if (first_word == "CONNECT")

{

OUT(ln << "# " << cmd);

if (!connect(ln, indent, cmd.substr(space_pos+1)))

return EXIT_FAILURE;

}

else if (first_word == "PLOT")

{

OUT(ln << "# " << cmd);

plot(ln, indent, cmd.substr(space_pos+1));

}

else if (first_word == "MULTIPLOT")

{

OUT(ln << "# " << cmd);

multiplot(ln, indent, cmd);

}

else if (first_word == "MACRO")

{

OUT(ln << "# " << cmd);

macro(ln, indent, cmd.substr(space_pos+1));

}

else

{

if (first_word.size() >= 4 && first_word[0] != '-')

OUT("? maybe unknown keyword " << first_word);

}

}

return EXIT_SUCCESS;

}

//! process a stream

SpGnuplot::SpGnuplot(const std::string& filename, TextLines& lines)

: m_lines(lines)

{

// construct output data file

m_datafilename = filename;

std::string::size_type dotpos = m_datafilename.rfind('.');

if (dotpos != std::string::npos)

m_datafilename = m_datafilename.substr(0, dotpos);

m_datafilename += "-data.txt";

// open output data file

if (!gopt_check_output)

{

m_datafile = new std::ofstream(m_datafilename.c_str());

if (!m_datafile->good()) {

OUT("Fatal error opening datafile " << m_datafilename << ": " << strerror(errno));

delete m_datafile;

return;

}

}

else

{

// open temporary data file

m_datafile = new std::ostringstream();

}

m_dataindex = 0;

// write data file preamble

{

std::ostream& df = *m_datafile;

df << std::string(80, '#') << std::endl

<< '#' << std::endl

<< "# DO NOT EDIT THIS FILE MANUALLY!" << std::endl

<< "# ALL CHANGES WILL BE LOST WHEN RECREATED!" << std::endl

<< '#' << std::endl

<< "# The data in this file was generated by sqlplot-tools" << std::endl

<< "# by processing \"" << filename << "\"." << std::endl

<< '#' << std::endl

<< std::string(80, '#') << std::endl << std::endl;

}

// process lines in place

if (process() != EXIT_SUCCESS)

exit(EXIT_FAILURE);

// verify processed output against file

if (gopt_check_output)

{

std::ifstream in(m_datafilename.c_str());

if (!in.good()) {

OUT("Error reading " << m_datafilename << ": " << strerror(errno));

exit(EXIT_FAILURE);

}

std::string checkdata = read_stream(in);

std::ostringstream* oss = (std::ostringstream*)m_datafile;

if (checkdata != oss->str())

{

OUT("Mismatch to expected output data file:");

simple_diff(oss->str(), checkdata);

OUT_THROW("Mismatch to expected output data file " << m_datafilename);

}

else

{

OUT("Good match to expected output data file " << m_datafilename);

}

}

delete m_datafile;

}

//! Process Gnuplot file

void sp_gnuplot(const std::string& filename, TextLines& lines)

{

SpGnuplot sp(filename, lines);

}