Skip to content

Tokenizer rewrite #175

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
TRPB merged 16 commits into from
Dec 9, 2017
Merged

Tokenizer rewrite #175

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
ec46d2d
Tokens are now always an object, never a plain array
TRPB Dec 3, 2017
0a6e068
pass by reference no longer needed as is object
TRPB Dec 3, 2017
e0f2304
Added comments in new tokenizer format, WIP
TRPB Dec 4, 2017
156d2fc
Tidying up interface
TRPB Dec 4, 2017
22f45f0
added literals to new tokenizer
TRPB Dec 8, 2017
a54581a
added strings to new tokenizer
TRPB Dec 8, 2017
2934fbe
Finished tokenizer rewrite
TRPB Dec 8, 2017
c75a573
removed unused functions from TokenizedString
TRPB Dec 8, 2017
3b052cf
Removed unused variables from Tokenizer class
TRPB Dec 8, 2017
80030e5
Added tokenizeRules class variable to Tokenizer class
TRPB Dec 8, 2017
c1a6e22
manual merge of #174 for new tokenizer class
TRPB Dec 8, 2017
b9154bd
Merge branch 'master' into Tokenizer-Rewrite
TRPB Dec 8, 2017
fe8abaa
update year
TRPB Dec 8, 2017
45e4c47
copyright year/version
TRPB Dec 8, 2017
a17dc5a
added missing copyright notices
TRPB Dec 8, 2017
a1074b4
removed superfluous check
TRPB Dec 8, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions src/Builder.php
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
<?php
/* @description Transformation Style Sheets - Revolutionising PHP templating *
* @author Tom Butler [email protected] *
* @copyright 2015 Tom Butler <[email protected]> | https://r.je/ *
* @copyright 2017 Tom Butler <[email protected]> | https://r.je/ *
* @license http://www.opensource.org/licenses/bsd-license.php BSD License *
* @version 1.0 */
* @version 1.2 */
namespace Transphporm;
/** Builds a Transphorm instance from the 3 constituent parts. XML template string, TSS string and data */
class Builder {
Expand Down
4 changes: 2 additions & 2 deletions src/Cache.php
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
<?php
/* @description Transformation Style Sheets - Revolutionising PHP templating *
* @author Tom Butler [email protected] *
* @copyright 2015 Tom Butler <[email protected]> | https://r.je/ *
* @copyright 2017 Tom Butler <[email protected]> | https://r.je/ *
* @license http://www.opensource.org/licenses/bsd-license.php BSD License *
* @version 1.0 */
* @version 1.2 */
namespace Transphporm;
class Cache {
private $cache;
Expand Down
4 changes: 2 additions & 2 deletions src/Config.php
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
<?php
/* @description Transformation Style Sheets - Revolutionising PHP templating *
* @author Tom Butler [email protected] *
* @copyright 2015 Tom Butler <[email protected]> | https://r.je/ *
* @copyright 2017 Tom Butler <[email protected]> | https://r.je/ *
* @license http://www.opensource.org/licenses/bsd-license.php BSD License *
* @version 1.0 */
* @version 1.2 */
namespace Transphporm;
class Config {
private $properties = [];
Expand Down
7 changes: 6 additions & 1 deletion src/Exception.php
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,10 @@
<?php
namespace Transphporm;
/* @description Transformation Style Sheets - Revolutionising PHP templating *
* @author Tom Butler [email protected] *
* @copyright 2017 Tom Butler <[email protected]> | https://r.je/ *
* @license http://www.opensource.org/licenses/bsd-license.php BSD License *
* @version 1.2 */
namespace Transphporm;
class Exception extends \Exception {
const PROPERTY = 'property';
const TSS_FUNCTION = 'function';
Expand Down
7 changes: 6 additions & 1 deletion src/FilePath.php
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,10 @@
<?php
namespace Transphporm;
/* @description Transformation Style Sheets - Revolutionising PHP templating *
* @author Tom Butler [email protected] *
* @copyright 2017 Tom Butler <[email protected]> | https://r.je/ *
* @license http://www.opensource.org/licenses/bsd-license.php BSD License *
* @version 1.2 */
namespace Transphporm;
class FilePath {
private $paths = ['.'];
private $baseDir;
Expand Down
4 changes: 2 additions & 2 deletions src/Formatter/Date.php
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
<?php
/* @description Transformation Style Sheets - Revolutionising PHP templating *
* @author Tom Butler [email protected] *
* @copyright 2015 Tom Butler <[email protected]> | https://r.je/ *
* @copyright 2017 Tom Butler <[email protected]> | https://r.je/ *
* @license http://www.opensource.org/licenses/bsd-license.php BSD License *
* @version 1.0 */
* @version 1.2 */
namespace Transphporm\Formatter;
/** Date/time formatting for use in formmat: property*/
class Date {
Expand Down
7 changes: 6 additions & 1 deletion src/Formatter/HTMLFormatter.php
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,10 @@
<?php
namespace Transphporm\Formatter;
/* @description Transformation Style Sheets - Revolutionising PHP templating *
* @author Tom Butler [email protected] *
* @copyright 2017 Tom Butler <[email protected]> | https://r.je/ *
* @license http://www.opensource.org/licenses/bsd-license.php BSD License *
* @version 1.2 */
namespace Transphporm\Formatter;
class HTMLFormatter {
private $templateFunction;

Expand Down
7 changes: 6 additions & 1 deletion src/Formatter/Nl2brFormat.php
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,10 @@
<?php
namespace Transphporm\Formatter;
/* @description Transformation Style Sheets - Revolutionising PHP templating *
* @author Tom Butler [email protected] *
* @copyright 2017 Tom Butler <[email protected]> | https://r.je/ *
* @license http://www.opensource.org/licenses/bsd-license.php BSD License *
* @version 1.2 */
namespace Transphporm\Formatter;
class Nl2brFormat {
public function nl2br($var) {
$parts = explode("\n", $var);
Expand Down
4 changes: 2 additions & 2 deletions src/Formatter/Number.php
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
<?php
/* @description Transformation Style Sheets - Revolutionising PHP templating *
* @author Tom Butler [email protected] *
* @copyright 2015 Tom Butler <[email protected]> | https://r.je/ *
* @copyright 2017 Tom Butler <[email protected]> | https://r.je/ *
* @license http://www.opensource.org/licenses/bsd-license.php BSD License *
* @version 1.0 */
* @version 1.2 */
namespace Transphporm\Formatter;
class Number {
private $locale;
Expand Down
4 changes: 2 additions & 2 deletions src/Formatter/StringFormatter.php
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
<?php
/* @description Transformation Style Sheets - Revolutionising PHP templating *
* @author Tom Butler [email protected] *
* @copyright 2015 Tom Butler <[email protected]> | https://r.je/ *
* @copyright 2017 Tom Butler <[email protected]> | https://r.je/ *
* @license http://www.opensource.org/licenses/bsd-license.php BSD License *
* @version 1.0 */
* @version 1.2 */
namespace Transphporm\Formatter;
class StringFormatter {
public function uppercase($val) {
Expand Down
4 changes: 2 additions & 2 deletions src/FunctionSet.php
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
<?php
/* @description Transformation Style Sheets - Revolutionising PHP templating *
* @author Tom Butler [email protected] *
* @copyright 2015 Tom Butler <[email protected]> | https://r.je/ *
* @copyright 2017 Tom Butler <[email protected]> | https://r.je/ *
* @license http://www.opensource.org/licenses/bsd-license.php BSD License *
* @version 1.0 */
* @version 1.2 */
namespace Transphporm;
/* Handles data() and iteration() function calls from the stylesheet */
class FunctionSet {
Expand Down
4 changes: 2 additions & 2 deletions src/Hook.php
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
<?php
/* @description Transformation Style Sheets - Revolutionising PHP templating *
* @author Tom Butler [email protected] *
* @copyright 2015 Tom Butler <[email protected]> | https://r.je/ *
* @copyright 2017 Tom Butler <[email protected]> | https://r.je/ *
* @license http://www.opensource.org/licenses/bsd-license.php BSD License *
* @version 1.0 */
* @version 1.2 */
namespace Transphporm;
interface Hook {
public function run(\DomElement $element);
Expand Down
4 changes: 2 additions & 2 deletions src/Hook/ElementData.php
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
<?php
/* @description Transformation Style Sheets - Revolutionising PHP templating *
* @author Tom Butler [email protected] *
* @copyright 2015 Tom Butler <[email protected]> | https://r.je/ *
* @copyright 2017 Tom Butler <[email protected]> | https://r.je/ *
* @license http://www.opensource.org/licenses/bsd-license.php BSD License *
* @version 1.0 */
* @version 1.2 */
namespace Transphporm\Hook;
/* Maps which data is applied to which element */
class ElementData {
Expand Down
4 changes: 2 additions & 2 deletions src/Hook/Formatter.php
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
<?php
/* @description Transformation Style Sheets - Revolutionising PHP templating *
* @author Tom Butler [email protected] *
* @copyright 2015 Tom Butler <[email protected]> | https://r.je/ *
* @copyright 2017 Tom Butler <[email protected]> | https://r.je/ *
* @license http://www.opensource.org/licenses/bsd-license.php BSD License *
* @version 1.0 */
* @version 1.2 */
namespace Transphporm\Hook;
/** Handles format: foo bar properties in the stylesheet */
class Formatter {
Expand Down
4 changes: 2 additions & 2 deletions src/Hook/PostProcess.php
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
<?php
/* @description Transformation Style Sheets - Revolutionising PHP templating *
* @author Tom Butler [email protected] *
* @copyright 2015 Tom Butler <[email protected]> | https://r.je/ *
* @copyright 2017 Tom Butler <[email protected]> | https://r.je/ *
* @license http://www.opensource.org/licenses/bsd-license.php BSD License *
* @version 1.0 */
* @version 1.2 */
namespace Transphporm\Hook;
/** Hooks into the template system, gets assigned as `ul li` or similar and `run()` is called with any elements that match */
class PostProcess implements \Transphporm\Hook {
Expand Down
4 changes: 2 additions & 2 deletions src/Hook/PropertyHook.php
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
<?php
/* @description Transformation Style Sheets - Revolutionising PHP templating *
* @author Tom Butler [email protected] *
* @copyright 2015 Tom Butler <[email protected]> | https://r.je/ *
* @copyright 2017 Tom Butler <[email protected]> | https://r.je/ *
* @license http://www.opensource.org/licenses/bsd-license.php BSD License *
* @version 1.0 */
* @version 1.2 */
namespace Transphporm\Hook;
/** Hooks into the template system, gets assigned as `ul li` or similar and `run()` is called with any elements that match */
class PropertyHook implements \Transphporm\Hook {
Expand Down
4 changes: 2 additions & 2 deletions src/Hook/PseudoMatcher.php
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
<?php
/* @description Transformation Style Sheets - Revolutionising PHP templating *
* @author Tom Butler [email protected] *
* @copyright 2015 Tom Butler <[email protected]> | https://r.je/ *
* @copyright 2017 Tom Butler <[email protected]> | https://r.je/ *
* @license http://www.opensource.org/licenses/bsd-license.php BSD License *
* @version 1.0 */
* @version 1.2 */
namespace Transphporm\Hook;
use \Transphporm\Parser\Tokenizer;
/** Determines whether $element matches the pseudo rule such as nth-child() or [attribute="value"] */
Expand Down
4 changes: 2 additions & 2 deletions src/Module.php
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
<?php
/* @description Transformation Style Sheets - Revolutionising PHP templating *
* @author Tom Butler [email protected] *
* @copyright 2015 Tom Butler <[email protected]> | https://r.je/ *
* @copyright 2017 Tom Butler <[email protected]> | https://r.je/ *
* @license http://www.opensource.org/licenses/bsd-license.php BSD License *
* @version 1.0 */
* @version 1.2 */
namespace Transphporm;
interface Module {
public function load(Config $config);
Expand Down
4 changes: 2 additions & 2 deletions src/Module/Basics.php
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
<?php
/* @description Transformation Style Sheets - Revolutionising PHP templating *
* @author Tom Butler [email protected] *
* @copyright 2015 Tom Butler <[email protected]> | https://r.je/ *
* @copyright 2017 Tom Butler <[email protected]> | https://r.je/ *
* @license http://www.opensource.org/licenses/bsd-license.php BSD License *
* @version 1.0 */
* @version 1.2 */
namespace Transphporm\Module;
/** Assigns all the basic properties repeat, comment, etc to a $builder instance */
class Basics implements \Transphporm\Module {
Expand Down
4 changes: 2 additions & 2 deletions src/Module/Format.php
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
<?php
/* @description Transformation Style Sheets - Revolutionising PHP templating *
* @author Tom Butler [email protected] *
* @copyright 2015 Tom Butler <[email protected]> | https://r.je/ *
* @copyright 2017 Tom Butler <[email protected]> | https://r.je/ *
* @license http://www.opensource.org/licenses/bsd-license.php BSD License *
* @version 1.0 */
* @version 1.2 */
namespace Transphporm\Module;
/** Module for loading a formatter with a locale */
class Format implements \Transphporm\Module {
Expand Down
4 changes: 2 additions & 2 deletions src/Module/Functions.php
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
<?php
/* @description Transformation Style Sheets - Revolutionising PHP templating *
* @author Tom Butler [email protected] *
* @copyright 2015 Tom Butler <[email protected]> | https://r.je/ *
* @copyright 2017 Tom Butler <[email protected]> | https://r.je/ *
* @license http://www.opensource.org/licenses/bsd-license.php BSD License *
* @version 1.0 */
* @version 1.2 */
namespace Transphporm\Module;
use Transphporm\TSSFunction\Math;
/** Assigns all the basic functions, data(), key(), iteration(), template(), etc */
Expand Down
4 changes: 2 additions & 2 deletions src/Module/Pseudo.php
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
<?php
/* @description Transformation Style Sheets - Revolutionising PHP templating *
* @author Tom Butler [email protected] *
* @copyright 2015 Tom Butler <[email protected]> | https://r.je/ *
* @copyright 2017 Tom Butler <[email protected]> | https://r.je/ *
* @license http://www.opensource.org/licenses/bsd-license.php BSD License *
* @version 1.0 */
* @version 1.2 */
namespace Transphporm\Module;
/** Module for loading a formatter with a locale */
class Pseudo implements \Transphporm\Module {
Expand Down
34 changes: 19 additions & 15 deletions src/Parser/CssToXpath.php
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
<?php
/* @description Transformation Style Sheets - Revolutionising PHP templating *
* @author Tom Butler [email protected] *
* @copyright 2015 Tom Butler <[email protected]> | https://r.je/ *
* @copyright 2017 Tom Butler <[email protected]> | https://r.je/ *
* @license http://www.opensource.org/licenses/bsd-license.php BSD License *
* @version 1.0 */
* @version 1.2 */
namespace Transphporm\Parser;
class CssToXpath {
private $specialChars = [Tokenizer::WHITESPACE, Tokenizer::DOT, Tokenizer::GREATER_THAN,
Expand All @@ -20,7 +20,7 @@ public function __construct(\Transphporm\FunctionSet $functionSet, $prefix = '',

$this->translators = [
Tokenizer::WHITESPACE => function($string) use ($prefix) { return '//' . $prefix . $string; },
Tokenizer::MULTIPLY => function () { return '*'; },
Tokenizer::MULTIPLY => function () { return '*'; },
'' => function($string) use ($prefix) { return '/' . $prefix . $string; },
Tokenizer::GREATER_THAN => function($string) use ($prefix) { return '/' . $prefix . $string; },
Tokenizer::NUM_SIGN => function($string) { return '[@id=\'' . $string . '\']'; },
Expand All @@ -43,15 +43,14 @@ public static function processAttr($attr, $element, $hash) {
$functionSet = self::$instances[$hash]->functionSet;
$functionSet->setElement($element[0]);

$attributes = array();
foreach($element[0]->attributes as $attribute_name => $attribute_node) {
$attributes[$attribute_name] = $attribute_node->nodeValue;
}
$attributes = [];
foreach($element[0]->attributes as $name => $node) {
$attributes[$name] = $node->nodeValue;
}

$parser = new \Transphporm\Parser\Value($functionSet, true);
$parser = new \Transphporm\Parser\Value($functionSet, true);
$return = $parser->parseTokens($attr, $attributes);

return $return[0] === '' ? false : $return[0];
return $return[0] === '' ? false : $return[0];
}

public function cleanup() {
Expand Down Expand Up @@ -89,12 +88,17 @@ public function getXpath($css) {
}

private function removeSpacesFromDirectDecend($css) {
$tokens = [];
foreach ($css->splitOnToken(Tokenizer::GREATER_THAN) as $token) {
foreach ($token->trim() as $t) $tokens[] = $t;
$tokens[] = ['type' => Tokenizer::GREATER_THAN];
$tokens = new Tokens;
$split = $css->splitOnToken(Tokenizer::GREATER_THAN);

if (count($split) <= 1) return $css;

for ($i = 0; $i < count($split); $i++) {
$tokens->add($split[$i]->trim());
if (isset($split[$i+1])) $tokens->add(['type' => Tokenizer::GREATER_THAN]);
}
return new Tokens(array_slice($tokens, 0, -1));

return $tokens;
}


Expand Down
4 changes: 2 additions & 2 deletions src/Parser/Last.php
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
<?php
/* @description Transformation Style Sheets - Revolutionising PHP templating *
* @author Tom Butler [email protected] *
* @copyright 2015 Tom Butler <[email protected]> | https://r.je/ *
* @copyright 2017 Tom Butler <[email protected]> | https://r.je/ *
* @license http://www.opensource.org/licenses/bsd-license.php BSD License *
* @version 1.0 */
* @version 1.2 */
namespace Transphporm\Parser;
/** Parses "string" and function(args) e.g. data(foo) or iteration(bar) */
class Last {
Expand Down
Loading