Merge context sensitive lexer RFC

Zend/tests/grammar/regression_001.phpt

@@ -0,0 +1,33 @@
+--TEST--
+Test to check static method calls syntax regression
+--FILE--
+<?php
+
+class Foo {
+	public static function function(){ echo __METHOD__, PHP_EOL; }
+}
+
+Foo::function();
+
+Foo::
+function();
+
+Foo::
+	 function();
+
+
+Foo::
+	 function(
+
+);
+
+echo "\nDone\n";
+
+--EXPECTF--
+
+Foo::function
+Foo::function
+Foo::function
+Foo::function
+
+Done

Zend/tests/grammar/regression_002.phpt

@@ -0,0 +1,22 @@
+--TEST--
+Test to ensure ::class still works
+--FILE--
+<?php
+
+class Foo {}
+
+var_dump(Foo::class);
+
+var_dump(Foo:: class);
+
+var_dump(Foo::	 CLASS);
+
+var_dump(Foo::
+
+CLASS);
+
+--EXPECTF--
+string(3) "Foo"
+string(3) "Foo"
+string(3) "Foo"
+string(3) "Foo"

Zend/tests/grammar/regression_003.phpt

@@ -0,0 +1,12 @@
+--TEST--
+Test to ensure ::class is still reserved in obj scope
+--FILE--
+<?php
+
+class Obj
+{
+    const CLASS = 'class';
+}
+
+--EXPECTF--
+Parse error: syntax error, unexpected 'CLASS' (T_CLASS) in %s on line 5

Zend/tests/grammar/regression_004.phpt

@@ -0,0 +1,15 @@
+--TEST--
+Test possible function naming regression on procedural scope
+--FILE--
+<?php
+
+class Obj
+{
+    function echo(){} // valid
+    function return(){} // valid
+}
+
+function echo(){} // not valid
+
+--EXPECTF--
+Parse error: syntax error, unexpected 'echo' (T_ECHO), expecting identifier (T_STRING) or '(' in %s on line 9

Zend/tests/grammar/regression_005.phpt

@@ -0,0 +1,14 @@
+--TEST--
+Test possible constant naming regression on procedural scope
+--FILE--
+<?php
+
+class Obj
+{
+    const return = 'yep';
+}
+
+const return = 'nope';
+
+--EXPECTF--
+Parse error: syntax error, unexpected 'return' (T_RETURN), expecting identifier (T_STRING) in %s on line 8

Zend/tests/grammar/regression_006.phpt

@@ -0,0 +1,30 @@
+--TEST--
+Test to ensure const list syntax declaration works
+--FILE--
+<?php
+
+class Obj
+{
+    const DECLARE = 'declare',
+          RETURN = 'return',
+          FUNCTION = 'function',
+          USE = 'use';
+}
+
+echo Obj::DECLARE, PHP_EOL;
+echo Obj::RETURN, PHP_EOL;
+echo Obj::FUNCTION, PHP_EOL;
+echo Obj::USE, PHP_EOL;
+echo Obj::
+
+    USE, PHP_EOL;
+echo "\nDone\n";
+
+--EXPECTF--
+declare
+return
+function
+use
+use
+
+Done

Zend/tests/grammar/regression_007.phpt

@@ -0,0 +1,44 @@
+--TEST--
+Test to ensure semi reserved words allow deference
+--FILE--
+<?php
+
+class Foo {
+    const use = 'yay';
+
+    public static function new() {
+        echo __METHOD__, PHP_EOL;
+        return new static();
+    }
+
+    public function self() {
+        echo __METHOD__, PHP_EOL;
+        return $this;
+    }
+}
+
+Foo::new()::new()::new();
+
+var_dump(
+    (new Foo)->self()::new()->self()->self()::use
+);
+
+Foo::{'new'}();
+
+var_dump(Foo::use);
+
+echo "\nDone\n";
+
+--EXPECTF--
+Foo::new
+Foo::new
+Foo::new
+Foo::self
+Foo::new
+Foo::self
+Foo::self
+string(3) "yay"
+Foo::new
+string(3) "yay"
+
+Done

Zend/tests/grammar/regression_008.phpt

@@ -0,0 +1,21 @@
+--TEST--
+Test to check regressions on string interpolation with class members access
+--FILE--
+<?php
+
+class Friday {
+    public $require = "fun";
+}
+
+$friday = new Friday;
+
+echo "$friday->require ($friday->require) {$friday->require}", PHP_EOL;
+
+echo "\nDone\n";
+
+
+--EXPECTF--
+
+fun (fun) fun
+
+Done

Zend/tests/grammar/regression_009.phpt

@@ -0,0 +1,18 @@
+--TEST--
+Test to check regressions on use statements and lexer state
+--FILE--
+<?php
+
+use A\B\C\D;
+
+class Foo
+{
+    private static $foo;
+
+}
+
+echo PHP_EOL, "Done", PHP_EOL;
+
+--EXPECTF--
+
+Done

Zend/tests/grammar/regression_010.phpt

@@ -0,0 +1,14 @@
+--TEST--
+Test to check regressions on T_IMPLEMENTS followed by a T_NS_SEPARATOR
+--FILE--
+<?php
+
+interface A{}
+
+class B implements\A {}
+
+echo "Done", PHP_EOL;
+
+--EXPECTF--
+
+Done

Zend/tests/grammar/regression_011.phpt

@@ -0,0 +1,18 @@
+--TEST--
+Testing instantiation using namespace:: prefix
+--FILE--
+<?php
+
+namespace foo;
+
+class bar {
+}
+
+class_alias('foo\bar', 'foo\baz');
+
+var_dump(new namespace\baz);
+
+?>
+--EXPECTF--
+object(foo\bar)#%d (0) {
+}

Zend/tests/grammar/regression_012.phpt

@@ -0,0 +1,13 @@
+--TEST--
+Testing for regression on const list syntax and arrays
+--FILE--
+<?php
+
+class A {
+    const A = [1, FOREACH];
+}
+
+?>
+--EXPECTF--
+
+Parse error: syntax error, unexpected 'FOREACH' (T_FOREACH), expecting ']' in %s on line %d

Zend/tests/grammar/regression_013.phpt

@@ -0,0 +1,13 @@
+--TEST--
+Testing for regression with encapsed variables in class declaration context
+--FILE--
+<?php
+
+class A { function foo() { "{${$a}}"; } function list() {} }
+
+echo "Done", PHP_EOL;
+
+?>
+--EXPECTF--
+
+Done

Zend/tests/grammar/semi_reserved_001.phpt

@@ -0,0 +1,188 @@
+--TEST--
+Test semi-reserved words as class methods
+--FILE--
+<?php
+
+class Obj
+{
+    function empty(){ echo __METHOD__, PHP_EOL; }
+    function callable(){ echo __METHOD__, PHP_EOL; }
+    function trait(){ echo __METHOD__, PHP_EOL; }
+    function extends(){ echo __METHOD__, PHP_EOL; }
+    function implements(){ echo __METHOD__, PHP_EOL; }
+    function const(){ echo __METHOD__, PHP_EOL; }
+    function enddeclare(){ echo __METHOD__, PHP_EOL; }
+    function endfor(){ echo __METHOD__, PHP_EOL; }
+    function endforeach(){ echo __METHOD__, PHP_EOL; }
+    function endif(){ echo __METHOD__, PHP_EOL; }
+    function endwhile(){ echo __METHOD__, PHP_EOL; }
+    function and(){ echo __METHOD__, PHP_EOL; }
+    function global(){ echo __METHOD__, PHP_EOL; }
+    function goto(){ echo __METHOD__, PHP_EOL; }
+    function instanceof(){ echo __METHOD__, PHP_EOL; }
+    function insteadof(){ echo __METHOD__, PHP_EOL; }
+    function interface(){ echo __METHOD__, PHP_EOL; }
+    function new(){ echo __METHOD__, PHP_EOL; }
+    function or(){ echo __METHOD__, PHP_EOL; }
+    function xor(){ echo __METHOD__, PHP_EOL; }
+    function try(){ echo __METHOD__, PHP_EOL; }
+    function use(){ echo __METHOD__, PHP_EOL; }
+    function var(){ echo __METHOD__, PHP_EOL; }
+    function exit(){ echo __METHOD__, PHP_EOL; }
+    function list(){ echo __METHOD__, PHP_EOL; }
+    function clone(){ echo __METHOD__, PHP_EOL; }
+    function include(){ echo __METHOD__, PHP_EOL; }
+    function include_once(){ echo __METHOD__, PHP_EOL; }
+    function throw(){ echo __METHOD__, PHP_EOL; }
+    function array(){ echo __METHOD__, PHP_EOL; }
+    function print(){ echo __METHOD__, PHP_EOL; }
+    function echo(){ echo __METHOD__, PHP_EOL; }
+    function require(){ echo __METHOD__, PHP_EOL; }
+    function require_once(){ echo __METHOD__, PHP_EOL; }
+    function return(){ echo __METHOD__, PHP_EOL; }
+    function else(){ echo __METHOD__, PHP_EOL; }
+    function elseif(){ echo __METHOD__, PHP_EOL; }
+    function default(){ echo __METHOD__, PHP_EOL; }
+    function break(){ echo __METHOD__, PHP_EOL; }
+    function continue(){ echo __METHOD__, PHP_EOL; }
+    function switch(){ echo __METHOD__, PHP_EOL; }
+    function yield(){ echo __METHOD__, PHP_EOL; }
+    function function(){ echo __METHOD__, PHP_EOL; }
+    function if(){ echo __METHOD__, PHP_EOL; }
+    function endswitch(){ echo __METHOD__, PHP_EOL; }
+    function finally(){ echo __METHOD__, PHP_EOL; }
+    function for(){ echo __METHOD__, PHP_EOL; }
+    function foreach(){ echo __METHOD__, PHP_EOL; }
+    function declare(){ echo __METHOD__, PHP_EOL; }
+    function case(){ echo __METHOD__, PHP_EOL; }
+    function do(){ echo __METHOD__, PHP_EOL; }
+    function while(){ echo __METHOD__, PHP_EOL; }
+    function as(){ echo __METHOD__, PHP_EOL; }
+    function catch(){ echo __METHOD__, PHP_EOL; }
+    function die(){ echo __METHOD__, PHP_EOL; }
+    function self(){ echo __METHOD__, PHP_EOL; }
+    function parent(){ echo __METHOD__, PHP_EOL; }
+}
+
+$obj = new Obj;
+
+$obj->empty();
+$obj->callable();
+$obj->trait();
+$obj->extends();
+$obj->implements();
+$obj->const();
+$obj->enddeclare();
+$obj->endfor();
+$obj->endforeach();
+$obj->endif();
+$obj->endwhile();
+$obj->and();
+$obj->global();
+$obj->goto();
+$obj->instanceof();
+$obj->insteadof();
+$obj->interface();
+$obj->new();
+$obj->or();
+$obj->xor();
+$obj->try();
+$obj->use();
+$obj->var();
+$obj->exit();
+$obj->list();
+$obj->clone();
+$obj->include();
+$obj->include_once();
+$obj->throw();
+$obj->array();
+$obj->print();
+$obj->echo();
+$obj->require();
+$obj->require_once();
+$obj->return();
+$obj->else();
+$obj->elseif();
+$obj->default();
+$obj->break();
+$obj->continue();
+$obj->switch();
+$obj->yield();
+$obj->function();
+$obj->if();
+$obj->endswitch();
+$obj->finally();
+$obj->for();
+$obj->foreach();
+$obj->declare();
+$obj->case();
+$obj->do();
+$obj->while();
+$obj->as();
+$obj->catch();
+$obj->die();
+$obj->self();
+$obj->parent();
+
+echo "\nDone\n";
+
+--EXPECTF--
+Obj::empty
+Obj::callable
+Obj::trait
+Obj::extends
+Obj::implements
+Obj::const
+Obj::enddeclare
+Obj::endfor
+Obj::endforeach
+Obj::endif
+Obj::endwhile
+Obj::and
+Obj::global
+Obj::goto
+Obj::instanceof
+Obj::insteadof
+Obj::interface
+Obj::new
+Obj::or
+Obj::xor
+Obj::try
+Obj::use
+Obj::var
+Obj::exit
+Obj::list
+Obj::clone
+Obj::include
+Obj::include_once
+Obj::throw
+Obj::array
+Obj::print
+Obj::echo
+Obj::require
+Obj::require_once
+Obj::return
+Obj::else
+Obj::elseif
+Obj::default
+Obj::break
+Obj::continue
+Obj::switch
+Obj::yield
+Obj::function
+Obj::if
+Obj::endswitch
+Obj::finally
+Obj::for
+Obj::foreach
+Obj::declare
+Obj::case
+Obj::do
+Obj::while
+Obj::as
+Obj::catch
+Obj::die
+Obj::self
+Obj::parent
+
+Done

Zend/tests/grammar/semi_reserved_002.phpt

@@ -0,0 +1,186 @@
+--TEST--
+Test semi-reserved words as static class methods
+--FILE--
+<?php
+
+class Obj
+{
+    static function empty(){ echo __METHOD__, PHP_EOL; }
+    static function callable(){ echo __METHOD__, PHP_EOL; }
+    static function trait(){ echo __METHOD__, PHP_EOL; }
+    static function extends(){ echo __METHOD__, PHP_EOL; }
+    static function implements(){ echo __METHOD__, PHP_EOL; }
+    static function const(){ echo __METHOD__, PHP_EOL; }
+    static function enddeclare(){ echo __METHOD__, PHP_EOL; }
+    static function endfor(){ echo __METHOD__, PHP_EOL; }
+    static function endforeach(){ echo __METHOD__, PHP_EOL; }
+    static function endif(){ echo __METHOD__, PHP_EOL; }
+    static function endwhile(){ echo __METHOD__, PHP_EOL; }
+    static function and(){ echo __METHOD__, PHP_EOL; }
+    static function global(){ echo __METHOD__, PHP_EOL; }
+    static function goto(){ echo __METHOD__, PHP_EOL; }
+    static function instanceof(){ echo __METHOD__, PHP_EOL; }
+    static function insteadof(){ echo __METHOD__, PHP_EOL; }
+    static function interface(){ echo __METHOD__, PHP_EOL; }
+    static function new(){ echo __METHOD__, PHP_EOL; }
+    static function or(){ echo __METHOD__, PHP_EOL; }
+    static function xor(){ echo __METHOD__, PHP_EOL; }
+    static function try(){ echo __METHOD__, PHP_EOL; }
+    static function use(){ echo __METHOD__, PHP_EOL; }
+    static function var(){ echo __METHOD__, PHP_EOL; }
+    static function exit(){ echo __METHOD__, PHP_EOL; }
+    static function list(){ echo __METHOD__, PHP_EOL; }
+    static function clone(){ echo __METHOD__, PHP_EOL; }
+    static function include(){ echo __METHOD__, PHP_EOL; }
+    static function include_once(){ echo __METHOD__, PHP_EOL; }
+    static function throw(){ echo __METHOD__, PHP_EOL; }
+    static function array(){ echo __METHOD__, PHP_EOL; }
+    static function print(){ echo __METHOD__, PHP_EOL; }
+    static function echo(){ echo __METHOD__, PHP_EOL; }
+    static function require(){ echo __METHOD__, PHP_EOL; }
+    static function require_once(){ echo __METHOD__, PHP_EOL; }
+    static function return(){ echo __METHOD__, PHP_EOL; }
+    static function else(){ echo __METHOD__, PHP_EOL; }
+    static function elseif(){ echo __METHOD__, PHP_EOL; }
+    static function default(){ echo __METHOD__, PHP_EOL; }
+    static function break(){ echo __METHOD__, PHP_EOL; }
+    static function continue(){ echo __METHOD__, PHP_EOL; }
+    static function switch(){ echo __METHOD__, PHP_EOL; }
+    static function yield(){ echo __METHOD__, PHP_EOL; }
+    static function function(){ echo __METHOD__, PHP_EOL; }
+    static function if(){ echo __METHOD__, PHP_EOL; }
+    static function endswitch(){ echo __METHOD__, PHP_EOL; }
+    static function finally(){ echo __METHOD__, PHP_EOL; }
+    static function for(){ echo __METHOD__, PHP_EOL; }
+    static function foreach(){ echo __METHOD__, PHP_EOL; }
+    static function declare(){ echo __METHOD__, PHP_EOL; }
+    static function case(){ echo __METHOD__, PHP_EOL; }
+    static function do(){ echo __METHOD__, PHP_EOL; }
+    static function while(){ echo __METHOD__, PHP_EOL; }
+    static function as(){ echo __METHOD__, PHP_EOL; }
+    static function catch(){ echo __METHOD__, PHP_EOL; }
+    static function die(){ echo __METHOD__, PHP_EOL; }
+    static function self(){ echo __METHOD__, PHP_EOL; }
+    static function parent(){ echo __METHOD__, PHP_EOL; }
+}
+
+Obj::empty();
+Obj::callable();
+Obj::trait();
+Obj::extends();
+Obj::implements();
+Obj::const();
+Obj::enddeclare();
+Obj::endfor();
+Obj::endforeach();
+Obj::endif();
+Obj::endwhile();
+Obj::and();
+Obj::global();
+Obj::goto();
+Obj::instanceof();
+Obj::insteadof();
+Obj::interface();
+Obj::new();
+Obj::or();
+Obj::xor();
+Obj::try();
+Obj::use();
+Obj::var();
+Obj::exit();
+Obj::list();
+Obj::clone();
+Obj::include();
+Obj::include_once();
+Obj::throw();
+Obj::array();
+Obj::print();
+Obj::echo();
+Obj::require();
+Obj::require_once();
+Obj::return();
+Obj::else();
+Obj::elseif();
+Obj::default();
+Obj::break();
+Obj::continue();
+Obj::switch();
+Obj::yield();
+Obj::function();
+Obj::if();
+Obj::endswitch();
+Obj::finally();
+Obj::for();
+Obj::foreach();
+Obj::declare();
+Obj::case();
+Obj::do();
+Obj::while();
+Obj::as();
+Obj::catch();
+Obj::die();
+Obj::self();
+Obj::parent();
+
+echo "\nDone\n";
+
+--EXPECTF--
+Obj::empty
+Obj::callable
+Obj::trait
+Obj::extends
+Obj::implements
+Obj::const
+Obj::enddeclare
+Obj::endfor
+Obj::endforeach
+Obj::endif
+Obj::endwhile
+Obj::and
+Obj::global
+Obj::goto
+Obj::instanceof
+Obj::insteadof
+Obj::interface
+Obj::new
+Obj::or
+Obj::xor
+Obj::try
+Obj::use
+Obj::var
+Obj::exit
+Obj::list
+Obj::clone
+Obj::include
+Obj::include_once
+Obj::throw
+Obj::array
+Obj::print
+Obj::echo
+Obj::require
+Obj::require_once
+Obj::return
+Obj::else
+Obj::elseif
+Obj::default
+Obj::break
+Obj::continue
+Obj::switch
+Obj::yield
+Obj::function
+Obj::if
+Obj::endswitch
+Obj::finally
+Obj::for
+Obj::foreach
+Obj::declare
+Obj::case
+Obj::do
+Obj::while
+Obj::as
+Obj::catch
+Obj::die
+Obj::self
+Obj::parent
+
+Done

Zend/tests/grammar/semi_reserved_003.phpt

@@ -0,0 +1,210 @@
+--TEST--
+Test semi-reserved words as class properties
+--FILE--
+<?php
+
+class Obj
+{
+    var $empty = 'empty';
+    var $callable = 'callable';
+    var $class = 'class';
+    var $trait = 'trait';
+    var $extends = 'extends';
+    var $implements = 'implements';
+    var $static = 'static';
+    var $abstract = 'abstract';
+    var $final = 'final';
+    var $public = 'public';
+    var $protected = 'protected';
+    var $private = 'private';
+    var $const = 'const';
+    var $enddeclare = 'enddeclare';
+    var $endfor = 'endfor';
+    var $endforeach = 'endforeach';
+    var $endif = 'endif';
+    var $endwhile = 'endwhile';
+    var $and = 'and';
+    var $global = 'global';
+    var $goto = 'goto';
+    var $instanceof = 'instanceof';
+    var $insteadof = 'insteadof';
+    var $interface = 'interface';
+    var $namespace = 'namespace';
+    var $new = 'new';
+    var $or = 'or';
+    var $xor = 'xor';
+    var $try = 'try';
+    var $use = 'use';
+    var $var = 'var';
+    var $exit = 'exit';
+    var $list = 'list';
+    var $clone = 'clone';
+    var $include = 'include';
+    var $include_once = 'include_once';
+    var $throw = 'throw';
+    var $array = 'array';
+    var $print = 'print';
+    var $echo = 'echo';
+    var $require = 'require';
+    var $require_once = 'require_once';
+    var $return = 'return';
+    var $else = 'else';
+    var $elseif = 'elseif';
+    var $default = 'default';
+    var $break = 'break';
+    var $continue = 'continue';
+    var $switch = 'switch';
+    var $yield = 'yield';
+    var $function = 'function';
+    var $if = 'if';
+    var $endswitch = 'endswitch';
+    var $finally = 'finally';
+    var $for = 'for';
+    var $foreach = 'foreach';
+    var $declare = 'declare';
+    var $case = 'case';
+    var $do = 'do';
+    var $while = 'while';
+    var $as = 'as';
+    var $catch = 'catch';
+    var $die = 'die';
+    var $self = 'self';
+}
+
+$obj = new Obj;
+
+echo $obj->empty, PHP_EOL;
+echo $obj->callable, PHP_EOL;
+echo $obj->class, PHP_EOL;
+echo $obj->trait, PHP_EOL;
+echo $obj->extends, PHP_EOL;
+echo $obj->implements, PHP_EOL;
+echo $obj->static, PHP_EOL;
+echo $obj->abstract, PHP_EOL;
+echo $obj->final, PHP_EOL;
+echo $obj->public, PHP_EOL;
+echo $obj->protected, PHP_EOL;
+echo $obj->private, PHP_EOL;
+echo $obj->const, PHP_EOL;
+echo $obj->enddeclare, PHP_EOL;
+echo $obj->endfor, PHP_EOL;
+echo $obj->endforeach, PHP_EOL;
+echo $obj->endif, PHP_EOL;
+echo $obj->endwhile, PHP_EOL;
+echo $obj->and, PHP_EOL;
+echo $obj->global, PHP_EOL;
+echo $obj->goto, PHP_EOL;
+echo $obj->instanceof, PHP_EOL;
+echo $obj->insteadof, PHP_EOL;
+echo $obj->interface, PHP_EOL;
+echo $obj->namespace, PHP_EOL;
+echo $obj->new, PHP_EOL;
+echo $obj->or, PHP_EOL;
+echo $obj->xor, PHP_EOL;
+echo $obj->try, PHP_EOL;
+echo $obj->use, PHP_EOL;
+echo $obj->var, PHP_EOL;
+echo $obj->exit, PHP_EOL;
+echo $obj->list, PHP_EOL;
+echo $obj->clone, PHP_EOL;
+echo $obj->include, PHP_EOL;
+echo $obj->include_once, PHP_EOL;
+echo $obj->throw, PHP_EOL;
+echo $obj->array, PHP_EOL;
+echo $obj->print, PHP_EOL;
+echo $obj->echo, PHP_EOL;
+echo $obj->require, PHP_EOL;
+echo $obj->require_once, PHP_EOL;
+echo $obj->return, PHP_EOL;
+echo $obj->else, PHP_EOL;
+echo $obj->elseif, PHP_EOL;
+echo $obj->default, PHP_EOL;
+echo $obj->break, PHP_EOL;
+echo $obj->continue, PHP_EOL;
+echo $obj->switch, PHP_EOL;
+echo $obj->yield, PHP_EOL;
+echo $obj->function, PHP_EOL;
+echo $obj->if, PHP_EOL;
+echo $obj->endswitch, PHP_EOL;
+echo $obj->finally, PHP_EOL;
+echo $obj->for, PHP_EOL;
+echo $obj->foreach, PHP_EOL;
+echo $obj->declare, PHP_EOL;
+echo $obj->case, PHP_EOL;
+echo $obj->do, PHP_EOL;
+echo $obj->while, PHP_EOL;
+echo $obj->as, PHP_EOL;
+echo $obj->catch, PHP_EOL;
+echo $obj->die, PHP_EOL;
+echo $obj->self, PHP_EOL;
+
+echo "\nDone\n";
+
+?>
+--EXPECTF--
+empty
+callable
+class
+trait
+extends
+implements
+static
+abstract
+final
+public
+protected
+private
+const
+enddeclare
+endfor
+endforeach
+endif
+endwhile
+and
+global
+goto
+instanceof
+insteadof
+interface
+namespace
+new
+or
+xor
+try
+use
+var
+exit
+list
+clone
+include
+include_once
+throw
+array
+print
+echo
+require
+require_once
+return
+else
+elseif
+default
+break
+continue
+switch
+yield
+function
+if
+endswitch
+finally
+for
+foreach
+declare
+case
+do
+while
+as
+catch
+die
+self
+
+Done

Zend/tests/grammar/semi_reserved_004.phpt

@@ -0,0 +1,210 @@
+--TEST--
+Test semi-reserved words as static class properties
+--FILE--
+<?php
+
+class Obj
+{
+    static $empty = 'empty';
+    static $callable = 'callable';
+    static $class = 'class';
+    static $trait = 'trait';
+    static $extends = 'extends';
+    static $implements = 'implements';
+    static $static = 'static';
+    static $abstract = 'abstract';
+    static $final = 'final';
+    static $public = 'public';
+    static $protected = 'protected';
+    static $private = 'private';
+    static $const = 'const';
+    static $enddeclare = 'enddeclare';
+    static $endfor = 'endfor';
+    static $endforeach = 'endforeach';
+    static $endif = 'endif';
+    static $endwhile = 'endwhile';
+    static $and = 'and';
+    static $global = 'global';
+    static $goto = 'goto';
+    static $instanceof = 'instanceof';
+    static $insteadof = 'insteadof';
+    static $interface = 'interface';
+    static $namespace = 'namespace';
+    static $new = 'new';
+    static $or = 'or';
+    static $xor = 'xor';
+    static $try = 'try';
+    static $use = 'use';
+    static $var = 'var';
+    static $exit = 'exit';
+    static $list = 'list';
+    static $clone = 'clone';
+    static $include = 'include';
+    static $include_once = 'include_once';
+    static $throw = 'throw';
+    static $array = 'array';
+    static $print = 'print';
+    static $echo = 'echo';
+    static $require = 'require';
+    static $require_once = 'require_once';
+    static $return = 'return';
+    static $else = 'else';
+    static $elseif = 'elseif';
+    static $default = 'default';
+    static $break = 'break';
+    static $continue = 'continue';
+    static $switch = 'switch';
+    static $yield = 'yield';
+    static $function = 'function';
+    static $if = 'if';
+    static $endswitch = 'endswitch';
+    static $finally = 'finally';
+    static $for = 'for';
+    static $foreach = 'foreach';
+    static $declare = 'declare';
+    static $case = 'case';
+    static $do = 'do';
+    static $while = 'while';
+    static $as = 'as';
+    static $catch = 'catch';
+    static $die = 'die';
+    static $self = 'self';
+    static $parent = 'parent';
+}
+
+echo Obj::$empty, PHP_EOL;
+echo Obj::$callable, PHP_EOL;
+echo Obj::$class, PHP_EOL;
+echo Obj::$trait, PHP_EOL;
+echo Obj::$extends, PHP_EOL;
+echo Obj::$implements, PHP_EOL;
+echo Obj::$static, PHP_EOL;
+echo Obj::$abstract, PHP_EOL;
+echo Obj::$final, PHP_EOL;
+echo Obj::$public, PHP_EOL;
+echo Obj::$protected, PHP_EOL;
+echo Obj::$private, PHP_EOL;
+echo Obj::$const, PHP_EOL;
+echo Obj::$enddeclare, PHP_EOL;
+echo Obj::$endfor, PHP_EOL;
+echo Obj::$endforeach, PHP_EOL;
+echo Obj::$endif, PHP_EOL;
+echo Obj::$endwhile, PHP_EOL;
+echo Obj::$and, PHP_EOL;
+echo Obj::$global, PHP_EOL;
+echo Obj::$goto, PHP_EOL;
+echo Obj::$instanceof, PHP_EOL;
+echo Obj::$insteadof, PHP_EOL;
+echo Obj::$interface, PHP_EOL;
+echo Obj::$namespace, PHP_EOL;
+echo Obj::$new, PHP_EOL;
+echo Obj::$or, PHP_EOL;
+echo Obj::$xor, PHP_EOL;
+echo Obj::$try, PHP_EOL;
+echo Obj::$use, PHP_EOL;
+echo Obj::$var, PHP_EOL;
+echo Obj::$exit, PHP_EOL;
+echo Obj::$list, PHP_EOL;
+echo Obj::$clone, PHP_EOL;
+echo Obj::$include, PHP_EOL;
+echo Obj::$include_once, PHP_EOL;
+echo Obj::$throw, PHP_EOL;
+echo Obj::$array, PHP_EOL;
+echo Obj::$print, PHP_EOL;
+echo Obj::$echo, PHP_EOL;
+echo Obj::$require, PHP_EOL;
+echo Obj::$require_once, PHP_EOL;
+echo Obj::$return, PHP_EOL;
+echo Obj::$else, PHP_EOL;
+echo Obj::$elseif, PHP_EOL;
+echo Obj::$default, PHP_EOL;
+echo Obj::$break, PHP_EOL;
+echo Obj::$continue, PHP_EOL;
+echo Obj::$switch, PHP_EOL;
+echo Obj::$yield, PHP_EOL;
+echo Obj::$function, PHP_EOL;
+echo Obj::$if, PHP_EOL;
+echo Obj::$endswitch, PHP_EOL;
+echo Obj::$finally, PHP_EOL;
+echo Obj::$for, PHP_EOL;
+echo Obj::$foreach, PHP_EOL;
+echo Obj::$declare, PHP_EOL;
+echo Obj::$case, PHP_EOL;
+echo Obj::$do, PHP_EOL;
+echo Obj::$while, PHP_EOL;
+echo Obj::$as, PHP_EOL;
+echo Obj::$catch, PHP_EOL;
+echo Obj::$die, PHP_EOL;
+echo Obj::$self, PHP_EOL;
+echo Obj::$parent, PHP_EOL;
+
+echo "\nDone\n";
+
+--EXPECTF--
+empty
+callable
+class
+trait
+extends
+implements
+static
+abstract
+final
+public
+protected
+private
+const
+enddeclare
+endfor
+endforeach
+endif
+endwhile
+and
+global
+goto
+instanceof
+insteadof
+interface
+namespace
+new
+or
+xor
+try
+use
+var
+exit
+list
+clone
+include
+include_once
+throw
+array
+print
+echo
+require
+require_once
+return
+else
+elseif
+default
+break
+continue
+switch
+yield
+function
+if
+endswitch
+finally
+for
+foreach
+declare
+case
+do
+while
+as
+catch
+die
+self
+parent
+
+Done

Zend/tests/grammar/semi_reserved_005.phpt

@@ -0,0 +1,189 @@
+--TEST--
+Test semi-reserved words as class constants
+--FILE--
+<?php
+
+class Obj
+{
+    const EMPTY = 'empty';
+    const CALLABLE = 'callable';
+    const TRAIT = 'trait';
+    const EXTENDS = 'extends';
+    const IMPLEMENTS = 'implements';
+    const CONST = 'const';
+    const ENDDECLARE = 'enddeclare';
+    const ENDFOR = 'endfor';
+    const ENDFOREACH = 'endforeach';
+    const ENDIF = 'endif';
+    const ENDWHILE = 'endwhile';
+    const AND = 'and';
+    const GLOBAL = 'global';
+    const GOTO = 'goto';
+    const INSTANCEOF = 'instanceof';
+    const INSTEADOF = 'insteadof';
+    const INTERFACE = 'interface';
+    const NAMESPACE = 'namespace';
+    const NEW = 'new';
+    const OR = 'or';
+    const XOR = 'xor';
+    const TRY = 'try';
+    const USE = 'use';
+    const VAR = 'var';
+    const EXIT = 'exit';
+    const LIST = 'list';
+    const CLONE = 'clone';
+    const INCLUDE = 'include';
+    const INCLUDE_ONCE = 'include_once';
+    const THROW = 'throw';
+    const ARRAY = 'array';
+    const PRINT = 'print';
+    const ECHO = 'echo';
+    const REQUIRE = 'require';
+    const REQUIRE_ONCE = 'require_once';
+    const RETURN = 'return';
+    const ELSE = 'else';
+    const ELSEIF = 'elseif';
+    const DEFAULT = 'default';
+    const BREAK = 'break';
+    const CONTINUE = 'continue';
+    const SWITCH = 'switch';
+    const YIELD = 'yield';
+    const FUNCTION = 'function';
+    const IF = 'if';
+    const ENDSWITCH = 'endswitch';
+    const FINALLY = 'finally';
+    const FOR = 'for';
+    const FOREACH = 'foreach';
+    const DECLARE = 'declare';
+    const CASE = 'case';
+    const DO = 'do';
+    const WHILE = 'while';
+    const AS = 'as';
+    const CATCH = 'catch';
+    const DIE = 'die';
+    const SELF = 'self';
+    const PARENT = 'parent';
+}
+
+echo Obj::EMPTY, PHP_EOL;
+echo Obj::CALLABLE, PHP_EOL;
+echo Obj::TRAIT, PHP_EOL;
+echo Obj::EXTENDS, PHP_EOL;
+echo Obj::IMPLEMENTS, PHP_EOL;
+echo Obj::CONST, PHP_EOL;
+echo Obj::ENDDECLARE, PHP_EOL;
+echo Obj::ENDFOR, PHP_EOL;
+echo Obj::ENDFOREACH, PHP_EOL;
+echo Obj::ENDIF, PHP_EOL;
+echo Obj::ENDWHILE, PHP_EOL;
+echo Obj::AND, PHP_EOL;
+echo Obj::GLOBAL, PHP_EOL;
+echo Obj::GOTO, PHP_EOL;
+echo Obj::INSTANCEOF, PHP_EOL;
+echo Obj::INSTEADOF, PHP_EOL;
+echo Obj::INTERFACE, PHP_EOL;
+echo Obj::NAMESPACE, PHP_EOL;
+echo Obj::NEW, PHP_EOL;
+echo Obj::OR, PHP_EOL;
+echo Obj::XOR, PHP_EOL;
+echo Obj::TRY, PHP_EOL;
+echo Obj::USE, PHP_EOL;
+echo Obj::VAR, PHP_EOL;
+echo Obj::EXIT, PHP_EOL;
+echo Obj::LIST, PHP_EOL;
+echo Obj::CLONE, PHP_EOL;
+echo Obj::INCLUDE, PHP_EOL;
+echo Obj::INCLUDE_ONCE, PHP_EOL;
+echo Obj::THROW, PHP_EOL;
+echo Obj::ARRAY, PHP_EOL;
+echo Obj::PRINT, PHP_EOL;
+echo Obj::ECHO, PHP_EOL;
+echo Obj::REQUIRE, PHP_EOL;
+echo Obj::REQUIRE_ONCE, PHP_EOL;
+echo Obj::RETURN, PHP_EOL;
+echo Obj::ELSE, PHP_EOL;
+echo Obj::ELSEIF, PHP_EOL;
+echo Obj::DEFAULT, PHP_EOL;
+echo Obj::BREAK, PHP_EOL;
+echo Obj::CONTINUE, PHP_EOL;
+echo Obj::SWITCH, PHP_EOL;
+echo Obj::YIELD, PHP_EOL;
+echo Obj::FUNCTION, PHP_EOL;
+echo Obj::IF, PHP_EOL;
+echo Obj::ENDSWITCH, PHP_EOL;
+echo Obj::FINALLY, PHP_EOL;
+echo Obj::FOR, PHP_EOL;
+echo Obj::FOREACH, PHP_EOL;
+echo Obj::DECLARE, PHP_EOL;
+echo Obj::CASE, PHP_EOL;
+echo Obj::DO, PHP_EOL;
+echo Obj::WHILE, PHP_EOL;
+echo Obj::AS, PHP_EOL;
+echo Obj::CATCH, PHP_EOL;
+echo Obj::DIE, PHP_EOL;
+echo Obj::SELF, PHP_EOL;
+echo Obj::PARENT, PHP_EOL;
+
+echo "\nDone\n";
+
+--EXPECTF--
+empty
+callable
+trait
+extends
+implements
+const
+enddeclare
+endfor
+endforeach
+endif
+endwhile
+and
+global
+goto
+instanceof
+insteadof
+interface
+namespace
+new
+or
+xor
+try
+use
+var
+exit
+list
+clone
+include
+include_once
+throw
+array
+print
+echo
+require
+require_once
+return
+else
+elseif
+default
+break
+continue
+switch
+yield
+function
+if
+endswitch
+finally
+for
+foreach
+declare
+case
+do
+while
+as
+catch
+die
+self
+parent
+
+Done

Zend/tests/grammar/semi_reserved_006.phpt

@@ -0,0 +1,80 @@
+--TEST--
+Test semi-reserved method and constant names and trait conflict resolution
+--FILE--
+<?php
+
+trait TraitA
+{
+    public function catch(){ echo __METHOD__, PHP_EOL; }
+    private function list(){ echo __METHOD__, PHP_EOL; }
+}
+
+trait TraitB
+{
+    static $list = ['a' => ['b' => ['c']]];
+
+    public static function catch(){ echo __METHOD__, PHP_EOL; }
+    private static function throw(){ echo __METHOD__, PHP_EOL; }
+    private static function self(){ echo __METHOD__, PHP_EOL; }
+}
+
+trait TraitC
+{
+    public static function exit(){ echo __METHOD__, PHP_EOL; }
+    protected static function try(){ echo __METHOD__, PHP_EOL; }
+}
+
+class Foo
+{
+    use TraitA, TraitB {
+        TraitA
+            ::
+            catch insteadof namespace\TraitB;
+        TraitA::list as public foreach;
+        TraitB::throw as public;
+        TraitB::self as public;
+    }
+
+    use TraitC {
+        try as public attempt;
+        exit as die;
+        \TraitC::exit as bye;
+        namespace\TraitC::exit as byebye;
+        TraitC
+            ::
+            exit as farewell;
+    }
+}
+
+(new Foo)->catch();
+(new Foo)->foreach();
+Foo::throw();
+Foo::self();
+var_dump(Foo::$list['a']);
+Foo::attempt();
+Foo::die();
+Foo::bye();
+Foo::byebye();
+Foo::farewell();
+
+echo "\nDone\n";
+
+--EXPECTF--
+TraitA::catch
+TraitA::list
+TraitB::throw
+TraitB::self
+array(1) {
+  ["b"]=>
+  array(1) {
+    [0]=>
+    string(1) "c"
+  }
+}
+TraitC::try
+TraitC::exit
+TraitC::exit
+TraitC::exit
+TraitC::exit
+
+Done

Zend/tests/grammar/semi_reserved_007.phpt

@@ -0,0 +1,37 @@
+--TEST--
+Edge case: self::self, self::parent, parent::self semi reserved constants access
+--FILE--
+<?php
+
+class Foo {
+    const self = "self";
+    const parent = "parent";
+    public function __construct() {
+        echo "From ", __METHOD__, ":", PHP_EOL;
+        echo self::self, PHP_EOL;
+        echo self::parent, PHP_EOL;
+    }
+}
+
+class Bar extends Foo {
+    public function __construct() {
+        parent::__construct();
+        echo "From ", __METHOD__, ":", PHP_EOL;
+        echo parent::self, PHP_EOL;
+        echo parent::parent, PHP_EOL;
+    }
+}
+
+new Bar;
+
+echo "\nDone\n";
+
+--EXPECTF--
+From Foo::__construct:
+self
+parent
+From Bar::__construct:
+self
+parent
+
+Done

Zend/tests/grammar/semi_reserved_008.phpt

@@ -0,0 +1,68 @@
+--TEST--
+Testing with comments around semi-reserved names (not intended to be legible)
+--FILE--
+<?php
+
+trait TraitA
+{
+    public static function list(){ echo __METHOD__, PHP_EOL; }
+    public static function /* comment */ catch(){ echo __METHOD__, PHP_EOL; }
+    private static function // comment
+        throw(){ echo __METHOD__, PHP_EOL; }
+    private static function
+    # comment
+    self(){ echo __METHOD__, PHP_EOL; }
+}
+
+trait TraitB
+{
+    public static function exit(){ echo __METHOD__, PHP_EOL; }
+    protected static function try(){ echo __METHOD__, PHP_EOL; }
+}
+
+class Foo
+{
+    use TraitA {
+        TraitA::
+            //
+            /** doc comment */
+            #
+        catch /* comment */ 
+            // comment
+            # comment
+        insteadof TraitB;
+
+        TraitA::list as public /**/ foreach;
+    }
+
+    use TraitB {
+        try /*comment*/ as public attempt;
+        exit // comment
+            as/*comment*/die; // non qualified
+        \TraitB::exit as bye; // full qualified
+        namespace\TraitB::exit #
+        as byebye; // even more full qualified
+        TraitB
+            ::
+            /**  */
+            exit as farewell; // full qualified with weird spacing
+    }
+}
+
+Foo /**/
+#
+//
+/**  */
+::
+/**/
+#
+//
+/**  */
+attempt();
+
+echo PHP_EOL, "Done", PHP_EOL;
+
+--EXPECTF--
+TraitB::try
+
+Done

Zend/tests/grammar/semi_reserved_009.phpt

@@ -0,0 +1,25 @@
+--TEST--
+Edge case: T_STRING<as> as T_STRING<?>
+--FILE--
+<?php
+
+trait TraitA
+{
+    public static function as(){ echo __METHOD__, PHP_EOL; }
+}
+
+class Foo
+{
+    use TraitA  {
+        as as try;
+    }
+}
+
+Foo::try();
+
+echo PHP_EOL, "Done", PHP_EOL;
+
+--EXPECTF--
+TraitA::as
+
+Done

Zend/tests/grammar/semi_reserved_010.phpt

@@ -0,0 +1,31 @@
+--TEST--
+Edge case: T_STRING<insteadof> insteadof T_STRING<?>
+--FILE--
+<?php
+
+trait TraitA
+{
+    public static function insteadof(){ echo __METHOD__, PHP_EOL; }
+}
+
+trait TraitB
+{
+    public static function insteadof(){ echo __METHOD__, PHP_EOL; }
+}
+
+class Foo
+{
+    use TraitA , TraitB {
+        TraitB::insteadof
+            insteadof TraitA;
+    }
+}
+
+Foo::insteadof();
+
+echo PHP_EOL, "Done", PHP_EOL;
+
+--EXPECTF--
+TraitB::insteadof
+
+Done

Zend/zend_compile.c

@@ -30,7 +30,6 @@
 #include "zend_interfaces.h"
 #include "zend_virtual_cwd.h"
 #include "zend_multibyte.h"
-#include "zend_language_scanner.h"
 #include "zend_inheritance.h"
 
 #define SET_NODE(target, src) do { \
@@ -557,7 +556,10 @@ static int zend_add_const_name_literal(zend_op_array *op_array, zend_string *nam
 		op.constant = zend_add_literal(CG(active_op_array), &_c); \
 	} while (0)
 
-void zend_stop_lexing(void) {
+void zend_stop_lexing(void)
+{
+	if(LANG_SCNG(on_event)) LANG_SCNG(on_event)(ON_STOP, END, 0);
+
 	LANG_SCNG(yy_cursor) = LANG_SCNG(yy_limit);
 }
 

Zend/zend_globals.h

@@ -249,6 +249,12 @@ struct _zend_ini_scanner_globals {
 	int scanner_mode;
 };
 
+typedef enum {
+	ON_TOKEN,
+	ON_FEEDBACK,
+	ON_STOP
+} zend_php_scanner_event;
+
 struct _zend_php_scanner_globals {
 	zend_file_handle *yy_in;
 	zend_file_handle *yy_out;
@@ -278,6 +284,9 @@ struct _zend_php_scanner_globals {
 
 	/* initial string length after scanning to first variable */
 	int scanned_string_len;
+
+	/* hooks */
+	void (* on_event)(zend_php_scanner_event event, int token, int line);
 };
 
 #endif /* ZEND_GLOBALS_H */

Zend/zend_language_parser.y

@@ -35,6 +35,7 @@
 #include "zend_globals.h"
 #include "zend_API.h"
 #include "zend_constants.h"
+#include "zend_language_scanner.h"
 
 #define YYSIZE_T size_t
 #define yytnamerr zend_yytnamerr
@@ -243,7 +244,7 @@ static YYSIZE_T zend_yytnamerr(char*, const char*);
 %type <ast> absolute_trait_method_reference trait_method_reference property echo_expr
 %type <ast> new_expr anonymous_class class_name class_name_reference simple_variable
 %type <ast> internal_functions_in_yacc
-%type <ast> exit_expr scalar backticks_expr lexical_var function_call member_name
+%type <ast> exit_expr scalar backticks_expr lexical_var function_call member_name property_name
 %type <ast> variable_class_name dereferencable_scalar class_name_scalar constant dereferencable
 %type <ast> callable_expr callable_variable static_member new_variable
 %type <ast> assignment_list_element array_pair encaps_var encaps_var_offset isset_variables
@@ -252,10 +253,11 @@ static YYSIZE_T zend_yytnamerr(char*, const char*);
 %type <ast> echo_expr_list unset_variables catch_list parameter_list class_statement_list
 %type <ast> implements_list case_list if_stmt_without_else
 %type <ast> non_empty_parameter_list argument_list non_empty_argument_list property_list
-%type <ast> class_const_list name_list trait_adaptations method_body non_empty_for_exprs
+%type <ast> class_const_list class_const_decl name_list trait_adaptations method_body non_empty_for_exprs
 %type <ast> ctor_arguments alt_if_stmt_without_else trait_adaptation_list lexical_vars
 %type <ast> lexical_var_list encaps_list array_pair_list non_empty_array_pair_list
 %type <ast> assignment_list isset_variable type return_type
+%type <ast> identifier
 
 %type <num> returns_ref function is_reference is_variadic variable_modifiers
 %type <num> method_modifiers trait_modifiers non_empty_member_modifiers member_modifier
@@ -269,6 +271,26 @@ start:
 	top_statement_list	{ CG(ast) = $1; }
 ;
 
+semi_reserved:
+	  T_INCLUDE | T_INCLUDE_ONCE | T_EVAL | T_REQUIRE | T_REQUIRE_ONCE | T_LOGICAL_OR | T_LOGICAL_XOR | T_LOGICAL_AND
+	| T_INSTANCEOF | T_NEW | T_CLONE | T_EXIT | T_IF | T_ELSEIF | T_ELSE | T_ENDIF | T_ECHO | T_DO | T_WHILE | T_ENDWHILE
+	| T_FOR | T_ENDFOR | T_FOREACH | T_ENDFOREACH | T_DECLARE | T_ENDDECLARE | T_AS | T_TRY | T_CATCH | T_FINALLY
+	| T_THROW | T_USE | T_INSTEADOF | T_GLOBAL | T_VAR | T_UNSET | T_ISSET | T_EMPTY | T_CONTINUE | T_GOTO
+	| T_FUNCTION | T_CONST | T_RETURN | T_PRINT | T_YIELD | T_LIST | T_SWITCH | T_ENDSWITCH | T_CASE | T_DEFAULT | T_BREAK
+	| T_ARRAY | T_CALLABLE | T_EXTENDS | T_IMPLEMENTS | T_NAMESPACE | T_TRAIT | T_INTERFACE
+	// | T_STATIC | T_ABSTRACT | T_FINAL | T_PRIVATE | T_PROTECTED | T_PUBLIC
+	// | T_CLASS
+;
+
+identifier:
+		T_STRING { $$ = $1; }
+	| 	semi_reserved  {
+			zval zv;
+			zend_lex_tstring(&zv);
+			$$ = zend_ast_create_zval(&zv);
+		}
+;
+
 top_statement_list:
 		top_statement_list top_statement { $$ = zend_ast_list_add($1, $2); }
 	|	/* empty */ { $$ = zend_ast_create_list(0, ZEND_AST_STMT_LIST); }
@@ -673,7 +695,7 @@ class_statement:
 			{ $$ = $2; RESET_DOC_COMMENT(); }
 	|	T_USE name_list trait_adaptations
 			{ $$ = zend_ast_create(ZEND_AST_USE_TRAIT, $2, $3); }
-	|	method_modifiers function returns_ref T_STRING '(' parameter_list ')'
+	|	method_modifiers function returns_ref identifier '(' parameter_list ')'
 		return_type backup_doc_comment method_body
 			{ $$ = zend_ast_create_decl(ZEND_AST_METHOD, $3 | $1, $2, $9,
 				  zend_ast_get_str($4), $6, NULL, $10, $8); }
@@ -708,20 +730,20 @@ trait_precedence:
 ;
 
 trait_alias:
-		trait_method_reference T_AS trait_modifiers T_STRING
+		trait_method_reference T_AS trait_modifiers identifier
 			{ $$ = zend_ast_create_ex(ZEND_AST_TRAIT_ALIAS, $3, $1, $4); }
 	|	trait_method_reference T_AS member_modifier
 			{ $$ = zend_ast_create_ex(ZEND_AST_TRAIT_ALIAS, $3, $1, NULL); }
 ;
 
 trait_method_reference:
-		T_STRING
+		identifier
 			{ $$ = zend_ast_create(ZEND_AST_METHOD_REFERENCE, NULL, $1); }
 	|	absolute_trait_method_reference { $$ = $1; }
 ;
 
 absolute_trait_method_reference:
-	name T_PAAMAYIM_NEKUDOTAYIM T_STRING
+	name T_PAAMAYIM_NEKUDOTAYIM identifier
 		{ $$ = zend_ast_create(ZEND_AST_METHOD_REFERENCE, $1, $3); }
 ;
 
@@ -773,8 +795,12 @@ property:
 ;
 
 class_const_list:
-		class_const_list ',' const_decl { $$ = zend_ast_list_add($1, $3); }
+		class_const_list ',' class_const_decl { $$ = zend_ast_list_add($1, $3); }
-	|	const_decl { $$ = zend_ast_create_list(1, ZEND_AST_CLASS_CONST_DECL, $1); }
+	|	class_const_decl { $$ = zend_ast_create_list(1, ZEND_AST_CLASS_CONST_DECL, $1); }
+;
+
+class_const_decl:
+	identifier '=' expr { $$ = zend_ast_create(ZEND_AST_CONST_ELEM, $1, $3); }
 ;
 
 const_decl:
@@ -1034,9 +1060,9 @@ scalar:
 
 constant:
 		name { $$ = zend_ast_create(ZEND_AST_CONST, $1); }
-	|	class_name T_PAAMAYIM_NEKUDOTAYIM T_STRING
+	|	class_name T_PAAMAYIM_NEKUDOTAYIM identifier
 			{ $$ = zend_ast_create(ZEND_AST_CLASS_CONST, $1, $3); }
-	|	variable_class_name T_PAAMAYIM_NEKUDOTAYIM T_STRING
+	|	variable_class_name T_PAAMAYIM_NEKUDOTAYIM identifier
 			{ $$ = zend_ast_create(ZEND_AST_CLASS_CONST, $1, $3); }
 ;
 
@@ -1080,7 +1106,7 @@ callable_variable:
 			{ $$ = zend_ast_create(ZEND_AST_DIM, $1, $3); }
 	|	dereferencable '{' expr '}'
 			{ $$ = zend_ast_create(ZEND_AST_DIM, $1, $3); }
-	|	dereferencable T_OBJECT_OPERATOR member_name argument_list
+	|	dereferencable T_OBJECT_OPERATOR property_name argument_list
 			{ $$ = zend_ast_create(ZEND_AST_METHOD_CALL, $1, $3, $4); }
 	|	function_call { $$ = $1; }
 ;
@@ -1090,7 +1116,7 @@ variable:
 			{ $$ = $1; }
 	|	static_member
 			{ $$ = $1; }
-	|	dereferencable T_OBJECT_OPERATOR member_name
+	|	dereferencable T_OBJECT_OPERATOR property_name
 			{ $$ = zend_ast_create(ZEND_AST_PROP, $1, $3); }
 ;
 
@@ -1114,7 +1140,7 @@ new_variable:
 			{ $$ = zend_ast_create(ZEND_AST_DIM, $1, $3); }
 	|	new_variable '{' expr '}'
 			{ $$ = zend_ast_create(ZEND_AST_DIM, $1, $3); }
-	|	new_variable T_OBJECT_OPERATOR member_name
+	|	new_variable T_OBJECT_OPERATOR property_name
 			{ $$ = zend_ast_create(ZEND_AST_PROP, $1, $3); }
 	|	class_name T_PAAMAYIM_NEKUDOTAYIM simple_variable
 			{ $$ = zend_ast_create(ZEND_AST_STATIC_PROP, $1, $3); }
@@ -1123,7 +1149,13 @@ new_variable:
 ;
 
 member_name:
-		T_STRING		{ $$ = $1; }
+		identifier { $$ = $1; }
+	|	'{' expr '}'	{ $$ = $2; }
+	|	simple_variable	{ $$ = zend_ast_create(ZEND_AST_VAR, $1); }
+;
+
+property_name:
+		T_STRING { $$ = $1; }
 	|	'{' expr '}'	{ $$ = $2; }
 	|	simple_variable	{ $$ = zend_ast_create(ZEND_AST_VAR, $1); }
 ;

Zend/zend_language_scanner.h

@@ -50,6 +50,9 @@ typedef struct _zend_lex_state {
 	zend_encoding_filter output_filter;
 	const zend_encoding *script_encoding;
 
+	/* hooks */
+	void (* on_event)(zend_php_scanner_event event, int token, int line);
+
 	zend_ast *ast;
 	zend_arena *ast_arena;
 } zend_lex_state;
@@ -66,6 +69,7 @@ ZEND_API void zend_restore_lexical_state(zend_lex_state *lex_state);
 ZEND_API int zend_prepare_string_for_scanning(zval *str, char *filename);
 ZEND_API void zend_multibyte_yyinput_again(zend_encoding_filter old_input_filter, const zend_encoding *old_encoding);
 ZEND_API int zend_multibyte_set_filter(const zend_encoding *onetime_encoding);
+ZEND_API void zend_lex_tstring(zval *zv);
 
 END_EXTERN_C()
 

ext/tokenizer/tests/bug67395.phpt

@@ -1,5 +1,7 @@
 --TEST--
 Bug 67395: token_name() does not return name for T_POW and T_POW_EQUAL token
+--SKIPIF--
+<?php if (!extension_loaded("tokenizer")) print "skip"; ?>
 --FILE--
 <?php
 

ext/tokenizer/tests/token_get_all_TOKEN_PARSE_000.phpt

@@ -0,0 +1,19 @@
+--TEST--
+Parse errors during token_get_all() with TOKEN_PARSE flag
+--SKIPIF--
+<?php if (!extension_loaded("tokenizer")) print "skip"; ?>
+--FILE--
+<?php
+
+try {
+    token_get_all('<?php invalid code;', TOKEN_PARSE);
+} catch (ParseException $e) {
+    echo $e->getMessage(), PHP_EOL;
+}
+
+echo "Done";
+
+?>
+--EXPECT--
+syntax error, unexpected 'code' (T_STRING)
+Done

ext/tokenizer/tests/token_get_all_TOKEN_PARSE_001.phpt

@@ -0,0 +1,81 @@
+--TEST--
+Semi reserved words support: member access
+--SKIPIF--
+<?php if (!extension_loaded("tokenizer")) print "skip"; ?>
+--FILE--
+<?php
+$tokens = token_get_all('<?php
+X::continue;
+X::$continue;
+$x->$continue;
+X::continue();
+$x->continue();
+X::class;
+
+class X {
+    const CONTINUE = 1;
+    public $x = self::CONTINUE + 1;
+}
+', TOKEN_PARSE);
+
+array_walk($tokens, function($tk) {
+  if(is_array($tk)) {
+    if(($t = token_name($tk[0])) == 'T_WHITESPACE') return;
+    echo "L{$tk[2]}: ".$t." {$tk[1]}", PHP_EOL;
+  }
+  else echo $tk, PHP_EOL;
+});
+
+echo "Done";
+
+?>
+--EXPECTF--
+L1: T_OPEN_TAG <?php
+
+L2: T_STRING X
+L2: T_DOUBLE_COLON ::
+L2: T_STRING continue
+;
+L3: T_STRING X
+L3: T_DOUBLE_COLON ::
+L3: T_VARIABLE $continue
+;
+L4: T_VARIABLE $x
+L4: T_OBJECT_OPERATOR ->
+L4: T_VARIABLE $continue
+;
+L5: T_STRING X
+L5: T_DOUBLE_COLON ::
+L5: T_STRING continue
+(
+)
+;
+L6: T_VARIABLE $x
+L6: T_OBJECT_OPERATOR ->
+L6: T_STRING continue
+(
+)
+;
+L7: T_STRING X
+L7: T_DOUBLE_COLON ::
+L7: T_CLASS class
+;
+L9: T_CLASS class
+L9: T_STRING X
+{
+L10: T_CONST const
+L10: T_STRING CONTINUE
+=
+L10: T_LNUMBER 1
+;
+L11: T_PUBLIC public
+L11: T_VARIABLE $x
+=
+L11: T_STRING self
+L11: T_DOUBLE_COLON ::
+L11: T_STRING CONTINUE
++
+L11: T_LNUMBER 1
+;
+}
+Done

ext/tokenizer/tests/token_get_all_TOKEN_PARSE_002.phpt

@@ -0,0 +1,68 @@
+--TEST--
+Semi reserved words support: class const
+--SKIPIF--
+<?php if (!extension_loaded("tokenizer")) print "skip"; ?>
+--FILE--
+<?php
+$tokens = token_get_all('<?php
+  class SomeClass {
+      const CONST = 1;
+      const CONTINUE = (self::CONST + 1);
+      const ARRAY = [1, self::CONTINUE => [3, 4], 5];
+  }
+', TOKEN_PARSE);
+
+array_walk($tokens, function($tk) {
+  if(is_array($tk)) {
+    if(($t = token_name($tk[0])) == 'T_WHITESPACE') return;
+    echo "L{$tk[2]}: ".$t." {$tk[1]}", PHP_EOL;
+  }
+  else echo $tk, PHP_EOL;
+});
+
+echo "Done";
+
+?>
+--EXPECTF--
+L1: T_OPEN_TAG <?php
+
+L2: T_CLASS class
+L2: T_STRING SomeClass
+{
+L3: T_CONST const
+L3: T_STRING CONST
+=
+L3: T_LNUMBER 1
+;
+L4: T_CONST const
+L4: T_STRING CONTINUE
+=
+(
+L4: T_STRING self
+L4: T_DOUBLE_COLON ::
+L4: T_STRING CONST
++
+L4: T_LNUMBER 1
+)
+;
+L5: T_CONST const
+L5: T_STRING ARRAY
+=
+[
+L5: T_LNUMBER 1
+,
+L5: T_STRING self
+L5: T_DOUBLE_COLON ::
+L5: T_STRING CONTINUE
+L5: T_DOUBLE_ARROW =>
+[
+L5: T_LNUMBER 3
+,
+L5: T_LNUMBER 4
+]
+,
+L5: T_LNUMBER 5
+]
+;
+}
+Done

ext/tokenizer/tests/token_get_all_error.phpt

@@ -19,7 +19,7 @@ var_dump( token_get_all());
 echo "-- Testing token_get_all() function with more than expected no. of arguments --\n";
 $source = '<?php ?>';
 $extra_arg = 10;
-var_dump( token_get_all($source, $extra_arg));
+var_dump( token_get_all($source, true, $extra_arg));
 
 echo "Done"
 ?>
@@ -28,10 +28,10 @@ echo "Done"
 
 -- Testing token_get_all() function with zero arguments --
 
-Warning: token_get_all() expects exactly 1 parameter, 0 given in %s on line %d
+Warning: token_get_all() expects at least 1 parameter, 0 given in %s on line 11
 NULL
 -- Testing token_get_all() function with more than expected no. of arguments --
 
-Warning: token_get_all() expects exactly 1 parameter, 2 given in %s on line %d
+Warning: token_get_all() expects at most 2 parameters, 3 given in %s on line 17
 NULL
-Done
+Done

ext/tokenizer/tokenizer.c

@@ -37,6 +37,12 @@
 #define zendcursor LANG_SCNG(yy_cursor)
 #define zendlimit  LANG_SCNG(yy_limit)
 
+#define TOKEN_PARSE 				1
+
+void tokenizer_token_get_all_register_constants(INIT_FUNC_ARGS) {
+	REGISTER_LONG_CONSTANT("TOKEN_PARSE", TOKEN_PARSE, CONST_CS|CONST_PERSISTENT);
+}
+
 /* {{{ arginfo */
 ZEND_BEGIN_ARG_INFO_EX(arginfo_token_get_all, 0, 0, 1)
 	ZEND_ARG_INFO(0, source)
@@ -83,6 +89,7 @@ ZEND_GET_MODULE(tokenizer)
 PHP_MINIT_FUNCTION(tokenizer)
 {
 	tokenizer_register_constants(INIT_FUNC_ARGS_PASSTHRU);
+	tokenizer_token_get_all_register_constants(INIT_FUNC_ARGS_PASSTHRU);
 	return SUCCESS;
 }
 /* }}} */
@@ -97,19 +104,33 @@ PHP_MINFO_FUNCTION(tokenizer)
 }
 /* }}} */
 
-static void tokenize(zval *return_value)
+static zend_bool tokenize(zval *return_value, zend_string *source)
 {
+	zval source_zval;
+	zend_lex_state original_lex_state;
 	zval token;
 	zval keyword;
 	int token_type;
 	zend_bool destroy;
 	int token_line = 1;
-	int need_tokens = -1; // for __halt_compiler lexing. -1 = disabled
+	int need_tokens = -1; /* for __halt_compiler lexing. -1 = disabled */
 
+	ZVAL_STR_COPY(&source_zval, source);
+	zend_save_lexical_state(&original_lex_state);
+
+	if (zend_prepare_string_for_scanning(&source_zval, "") == FAILURE) {
+		zend_restore_lexical_state(&original_lex_state);
+		return 0;
+	}
+
+	LANG_SCNG(yy_state) = yycINITIAL;
 	array_init(return_value);
 
 	ZVAL_NULL(&token);
 	while ((token_type = lex_scan(&token))) {
+
+		if(token_type == T_ERROR) break;
+
 		destroy = 1;
 		switch (token_type) {
 			case T_CLOSE_TAG:
@@ -123,8 +144,6 @@ static void tokenize(zval *return_value)
 			case T_DOC_COMMENT:
 				destroy = 0;
 				break;
-			case T_ERROR:
-				return;
 		}
 
 		if (token_type >= 256) {
@@ -147,13 +166,13 @@ static void tokenize(zval *return_value)
 		}
 		ZVAL_NULL(&token);
 
-		// after T_HALT_COMPILER collect the next three non-dropped tokens
+		/* after T_HALT_COMPILER collect the next three non-dropped tokens */
 		if (need_tokens != -1) {
 			if (token_type != T_WHITESPACE && token_type != T_OPEN_TAG
-			    && token_type != T_COMMENT && token_type != T_DOC_COMMENT
+				&& token_type != T_COMMENT && token_type != T_DOC_COMMENT
-			    && --need_tokens == 0
+				&& --need_tokens == 0
 			) {
-				// fetch the rest into a T_INLINE_HTML
+				/* fetch the rest into a T_INLINE_HTML */
 				if (zendcursor != zendlimit) {
 					array_init(&keyword);
 					add_next_index_long(&keyword, T_INLINE_HTML);
@@ -169,34 +188,113 @@ static void tokenize(zval *return_value)
 
 		token_line = CG(zend_lineno);
 	}
+
+	zval_dtor(&source_zval);
+	zend_restore_lexical_state(&original_lex_state);
+
+	return 1;
 }
 
+zval token_stream;
+
+void on_event(zend_php_scanner_event event, int token, int line)
+{
+	zval keyword;
+	HashTable *tokens_ht;
+	zval *token_zv;
+
+	switch(event) {
+		case ON_TOKEN:
+			if (token == T_ERROR || token == END) break;
+			if (token >= 256) {
+				array_init(&keyword);
+				add_next_index_long(&keyword, token);
+				add_next_index_stringl(&keyword, (char *)LANG_SCNG(yy_text), LANG_SCNG(yy_leng));
+				add_next_index_long(&keyword, line);
+				add_next_index_zval(&token_stream, &keyword);
+			} else {
+				add_next_index_stringl(&token_stream, (char *)LANG_SCNG(yy_text), LANG_SCNG(yy_leng));
+			}
+			break;
+		case ON_FEEDBACK:
+			tokens_ht = Z_ARRVAL(token_stream);
+			token_zv = zend_hash_index_find(tokens_ht, zend_hash_num_elements(tokens_ht) - 1);
+			if (token_zv && Z_TYPE_P(token_zv) == IS_ARRAY) {
+				ZVAL_LONG(zend_hash_index_find(Z_ARRVAL_P(token_zv), 0), token);
+			}
+			break;
+		case ON_STOP:
+			if (LANG_SCNG(yy_cursor) != LANG_SCNG(yy_limit)) {
+				array_init(&keyword);
+				add_next_index_long(&keyword, T_INLINE_HTML);
+				add_next_index_stringl(&keyword,
+					(char *)LANG_SCNG(yy_cursor), LANG_SCNG(yy_limit) - LANG_SCNG(yy_cursor));
+				add_next_index_long(&keyword, CG(zend_lineno));
+				add_next_index_zval(&token_stream, &keyword);
+			}
+			break;
+	}
+}
+
+static zend_bool tokenize_parse(zval *return_value, zend_string *source)
+{
+	zval source_zval;
+	zend_lex_state original_lex_state;
+	zend_bool original_in_compilation;
+	zend_bool success;
+
+	ZVAL_STR_COPY(&source_zval, source);
+
+	original_in_compilation = CG(in_compilation);
+	CG(in_compilation) = 1;
+	zend_save_lexical_state(&original_lex_state);
+
+	if ((success = (zend_prepare_string_for_scanning(&source_zval, "") == SUCCESS))) {
+		CG(ast) = NULL;
+		CG(ast_arena) = zend_arena_create(1024 * 32);
+		LANG_SCNG(yy_state) = yycINITIAL;
+		LANG_SCNG(on_event) = on_event;
+
+		array_init(&token_stream);
+		if((success = (zendparse() == SUCCESS))) {
+			ZVAL_ZVAL(return_value, &token_stream, 1, 0);
+		}
+		zval_dtor(&token_stream);
+
+		zend_ast_destroy(CG(ast));
+		zend_arena_destroy(CG(ast_arena));
+	}
+
+	/* restore compiler and scanner global states */
+	zend_restore_lexical_state(&original_lex_state);
+	CG(in_compilation) = original_in_compilation;
+
+	zval_dtor(&source_zval);
+
+	return success;
+}
+
+/* }}} */
+
 /* {{{ proto array token_get_all(string source)
  */
 PHP_FUNCTION(token_get_all)
 {
 	zend_string *source;
-	zval source_zval;
+	zend_long flags = 0;
-	zend_lex_state original_lex_state;
+	zend_bool success;
 
-	if (zend_parse_parameters(ZEND_NUM_ARGS(), "S", &source) == FAILURE) {
+	if (zend_parse_parameters(ZEND_NUM_ARGS(), "S|l", &source, &flags) == FAILURE) {
 		return;
 	}
 
-	ZVAL_STR_COPY(&source_zval, source);
+	if (flags & TOKEN_PARSE) {
-	zend_save_lexical_state(&original_lex_state);
+		success = tokenize_parse(return_value, source);
-
+	} else {
-	if (zend_prepare_string_for_scanning(&source_zval, "") == FAILURE) {
+		success = tokenize(return_value, source);
-		zend_restore_lexical_state(&original_lex_state);
-		RETURN_FALSE;
 	}
 
-	LANG_SCNG(yy_state) = yycINITIAL;
+	if (!success) RETURN_FALSE;
-
-	tokenize(return_value);
-
-	zend_restore_lexical_state(&original_lex_state);
-	zval_dtor(&source_zval);
 }
 /* }}} */
 

tests/basic/bug51709_1.phpt

@@ -1,16 +0,0 @@
---TEST--
-Bug #51709 (Can't use keywords as method names)
---FILE--
-<?php
-
-class foo {
-        static function for() {
-                echo "1";
-        }
-}
-
-?>
-===DONE===
-<?php exit(0); ?>
---EXPECTF--
-Parse error: syntax error, unexpected %s, expecting %s in %sbug51709_1.php on line %d

tests/basic/bug51709_2.phpt

@@ -1,16 +0,0 @@
---TEST--
-Bug #51709 (Can't use keywords as method names)
---FILE--
-<?php
-
-class foo {
-        static function goto() {
-                echo "1";
-        }
-}
-
-?>
-===DONE===
-<?php exit(0); ?>
---EXPECTF--
-Parse error: syntax error, unexpected %s, expecting %s in %sbug51709_2.php on line %d