[Templates-cvs] cvs commit: TT3/tt2/parser Grammar.pm.skel Parser.yp README yc
cvs@template-toolkit.org
cvs@template-toolkit.org
Thu, 25 Mar 2004 16:00:36 +0000
cvs 04/03/25 16:00:36
Added: tt2/parser Grammar.pm.skel Parser.yp README yc
Log:
* added the tt2 directory containing the old parser source and anything
else we might want to drop in here.
Revision Changes Path
1.1 TT3/tt2/parser/Grammar.pm.skel
Index: Grammar.pm.skel
===================================================================
#============================================================= -*-Perl-*-
#
# Template::TT2::Grammar
#
# DESCRIPTION
# Grammar file for the Template Toolkit v2 language containing token
# definitions and parser state/rules tables generated by Parse::Yapp.
#
# AUTHOR
# Andy Wardley <abw@wardley.org>
#
# COPYRIGHT
# Copyright (C) 1996-2004 Andy Wardley. All Rights Reserved.
# Copyright (C) 1998-2002 Canon Research Centre Europe Ltd.
# Copyright (C) 2004 Fotango Ltd.
#
# This module is free software; you can redistribute it and/or
# modify it under the same terms as Perl itself.
#
# REVISION
# $Id: Grammar.pm.skel,v 1.1 2004/03/25 16:00:36 abw Exp $
#
# IMPORTANT NOTE
# This module is constructed from the parser/Grammar.pm.skel file by
# running the parser/yc script. You only need to do this if # you
# have modified the grammar in the parser/Parser.yp file and need #
# to-recompile it. See the README in the 'parser' directory for
# more information.
#
#========================================================================
package Template::TT2::Grammar;
require 5.004;
use strict;
use vars qw( $VERSION );
$VERSION = sprintf("%d.%02d", q$Revision: 1.1 $ =~ /(\d+)\.(\d+)/);
my (@RESERVED, %CMPOP, $LEXTABLE, $RULES, $STATES);
my ($factory, $rawstart);
#========================================================================
# Reserved words, comparison and binary operators
#========================================================================
@RESERVED = qw(
GET CALL SET DEFAULT INSERT INCLUDE PROCESS WRAPPER BLOCK END
USE PLUGIN FILTER MACRO PERL RAWPERL TO STEP AND OR NOT DIV MOD
IF UNLESS ELSE ELSIF FOR NEXT WHILE SWITCH CASE META IN
TRY THROW CATCH FINAL LAST RETURN STOP CLEAR VIEW DEBUG
);
# for historical reasons, != and == are converted to ne and eq to perform
# stringwise comparison (mainly because it doesn't generate "non-numerical
# comparison" warnings which != and == can) but the others (e.g. < > <= >=)
# are not converted to their stringwise equivalents. I added 'gt' et al,
# briefly for v2.04d and then took them out again in 2.04e.
%CMPOP = qw(
!= ne
== eq
< <
> >
>= >=
<= <=
);
#========================================================================
# Lexer Token Table
#========================================================================
# lookup table used by lexer is initialised with special-cases
$LEXTABLE = {
'FOREACH' => 'FOR',
'BREAK' => 'LAST',
'&&' => 'AND',
'||' => 'OR',
'!' => 'NOT',
'|' => 'FILTER',
'.' => 'DOT',
'_' => 'CAT',
'..' => 'TO',
'=' => 'ASSIGN',
'=>' => 'ASSIGN',
',' => 'COMMA',
'\\' => 'REF',
'and' => 'AND', # explicitly specified so that qw( and or
'or' => 'OR', # not ) can always be used in lower case,
'not' => 'NOT', # regardless of ANYCASE flag
'mod' => 'MOD',
'div' => 'DIV',
};
# localise the temporary variables needed to complete lexer table
{
my @tokens = qw< ( ) [ ] { } ${ $ + / ; : ? >;
my @cmpop = keys %CMPOP;
my @binop = qw( - * % ); # '+' and '/' above, in @tokens
# fill lexer table, slice by slice, with reserved words and operators
@$LEXTABLE{ @RESERVED, @cmpop, @binop, @tokens }
= ( @RESERVED, ('CMPOP') x @cmpop, ('BINOP') x @binop, @tokens );
}
#========================================================================
# CLASS METHODS
#========================================================================
sub new {
my $class = shift;
bless {
LEXTABLE => $LEXTABLE,
STATES => $STATES,
RULES => $RULES,
}, $class;
}
sub install_factory {
my ($self, $new_factory) = @_;
$factory = $new_factory;
}
#========================================================================
# States
#========================================================================
$STATES = <<$states>>;
#========================================================================
# Rules
#========================================================================
$RULES = <<$rules>>;
1;
__END__
=head1 NAME
Template::TT2::Grammar - state and rule tables for TT2 grammar
=head1 SYNOPSIS
# no user-serviceable parts inside
=head1 DESCRIPTION
The Template::TT2::Grammar module defines the state and rule tables
for version 2 of the Template Toolkit template language (TT2). It is
generated automatically from the files in the tt2/parser directory.
You shouldn't need to worry too much about this module. Move along
now, there's nothing to see.
=head1 AUTHOR
Andy Wardley E<lt>abw@wardley.orgE<gt>
=head1 VERSION
$Revision: 1.1 $
=head1 COPYRIGHT
Copyright (C) 1996-2004 Andy Wardley. All Rights Reserved.
Copyright (C) 1998-2002 Canon Research Centre Europe Ltd.
Copyright (C) 2004 Fotango Ltd.
This module is free software; you can redistribute it and/or
modify it under the same terms as Perl itself.
=head1 SEE ALSO
See L<Template::TT2::Parser> for further information on how the grammar
is used.
=cut
# Local Variables:
# mode: perl
# perl-indent-level: 4
# indent-tabs-mode: nil
# End:
#
# vim: expandtab shiftwidth=4:
1.1 TT3/tt2/parser/Parser.yp
Index: Parser.yp
===================================================================
#============================================================= -*-Perl-*-
#
# Parser.yp
#
# DESCRIPTION
# Definition of the parser grammar for the Template Toolkit language.
#
# AUTHOR
# Andy Wardley <abw@wardley.org>
#
# HISTORY
# January 2000 (or thereabouts)
# Totally re-written for version 2, based on Doug Steinwand's
# implementation which compiles templates to Perl code. The generated
# code is _considerably_ faster, more portable and easier to process.
#
# February 2004
# Minor modifications to work it into the TT3 framework.
#
# WARNINGS
# Expect 1 reduce/reduce conflict. This can safely be ignored.
# Now also expect 1 shift/reduce conflict, created by adding a rule
# to 'args' to allow assignments of the form 'foo.bar = baz'. It
# should be possible to fix the problem by rewriting some rules, but
# I'm loathed to hack it up too much right now. Maybe later.
#
# COPYRIGHT
# Copyright (C) 1996-2004 Andy Wardley. All Rights Reserved.
# Copyright (C) 1998-2002 Canon Research Centre Europe Ltd.
# Copyright (C) 2004 Fotango Ltd.
#
# This module is free software; you can redistribute it and/or
# modify it under the same terms as Perl itself.
#
# REVISION
# $Id: Parser.yp,v 1.1 2004/03/25 16:00:36 abw Exp $
#
#========================================================================
%right ASSIGN
%right '?' ':'
%left COMMA
%left AND OR
%left NOT
%left CAT
%left DOT
%left CMPOP
%left BINOP
%left '+'
%left '/'
%left DIV
%left MOD
%left TO
%%
#--------------------------------------------------------------------------
# START AND TOP-LEVEL RULES
#--------------------------------------------------------------------------
template:
block { $factory->template($_[1]) }
;
block:
chunks { $factory->block($_[1]) }
| /* NULL */ { $factory->block() }
;
chunks:
chunks chunk { push(@{$_[1]}, $_[2]) if defined $_[2]; $_[1] }
| chunk { defined $_[1] ? [ $_[1] ] : [ ] }
;
chunk: TEXT { $factory->textblock($_[1]) }
| statement ';' { return '' unless $_[1];
$_[0]->location() . $_[1];
}
;
statement:
directive
| defblock
| anonblock
| capture
| macro
| use
| view
| rawperl
| expr { $factory->get($_[1]) }
| META metadata { $_[0]->add_metadata($_[2]); }
| /* empty statement */
;
directive:
setlist { $factory->set($_[1]) }
| atomdir
| condition
| switch
| loop
| try
| perl
;
#--------------------------------------------------------------------------
# DIRECTIVE RULES
#--------------------------------------------------------------------------
atomexpr:
expr { $factory->get($_[1]) }
| atomdir
;
atomdir:
GET expr { $factory->get($_[2]) }
| CALL expr { $factory->call($_[2]) }
| SET setlist { $factory->set($_[2]) }
| DEFAULT setlist { $factory->default($_[2]) }
| INSERT nameargs { $factory->insert($_[2]) }
| INCLUDE nameargs { $factory->include($_[2]) }
| PROCESS nameargs { $factory->process($_[2]) }
| THROW nameargs { $factory->throw($_[2]) }
| RETURN { $factory->return() }
| STOP { $factory->stop() }
| CLEAR { "\$output = '';"; }
| LAST { $_[0]->{ INFOR } || $_[0]->{ INWHILE }
? 'last LOOP;' : 'last;' }
| NEXT { $_[0]->{ INFOR }
? $factory->next()
: ($_[0]->{ INWHILE } ? 'next LOOP;' : 'next;') }
| DEBUG nameargs { if ($_[2]->[0]->[0] =~ /^'(on|off)'$/) {
$_[0]->{ DEBUG_DIRS } = ($1 eq 'on');
$factory->generate_debug($_[2]);
}
else {
$_[0]->{ DEBUG_DIRS }
? $factory->generate_debug($_[2]) : '';
}
}
| wrapper
| filter
;
condition:
IF expr ';' block else END { $factory->if(@_[2, 4, 5]) }
| atomexpr IF expr { $factory->if(@_[3, 1]) }
| UNLESS expr ';' block else END { $factory->if("!($_[2])", @_[4, 5]) }
| atomexpr UNLESS expr { $factory->if("!($_[3])", $_[1]) }
;
else:
ELSIF expr ';' block else { unshift(@{$_[5]}, [ @_[2, 4] ]); $_[5]; }
| ELSE ';' block { [ $_[3] ] }
| /* NULL */ { [ undef ] }
;
switch:
SWITCH expr ';' block case END { $factory->switch(@_[2, 5]) }
;
case:
CASE term ';' block case { unshift(@{$_[5]}, [ @_[2, 4] ]); $_[5]; }
| CASE DEFAULT ';' block { [ $_[4] ] }
| CASE ';' block { [ $_[3] ] }
| /* NULL */ { [ undef ] }
;
loop:
FOR loopvar ';' { $_[0]->{ INFOR }++ }
block END { $_[0]->{ INFOR }--;
$factory->foreach(@{$_[2]}, $_[5]) }
| atomexpr FOR loopvar { $factory->foreach(@{$_[3]}, $_[1]) }
| WHILE expr ';' { $_[0]->{ INWHILE }++ }
block END { $_[0]->{ INWHILE }--;
$factory->while(@_[2, 5])
}
| atomexpr WHILE expr { $factory->while(@_[3, 1]) }
;
loopvar:
IDENT ASSIGN term args { [ @_[1, 3, 4] ] }
| IDENT IN term args { [ @_[1, 3, 4] ] }
| term args { [ 0, @_[1, 2] ] }
;
wrapper:
WRAPPER nameargs ';' block END { $factory->wrapper(@_[2, 4]) }
| atomexpr WRAPPER nameargs { $factory->wrapper(@_[3, 1]) }
;
try:
TRY ';' block final END { $factory->try(@_[3, 4]) }
;
final:
CATCH filename ';' block final { unshift(@{$_[5]}, [ @_[2,4] ]); $_[5]; }
| CATCH DEFAULT ';' block final { unshift(@{$_[5]}, [ undef, $_[4] ]); $_[5]; }
| CATCH ';' block final { unshift(@{$_[4]}, [ undef, $_[3] ]); $_[4]; }
| FINAL ';' block { [ $_[3] ] }
| /* NULL */ { [ 0 ] } # no final
;
use:
USE lnameargs { $factory->use($_[2]) }
;
view:
VIEW nameargs ';' { $_[0]->push_defblock(); }
block END { $factory->view(@_[2,5], $_[0]->pop_defblock) }
;
perl:
PERL ';' { ${$_[0]->{ INPERL }}++; }
block END { ${$_[0]->{ INPERL }}--;
$_[0]->{ EVAL_PERL }
? $factory->perl($_[4])
: $factory->no_perl();
}
;
rawperl:
RAWPERL { ${$_[0]->{ INPERL }}++;
$rawstart = ${$_[0]->{'LINE'}}; }
';' TEXT END { ${$_[0]->{ INPERL }}--;
$_[0]->{ EVAL_PERL }
? $factory->rawperl($_[4], $rawstart)
: $factory->no_perl();
}
;
filter:
FILTER lnameargs ';' block END { $factory->filter(@_[2,4]) }
| atomexpr FILTER lnameargs { $factory->filter(@_[3,1]) }
;
defblock:
defblockname blockargs ';' template END
{ my $name = join('/', @{ $_[0]->{ DEFBLOCKS } });
pop(@{ $_[0]->{ DEFBLOCKS } });
$_[0]->define_block($name, $_[4]);
undef
}
;
defblockname:
BLOCK blockname { push(@{ $_[0]->{ DEFBLOCKS } }, $_[2]); $_[2]; }
;
blockname:
filename
| LITERAL { $_[1] =~ s/^'(.*)'$/$1/; $_[1] }
;
blockargs:
metadata
| /* NULL */
;
anonblock:
BLOCK blockargs ';' block END
{ $factory->anon_block($_[4]) }
;
capture:
ident ASSIGN mdir { $factory->capture(@_[1, 3]) }
;
macro:
MACRO IDENT '(' margs ')' mdir { $factory->macro(@_[2, 6, 4]) }
| MACRO IDENT mdir { $factory->macro(@_[2, 3]) }
;
mdir:
directive
| BLOCK ';' block END { $_[3] }
;
margs:
margs IDENT { push(@{$_[1]}, $_[2]); $_[1] }
| margs COMMA { $_[1] }
| IDENT { [ $_[1] ] }
;
metadata:
metadata meta { push(@{$_[1]}, @{$_[2]}); $_[1] }
| metadata COMMA
| meta
;
meta:
IDENT ASSIGN LITERAL { for ($_[3]) { s/^'//; s/'$//; s/\\'/'/g };
[ @_[1,3] ]
}
| IDENT ASSIGN '"' TEXT '"' { [ @_[1,4] ] }
| IDENT ASSIGN NUMBER { [ @_[1,3] ] }
;
#--------------------------------------------------------------------------
# FUNDAMENTAL ELEMENT RULES
#--------------------------------------------------------------------------
term:
lterm
| sterm
;
lterm:
'[' list ']' { "[ $_[2] ]" }
| '[' range ']' { "[ $_[2] ]" }
| '[' ']' { "[ ]" }
| '{' hash '}' { "{ $_[2] }" }
;
sterm:
ident { $factory->ident($_[1]) }
| REF ident { $factory->identref($_[2]) }
| '"' quoted '"' { $factory->quoted($_[2]) }
| LITERAL
| NUMBER
;
list:
list term { "$_[1], $_[2]" }
| list COMMA
| term
;
range:
sterm TO sterm { $_[1] . '..' . $_[3] }
;
hash:
params
| /* NULL */ { "" }
;
params:
params param { "$_[1], $_[2]" }
| params COMMA
| param
;
param:
LITERAL ASSIGN expr { "$_[1] => $_[3]" }
| item ASSIGN expr { "$_[1] => $_[3]" }
;
ident:
ident DOT node { push(@{$_[1]}, @{$_[3]}); $_[1] }
| ident DOT NUMBER { push( @{$_[1]}, map { ($_, 0) }
split(/\./, $_[3]) );
$_[1];
}
| node
;
node:
item { [ $_[1], 0 ] }
| item '(' args ')' { [ $_[1], $factory->args($_[3]) ] }
;
item:
IDENT { "'$_[1]'" }
| '${' sterm '}' { $_[2] }
| '$' IDENT { $_[0]->{ V1DOLLAR }
? "'$_[2]'"
: $factory->ident(["'$_[2]'", 0])
}
;
expr:
expr BINOP expr { "$_[1] $_[2] $_[3]" }
| expr '/' expr { "$_[1] $_[2] $_[3]" }
| expr '+' expr { "$_[1] $_[2] $_[3]" }
| expr DIV expr { "int($_[1] / $_[3])" }
| expr MOD expr { "$_[1] % $_[3]" }
| expr CMPOP expr { "$_[1] $CMPOP{ $_[2] } $_[3]" }
| expr CAT expr { "$_[1] . $_[3]" }
| expr AND expr { "$_[1] && $_[3]" }
| expr OR expr { "$_[1] || $_[3]" }
| NOT expr { "! $_[2]" }
| expr '?' expr ':' expr { "$_[1] ? $_[3] : $_[5]" }
| '(' assign ')' { $factory->assign(@{$_[2]}) }
| '(' expr ')' { "($_[2])" }
| term
;
setlist:
setlist assign { push(@{$_[1]}, @{$_[2]}); $_[1] }
| setlist COMMA
| assign
;
assign:
ident ASSIGN expr { [ $_[1], $_[3] ] }
| LITERAL ASSIGN expr { [ @_[1,3] ] }
;
# The 'args' production constructs a list of named and positional
# parameters. Named parameters are stored in a list in element 0
# of the args list. Remaining elements contain positional parameters
args:
args term { push(@{$_[1]}, $_[2]); $_[1] }
| args param { push(@{$_[1]->[0]}, $_[2]); $_[1] }
| args ident ASSIGN expr { push(@{$_[1]->[0]}, "'', " .
$factory->assign(@_[2,4])); $_[1] }
| args COMMA { $_[1] }
| /* init */ { [ [ ] ] }
;
# These are special case parameters used by INCLUDE, PROCESS, etc., which
# interpret barewords as quoted strings rather than variable identifiers;
# a leading '$' is used to explicitly specify a variable. It permits '/',
# '.' and '::' characters, allowing it to be used to specify filenames, etc.
# without requiring quoting.
lnameargs:
lvalue ASSIGN nameargs { push(@{$_[3]}, $_[1]); $_[3] }
| nameargs
;
lvalue:
item
| '"' quoted '"' { $factory->quoted($_[2]) }
| LITERAL
;
nameargs:
'$' ident args { [ [$factory->ident($_[2])], $_[3] ] }
| names args { [ @_[1,2] ] }
| names '(' args ')' { [ @_[1,3] ] }
;
names:
names '+' name { push(@{$_[1]}, $_[3]); $_[1] }
| name { [ $_[1] ] }
;
name:
'"' quoted '"' { $factory->quoted($_[2]) }
| filename { "'$_[1]'" }
| LITERAL
;
filename:
filename DOT filepart { "$_[1].$_[3]" }
| filepart
;
filepart:
FILENAME
| IDENT
| NUMBER
;
# The 'quoted' production builds a list of 'quotable' items that might
# appear in a quoted string, namely text and identifiers. The lexer
# adds an explicit ';' after each directive it finds to help the
# parser identify directive/text boundaries; we're not interested in
# them here so we can simply accept and ignore by returning undef
quoted:
quoted quotable { push(@{$_[1]}, $_[2])
if defined $_[2]; $_[1] }
| /* NULL */ { [ ] }
;
quotable:
ident { $factory->ident($_[1]) }
| TEXT { $factory->text($_[1]) }
| ';' { undef }
;
%%
1.1 TT3/tt2/parser/README
Index: README
===================================================================
#========================================================================
# Template::TT2 - parser
#========================================================================
This directory contains the YAPP grammar for the Template::TT2 parser. You
only need to worry about the files in this directory if you want to modify
the template parser grammar. If you're doing such a thing, then it is
assumed that you have some idea of what you're doing.
However, TT3 is now available and offers much better support for hacking
on the language, so you probably want to look there instead. If you are
sure that you want to be here then read on...
Files:
Parser.yp Yapp grammar file for the Template parser.
Grammar.pm.skel Skeleton file for ../lib/Template/TT2/Grammar.pm.
yc Simple shell cript to compile grammar and build new
../../lib/Template/TT3/Grammer.pm file from
Grammar.pm.skel and the output rules and states
generated from the grammar.
Parser.output Output file generated by the yapp parser. This is
for information and debugging purposes only and can
otherwise be ignored.
README This file
If you don't know what you're doing and would like to, then I can
recommend "Lex and Yacc" by John R. Levine, Tony Mason & Doug Brown
(O'Reilly, ISBN: 1-56592-000-7) which gives a good introduction
to the principles of an LALR parser and how to define grammars in YACC.
YAPP is identical to YACC in all the important ways. See also the
Parse::Yapp documentation and the comments in Template::Parser for more
info. For an in-depth study of parser and compiler theory, consult
"Compiler Theory and Practice", a.k.a. "The Dragon Book", by Alfred
V. Aho, Ravi Sethi and Jeffrey D.Ullman (Addison-Wesley, ISBN:
0-201-10194-7)
The parser grammar is compiled by 'yapp', the front-end script to
Francois Desarmenien's Parse::Yapp module(s). You will need Parse::Yapp
version 0.32 or later, available from CPAN, to compile the grammar.
The grammar file that yapp produces (../../Template/TT2/Grammar.pm)
contains the rule and state tables for the grammar. These are then
loaded by Template::TT2::Parser and used to run the DFA which is
implemented by the parse_directive() method. This has been derived
from the standalone parser created by Parse::Yapp.
Having modified the Parser.yp file to add your language changes,
simply run the following command to compile the grammar and install it
in the right place in the ../../lib/Template/TT2/Grammar.pm file.
./yc
You can then make, make test, make install, or whatever you normally
do, and the new grammar should be used by the template processor. To
revert to the original grammar, simply copy the original distribution
Parser.yp file back into this directory and repeat the above process.
To create a separate grammar, copy and modify the Parser.yp and
Grammar.pm.skel files as you wish and then run yapp to compile them:
yapp -v -s -o ../../lib/Template/TT2/MyGrammar.pm \
-t MyGrammar.pm.skel MyParser.yp
You can then instantiate you own grammar and pass this to the
Template constructor.
my $template = Template::TT2->new({
GRAMMAR => Template::TT2::MyGrammar->new(),
});
Changing the grammar is a simple process, in theory at least, if
you're familiar with YAPP/YACC. In practice, it also requires some
insight into the inner working of the Template Toolkit v2. You can
read more about this in the *fantastic* Badger Book (Perl Template
Toolkit, O'Reilly) which has a chapter all about it.
Andy Wardley <abw@wardley.org>
1.1 TT3/tt2/parser/yc
Index: yc
===================================================================
#!/bin/sh
#========================================================================
#
# yc - yapp compile
#
# This calls 'yapp', distributed with the Parse::Yapp module, to
# compile the parser grammar and construct the Template/TT2/Grammar.pm
# file under the '../../lib' directory above us. The grammar is defined
# in ./Parser.yp. The skeleton file Grammar.pm.skel is used as a template
# for creating the grammar file. An output file 'Parser.output' is
# generated containing a summary of the rule and state tables.
#
# You only need to run this script if you have changed the grammar and
# wish to recompile it.
#
# Andy Wardley <abw@wardley.org>
#
#========================================================================
: ${GRAMMAR:="Parser.yp"}
: ${OUTPUT:="../../lib/Template/TT2/Grammar.pm"}
: ${TEMPLATE:="Grammar.pm.skel"}
echo "Compiling parser grammar (${GRAMMAR} -> ${OUTPUT})"
yapp -v -s -o ${OUTPUT} -t ${TEMPLATE} ${GRAMMAR}