antlr-代码分析(4)-LabeledExprParser
antlr使用的代码如下
String inputFile = "D:\\code\\zsh_demo\\zsh-demo-data\\ANTLR\\src\\main\\java\\split\\file";
InputStream is = System.in;
if ( inputFile!=null ) is = new FileInputStream(inputFile);
ANTLRInputStream input = new ANTLRInputStream(is);
LabeledExprLexer lexer = new LabeledExprLexer(input);
CommonTokenStream tokens = new CommonTokenStream(lexer);
LabeledExprParser parser = new LabeledExprParser(tokens);
ParseTree tree = parser.prog(); // parse
System.out.println(tree.toStringTree(parser));
在调用LabeledExprLexer之后，生成了lexer
CommonTokenStream
CommonTokenStream->BufferedTokenStream->TokenStream->IntStream
CommonTokenStream
回归一下
public class CommonTokenStream extends BufferedTokenStream {
//用来过滤tokens，默认值是0
protected int channel = Token.DEFAULT_CHANNEL;
//构造一个新的
public CommonTokenStream(TokenSource tokenSource) {
super(tokenSource);
}
//构造时可以指定channel号
public CommonTokenStream(TokenSource tokenSource, int channel) {
this(tokenSource);
this.channel = channel;
}
adjustSeekIndex
LB
LT
getNumberOfOnChannelTokens
}
BufferedTokenStream
public class BufferedTokenStream implements TokenStream {
//从这里获取token
protected TokenSource tokenSource;
//从TokenSource中获取的所有token
protected List<Token> tokens = new ArrayList<Token>(100);
//现在token的指针位置
protected int p = -1;
//是否已经把EOF获取到tokens里
protected boolean fetchedEOF;
//构造时传入TokenSource
public BufferedTokenStream(TokenSource tokenSource) {
if (tokenSource == null) {
throw new NullPointerException("tokenSource cannot be null");
}
this.tokenSource = tokenSource;
}
public void consume() {
boolean skipEofCheck;
if (p >= 0) {
if (fetchedEOF) {
// the last token in tokens is EOF. skip check if p indexes any
// fetched token except the last.
skipEofCheck = p < tokens.size() - 1;
}
else {
// no EOF token in tokens. skip check if p indexes a fetched token.
skipEofCheck = p < tokens.size();
}
}
else {
// not yet initialized
skipEofCheck = false;
}
if (!skipEofCheck && LA(1) == EOF) {
throw new IllegalStateException("cannot consume EOF");
}
if (sync(p + 1)) {
p = adjustSeekIndex(p + 1);
}
}
...
}
这样获取了tokens
LabeledExprParser
Parser->Recognizer
parser类似Lexer，都是继承自Recognizer
这是基本是解析器能够支持的全部功能，大部分都是错误处理
这里的识别器Recognizer传入的上是Recognizer<Token, ParserATNSimulator>
ParseTreeListener
触发ParseTreeWalker的最小核心
public interface ParseTreeListener {
void visitTerminal(TerminalNode node);
void visitErrorNode(ErrorNode node);
void enterEveryRule(ParserRuleContext ctx);
void exitEveryRule(ParserRuleContext ctx);
}
TraceListener
public class TraceListener implements ParseTreeListener {
@Override
public void enterEveryRule(ParserRuleContext ctx) {
System.out.println("enter " + getRuleNames()[ctx.getRuleIndex()] +
", LT(1)=" + _input.LT(1).getText());
}
@Override
public void visitTerminal(TerminalNode node) {
System.out.println("consume "+node.getSymbol()+" rule "+
getRuleNames()[_ctx.getRuleIndex()]);
}
@Override
public void visitErrorNode(ErrorNode node) {
}
@Override
public void exitEveryRule(ParserRuleContext ctx) {
System.out.println("exit "+getRuleNames()[ctx.getRuleIndex()]+
", LT(1)="+_input.LT(1).getText());
}
}
TrimToSizeListener
public static class TrimToSizeListener implements ParseTreeListener {
public static final TrimToSizeListener INSTANCE = new TrimToSizeListener();
@Override
public void enterEveryRule(ParserRuleContext ctx) { }
@Override
public void visitTerminal(TerminalNode node) { }
@Override
public void visitErrorNode(ErrorNode node) { }
@Override
public void exitEveryRule(ParserRuleContext ctx) {
if (ctx.children instanceof ArrayList) {
((ArrayList<?>)ctx.children).trimToSize();
}
}
}
Parser的具体内容
private static final Map<String, ATN> bypassAltsAtnCache =
new WeakHashMap<String, ATN>();
protected ANTLRErrorStrategy _errHandler = new DefaultErrorStrategy();
protected TokenStream _input;
protected final IntegerStack _precedenceStack;
{
_precedenceStack = new IntegerStack();
_precedenceStack.push(0);
}
protected ParserRuleContext _ctx;
protected boolean _buildParseTrees = true;
private TraceListener _tracer;
//核心的部分是_parseListeners
protected List<ParseTreeListener> _parseListeners;
public void setTrimParseTree(boolean trimParseTrees) {
if (trimParseTrees) {
if (getTrimParseTree()) return;
addParseListener(TrimToSizeListener.INSTANCE);
}
else {
removeParseListener(TrimToSizeListener.INSTANCE);
}
}
public List<ParseTreeListener> getParseListeners()
public void addParseListener(ParseTreeListener listener)
protected int _syntaxErrors;
//统配匹配当前的输入符号
public Token matchWildcard() throws RecognitionException
public Token getCurrentToken() {
return _input.LT(1);
}
public Token consume() {
Token o = getCurrentToken();
if (o.getType() != EOF) {
getInputStream().consume();
}
boolean hasListener = _parseListeners != null && !_parseListeners.isEmpty();
if (_buildParseTrees || hasListener) {
if ( _errHandler.inErrorRecoveryMode(this) ) {
ErrorNode node = _ctx.addErrorNode(o);
if (_parseListeners != null) {
for (ParseTreeListener listener : _parseListeners) {
listener.visitErrorNode(node);
}
}
}
else {
TerminalNode node = _ctx.addChild(o);
if (_parseListeners != null) {
for (ParseTreeListener listener : _parseListeners) {
listener.visitTerminal(node);
}
}
}
}
return o;
}
public void enterRule(ParserRuleContext localctx, int state, int ruleIndex) {
setState(state);
_ctx = localctx;
_ctx.start = _input.LT(1);
if (_buildParseTrees) addContextToParseTree();
if ( _parseListeners != null) triggerEnterRuleEvent();
}
public void exitRule() {
if ( matchedEOF ) {
// if we have matched EOF, it cannot consume past EOF so we use LT(1) here
_ctx.stop = _input.LT(1); // LT(1) will be end of file
}
else {
_ctx.stop = _input.LT(-1); // stop node is what we just matched
}
// trigger event on _ctx, before it reverts to parent
if ( _parseListeners != null) triggerExitRuleEvent();
setState(_ctx.invokingState);
_ctx = (ParserRuleContext)_ctx.parent;
}
LabeledExprParser
public class LabeledExprParser extends Parser {
protected static final DFA[] _decisionToDFA;
protected static final PredictionContextCache _sharedContextCache =
new PredictionContextCache();
public static final int
T__0=1, T__1=2, T__2=3, MUL=4, DIV=5, ADD=6, SUB=7, ID=8, INT=9, NEWLINE=10,
WS=11;
public static final int
RULE_prog = 0, RULE_stat = 1, RULE_expr = 2;
public static final String[] ruleNames = {
"prog", "stat", "expr"
};
private static final String[] _LITERAL_NAMES = {
null, "'='", "'('", "')'", "'*'", "'/'", "'+'", "'-'"
};
private static final String[] _SYMBOLIC_NAMES = {
null, null, null, null, "MUL", "DIV", "ADD", "SUB", "ID", "INT", "NEWLINE",
"WS"
};
public static final Vocabulary VOCABULARY = new VocabularyImpl(_LITERAL_NAMES, _SYMBOLIC_NAMES);
public static final String[] tokenNames;
static {
tokenNames = new String[_SYMBOLIC_NAMES.length];
for (int i = 0; i < tokenNames.length; i++) {
tokenNames[i] = VOCABULARY.getLiteralName(i);
if (tokenNames[i] == null) {
tokenNames[i] = VOCABULARY.getSymbolicName(i);
}
if (tokenNames[i] == null) {
tokenNames[i] = "<INVALID>";
}
}
}
...
public LabeledExprParser(TokenStream input) {
super(input);
_interp = new ParserATNSimulator(this,_ATN,_decisionToDFA,_sharedContextCache);
}
}
StatContext
public static class StatContext extends ParserRuleContext {
public StatContext(ParserRuleContext parent, int invokingState) {
super(parent, invokingState);
}
@Override public int getRuleIndex() { return RULE_stat; }
public StatContext() { }
public void copyFrom(StatContext ctx) {
super.copyFrom(ctx);
}
}
BlankContext
public static class BlankContext extends StatContext {
public TerminalNode NEWLINE() { return getToken(LabeledExprParser.NEWLINE, 0); }
public BlankContext(StatContext ctx) { copyFrom(ctx); }
@Override
public void enterRule(ParseTreeListener listener) {
if ( listener instanceof LabeledExprListener ) ((LabeledExprListener)listener).enterBlank(this);
}
@Override
public void exitRule(ParseTreeListener listener) {
if ( listener instanceof LabeledExprListener ) ((LabeledExprListener)listener).exitBlank(this);
}
@Override
public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
if ( visitor instanceof LabeledExprVisitor ) return ((LabeledExprVisitor<? extends T>)visitor).visitBlank(this);
else return visitor.visitChildren(this);
}
}
AssignContext
public static class AssignContext extends StatContext {
public TerminalNode ID() { return getToken(LabeledExprParser.ID, 0); }
public ExprContext expr() {
return getRuleContext(ExprContext.class,0);
}
public TerminalNode NEWLINE() { return getToken(LabeledExprParser.NEWLINE, 0); }
public AssignContext(StatContext ctx) { copyFrom(ctx); }
@Override
public void enterRule(ParseTreeListener listener) {
if ( listener instanceof LabeledExprListener ) ((LabeledExprListener)listener).enterAssign(this);
}
@Override
public void exitRule(ParseTreeListener listener) {
if ( listener instanceof LabeledExprListener ) ((LabeledExprListener)listener).exitAssign(this);
}
@Override
public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
if ( visitor instanceof LabeledExprVisitor ) return ((LabeledExprVisitor<? extends T>)visitor).visitAssign(this);
else return visitor.visitChildren(this);
}
}
内部类ProgContext->ParserRuleContext
public static class ProgContext extends ParserRuleContext {
public List<StatContext> stat() {
return getRuleContexts(StatContext.class);
}
public StatContext stat(int i) {
return getRuleContext(StatContext.class,i);
}
//初始化设置parent和invokingState
public ProgContext(ParserRuleContext parent, int invokingState) {
super(parent, invokingState);
}
@Override
public int getRuleIndex() { return RULE_prog; }
@Override
public void enterRule(ParseTreeListener listener) {
if ( listener instanceof LabeledExprListener ) ((LabeledExprListener)listener).enterProg(this);
}
@Override
public void exitRule(ParseTreeListener listener) {
if ( listener instanceof LabeledExprListener ) ((LabeledExprListener)listener).exitProg(this);
}
@Override
public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
if ( visitor instanceof LabeledExprVisitor ) return ((LabeledExprVisitor<? extends T>)visitor).visitProg(this);
else return visitor.visitChildren(this);
}
}
不清楚了，LabeledExprParser的内部类实在太多，从函数调用的角度看吧
看下面的代码段
CommonTokenStream tokens = new CommonTokenStream(lexer);
LabeledExprParser parser = new LabeledExprParser(tokens);
ParseTree tree = parser.prog(); // parse
System.out.println(tree.toStringTree(parser));
prog方法
public final ProgContext prog() throws RecognitionException {
//初始化时为null -1
ProgContext _localctx = new ProgContext(_ctx, getState());
enterRule(_localctx, 0, RULE_prog);
int _la;
//循环解析
try {
enterOuterAlt(_localctx, 1);
{
setState(7);
_errHandler.sync(this);
_la = _input.LA(1);
do {
{
{
setState(6);
stat();
}
}
setState(9);
_errHandler.sync(this);
_la = _input.LA(1);
} while ( (((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << T__1) | (1L << ID) | (1L << INT) | (1L << NEWLINE))) != 0) );
}
}
catch (RecognitionException re) {
_localctx.exception = re;
_errHandler.reportError(this, re);
_errHandler.recover(this, re);
}
finally {
exitRule();
}
return _localctx;
}
ProgContext的构造过程，把parenet设置为null状态设置为-1；
ProgContext是ParserRuleContext的子类
然后调用enterRule方法enterRule(_localctx, 0, RULE_prog);第一个参数是刚刚创建的ProgContext，RULE_prog为第一个规则为0
public void enterRule(ParserRuleContext localctx, int state, int ruleIndex) {
setState(state);
_ctx = localctx;
_ctx.start = _input.LT(1); //获取第一个token设置为start
//添加到解析树上
if (_buildParseTrees) addContextToParseTree();
//listner
if ( _parseListeners != null) triggerEnterRuleEvent();
}
然后调用addContextToParseTree把子节点添加到解析树上
triggerEnterRuleEvent方法生成listner
最后生成一个ProgContext
浙公网安备 33010602011771号