antlr-代码分析(3)-ant$LabeledExprLexer
LabeledExprLexer
再次看下LabeledExprLexer的有关问题
public class LabeledExprLexer extends Lexer {
protected static final DFA[] _decisionToDFA;
protected static final PredictionContextCache _sharedContextCache =
new PredictionContextCache();
public static final int
T__0=1, T__1=2, T__2=3, MUL=4, DIV=5, ADD=6, SUB=7, ID=8, INT=9, NEWLINE=10,
WS=11;
public static String[] modeNames = {
"DEFAULT_MODE"
};
public static final String[] ruleNames = {
"T__0", "T__1", "T__2", "MUL", "DIV", "ADD", "SUB", "ID", "INT", "NEWLINE",
"WS"
};
private static final String[] _LITERAL_NAMES = {
null, "'='", "'('", "')'", "'*'", "'/'", "'+'", "'-'"
};
private static final String[] _SYMBOLIC_NAMES = {
null, null, null, null, "MUL", "DIV", "ADD", "SUB", "ID", "INT", "NEWLINE",
"WS"
};
public static final Vocabulary VOCABULARY = new VocabularyImpl(_LITERAL_NAMES, _SYMBOLIC_NAMES);
/**
* @deprecated Use {@link #VOCABULARY} instead.
*/
@Deprecated
public static final String[] tokenNames;
static {
tokenNames = new String[_SYMBOLIC_NAMES.length];
for (int i = 0; i < tokenNames.length; i++) {
tokenNames[i] = VOCABULARY.getLiteralName(i);
if (tokenNames[i] == null) {
tokenNames[i] = VOCABULARY.getSymbolicName(i);
}
if (tokenNames[i] == null) {
tokenNames[i] = "<INVALID>";
}
}
}
@Override
@Deprecated
public String[] getTokenNames() {
return tokenNames;
}
public Vocabulary getVocabulary() {
return VOCABULARY;
}
public LabeledExprLexer(CharStream input) {
super(input);
_interp = new LexerATNSimulator(this,_ATN,_decisionToDFA,_sharedContextCache);
}
@Override
public String getGrammarFileName() { return "LabeledExpr.g4"; }
@Override
public String[] getRuleNames() { return ruleNames; }
@Override
public String getSerializedATN() { return _serializedATN; }
@Override
public String[] getModeNames() { return modeNames; }
@Override
public ATN getATN() { return _ATN; }
public static final ATN _ATN =
new ATNDeserializer().deserialize(_serializedATN.toCharArray());
static {
_decisionToDFA = new DFA[_ATN.getNumberOfDecisions()];
for (int i = 0; i < _ATN.getNumberOfDecisions(); i++) {
_decisionToDFA[i] = new DFA(_ATN.getDecisionState(i), i);
}
}
}
现在学习几个概念ATN、DFA、PredictionContextCache、LexerATNSimulator、ATNDeserializer、ATNState、DecisionState
ATN
public class ATN {
public static final int INVALID_ALT_NUMBER = 0;
public final List<ATNState> states = new ArrayList<ATNState>(); //ANT状态流
public final List<DecisionState> decisionToState = new ArrayList<DecisionState>();
...
}
ATNState表示ANT的状态，ATNState内部有个List存储这些状态的名字
public static final List<String> serializationNames =
Collections.unmodifiableList(Arrays.asList(
"INVALID",
"BASIC",
"RULE_START",
"BLOCK_START",
"PLUS_BLOCK_START",
"STAR_BLOCK_START",
"TOKEN_START",
"RULE_STOP",
"BLOCK_END",
"STAR_LOOP_BACK",
"STAR_LOOP_ENTRY",
"PLUS_LOOP_BACK",
"LOOP_END"
));
//还有个List存储这些状态对应的转换
protected final List<Transition> transitions =
new ArrayList<Transition>(INITIAL_NUM_TRANSITIONS);
看下RuleStartState，继承自ATNState定义了是否是getStateType返回的内容即可；
public final class RuleStartState extends ATNState {
public RuleStopState stopState;
public boolean isLeftRecursiveRule;
@Override
public int getStateType() {
return RULE_START;
}
}
Transition是ATN中两个ATNstates之间的状态转换
//记录Transition的名字
public static final List<String> serializationNames =
Collections.unmodifiableList(Arrays.asList(
"INVALID",
"EPSILON",
"RANGE",
"RULE",
"PREDICATE",
"ATOM",
"ACTION",
"SET",
"NOT_SET",
"WILDCARD",
"PRECEDENCE"
));
//记录Transition到数字的映射
public static final Map<Class<? extends Transition>, Integer> serializationTypes =
Collections.unmodifiableMap(new HashMap<Class<? extends Transition>, Integer>() {{
put(EpsilonTransition.class, EPSILON);
put(RangeTransition.class, RANGE);
put(RuleTransition.class, RULE);
put(PredicateTransition.class, PREDICATE);
put(AtomTransition.class, ATOM);
put(ActionTransition.class, ACTION);
put(SetTransition.class, SET);
put(NotSetTransition.class, NOT_SET);
put(WildcardTransition.class, WILDCARD);
put(PrecedencePredicateTransition.class, PRECEDENCE);
}});
//transition的目标状态
public ATNState target;
//构造时需要传入目标状态
protected Transition(ATNState target) {
if (target == null) {
throw new NullPointerException("target cannot be null.");
}
this.target = target;
}
看一个Transition的实现类RuleTransition
public final class RuleTransition extends Transition {
public final int ruleIndex; //rule号
public final int precedence; //
//构造时候需要传入RuleStartState、ruleIndex、ATNState
public RuleTransition(RuleStartState ruleStart,
int ruleIndex,
ATNState followState)
{
this(ruleStart, ruleIndex, 0, followState);
}
public RuleTransition(RuleStartState ruleStart,
int ruleIndex,
int precedence,
ATNState followState)
{
super(ruleStart); //把target设置为ruleStart
this.ruleIndex = ruleIndex;
this.precedence = precedence;
this.followState = followState;
}
//获取的状态为Rule
public int getSerializationType() {
return RULE;
}
}
再看ANT内容
public class ATN {
public static final int INVALID_ALT_NUMBER = 0;
public final List<ATNState> states = new ArrayList<ATNState>(); //ANT状态流
public final List<DecisionState> decisionToState = new ArrayList<DecisionState>();
public RuleStartState[] ruleToStartState;
public RuleStopState[] ruleToStopState;
//TokensStartState继承自DecisionState
public final Map<String, TokensStartState> modeNameToStartState =
new LinkedHashMap<String, TokensStartState>();
//ant的类型
public final ATNType grammarType;
//ATN中转换的最大值
public final int maxTokenType;
//转换过程中的action
public LexerAction[] lexerActions;
public final List<TokensStartState> modeToStartState = new ArrayList<TokensStartState>();
//构造
public ATN(ATNType grammarType, int maxTokenType) {
this.grammarType = grammarType;
this.maxTokenType = maxTokenType;
}
public IntervalSet nextTokens(ATNState s, RuleContext ctx) {
LL1Analyzer anal = new LL1Analyzer(this);
IntervalSet next = anal.LOOK(s, ctx);
return next;
}
public void addState(ATNState state) {
if (state != null) {
state.atn = this;
state.stateNumber = states.size();
}
states.add(state);
}
public void removeState(ATNState state) {
states.set(state.stateNumber, null); // just free mem, don't shift states in list
}
public int defineDecisionState(DecisionState s) {
decisionToState.add(s);
s.decision = decisionToState.size()-1;
return s.decision;
}
public DecisionState getDecisionState(int decision) {
if ( !decisionToState.isEmpty() ) {
return decisionToState.get(decision);
}
return null;
}
public int getNumberOfDecisions() {
return decisionToState.size();
}
public IntervalSet getExpectedTokens(int stateNumber, RuleContext context) {
if (stateNumber < 0 || stateNumber >= states.size()) {
throw new IllegalArgumentException("Invalid state number.");
}
RuleContext ctx = context;
ATNState s = states.get(stateNumber);
IntervalSet following = nextTokens(s);
if (!following.contains(Token.EPSILON)) {
return following;
}
IntervalSet expected = new IntervalSet();
expected.addAll(following);
expected.remove(Token.EPSILON);
while (ctx != null && ctx.invokingState >= 0 && following.contains(Token.EPSILON)) {
ATNState invokingState = states.get(ctx.invokingState);
RuleTransition rt = (RuleTransition)invokingState.transition(0);
following = nextTokens(rt.followState);
expected.addAll(following);
expected.remove(Token.EPSILON);
ctx = ctx.parent;
}
if (following.contains(Token.EPSILON)) {
expected.add(Token.EOF);
}
return expected;
}
}
DecisionState也是ATNState的子类
public abstract class DecisionState extends ATNState {
public int decision = -1;
public boolean nonGreedy;
}
典型的子类
public final class TokensStartState extends DecisionState {
@Override
public int getStateType() {
return TOKEN_START;
}
}
public final class StarLoopEntryState extends DecisionState {
public StarLoopbackState loopBackState;
public boolean isPrecedenceDecision;
@Override
public int getStateType() {
return STAR_LOOP_ENTRY;
}
}
RuleStartState和RuleStopState是开始和结束状态
public final class RuleStartState extends ATNState {
public RuleStopState stopState;
public boolean isLeftRecursiveRule;
@Override
public int getStateType() {
return RULE_START;
}
}
public final class RuleStopState extends ATNState {
@Override
public int getStateType() {
return RULE_STOP;
}
}
LexerAction
public interface LexerAction {
//获取action的序列化类型
LexerActionType getActionType();
//是否是位置依赖的lexer action
boolean isPositionDependent();
void execute(Lexer lexer);
}
DFA
DFA
public class DFA {
//获取所有的DFAState集合
public final Map<DFAState, DFAState> states = new HashMap<DFAState, DFAState>();
public volatile DFAState s0;
public final int decision;
//从哪个DecisionState创建的这个DFA
public final DecisionState atnStartState;
//是否是为了precedence的decision
private final boolean precedenceDfa;
public final boolean isPrecedenceDfa() {
return precedenceDfa;
}
public DFA(DecisionState atnStartState) {
this(atnStartState, 0);
}
public DFA(DecisionState atnStartState, int decision) {
this.atnStartState = atnStartState;
this.decision = decision;
boolean precedenceDfa = false;
if (atnStartState instanceof StarLoopEntryState) {
if (((StarLoopEntryState)atnStartState).isPrecedenceDecision) {
precedenceDfa = true;
DFAState precedenceState = new DFAState(new ATNConfigSet());
precedenceState.edges = new DFAState[0];
precedenceState.isAcceptState = false;
precedenceState.requiresFullContext = false;
this.s0 = precedenceState;
}
}
this.precedenceDfa = precedenceDfa;
}
public final DFAState getPrecedenceStartState(int precedence) {
if (!isPrecedenceDfa()) {
throw new IllegalStateException("Only precedence DFAs may contain a precedence start state.");
}
// s0.edges is never null for a precedence DFA
if (precedence < 0 || precedence >= s0.edges.length) {
return null;
}
return s0.edges[precedence];
}
public final void setPrecedenceStartState(int precedence, DFAState startState) {
if (!isPrecedenceDfa()) {
throw new IllegalStateException("Only precedence DFAs may contain a precedence start state.");
}
if (precedence < 0) {
return;
}
// synchronization on s0 here is ok. when the DFA is turned into a
// precedence DFA, s0 will be initialized once and not updated again
synchronized (s0) {
// s0.edges is never null for a precedence DFA
if (precedence >= s0.edges.length) {
s0.edges = Arrays.copyOf(s0.edges, precedence + 1);
}
s0.edges[precedence] = startState;
}
}
public List<DFAState> getStates() {
List<DFAState> result = new ArrayList<DFAState>(states.keySet());
Collections.sort(result, new Comparator<DFAState>() {
@Override
public int compare(DFAState o1, DFAState o2) {
return o1.stateNumber - o2.stateNumber;
}
});
return result;
}
}
DFAState
代表一些可能得ATN配置
public class DFAState {
public int stateNumber = -1;
public ATNConfigSet configs = new ATNConfigSet();
public DFAState[] edges;
public boolean isAcceptState = false;
public int prediction;
//LexerActionExecutor词法action执行器
public LexerActionExecutor lexerActionExecutor;
public boolean requiresFullContext;
//PredPrediction数组
public PredPrediction[] predicates;
public static class PredPrediction {
public SemanticContext pred; // never null; at least SemanticContext.NONE
public int alt;
public PredPrediction(SemanticContext pred, int alt) {
this.alt = alt;
this.pred = pred;
}
@Override
public String toString() {
return "("+pred+", "+alt+ ")";
}
}
//构造时可以设置stateNumber或者ATNConfigSet
public DFAState() { }
public DFAState(int stateNumber) { this.stateNumber = stateNumber; }
public DFAState(ATNConfigSet configs) { this.configs = configs; }
//获得ATN配置的alts，从ATNConfigSet中获取所有ATNConfig的alt
public Set<Integer> getAltSet() {
Set<Integer> alts = new HashSet<Integer>();
if ( configs!=null ) {
for (ATNConfig c : configs) {
alts.add(c.alt);
}
}
if ( alts.isEmpty() ) return null;
return alts;
}
}
ATNConfig
代码如下
构造函数很多,典型的如下
public ATNConfig(ATNConfig c, ATNState state,
PredictionContext context,
SemanticContext semanticContext)
{
this.state = state;
this.alt = c.alt;
this.context = context;
this.semanticContext = semanticContext;
this.reachesIntoOuterContext = c.reachesIntoOuterContext;
}
其他部分
public class ATNConfig {
private static final int SUPPRESS_PRECEDENCE_FILTER = 0x40000000;
public final ATNState state;
public final int alt; //存储的alt或者lexer rule
public PredictionContext context;
public int reachesIntoOuterContext;
public final SemanticContext semanticContext;
public final int getOuterContextDepth() {
return reachesIntoOuterContext & ~SUPPRESS_PRECEDENCE_FILTER;
}
public final boolean isPrecedenceFilterSuppressed() {
return (reachesIntoOuterContext & SUPPRESS_PRECEDENCE_FILTER) != 0;
}
public final void setPrecedenceFilterSuppressed(boolean value) {
if (value) {
this.reachesIntoOuterContext |= 0x40000000;
}
else {
this.reachesIntoOuterContext &= ~SUPPRESS_PRECEDENCE_FILTER;
}
}
}
ATNConfigSet
代码如下
这个是一个set，内存存储了ATNConfig
构造函数
public ATNConfigSet(boolean fullCtx) {
configLookup = new ConfigHashSet();
this.fullCtx = fullCtx;
}
public ATNConfigSet() { this(true); }
public ATNConfigSet(ATNConfigSet old) {
this(old.fullCtx);
addAll(old);
this.uniqueAlt = old.uniqueAlt;
this.conflictingAlts = old.conflictingAlts;
this.hasSemanticContext = old.hasSemanticContext;
this.dipsIntoOuterContext = old.dipsIntoOuterContext;
}
其他部分
public class ATNConfigSet implements Set<ATNConfig> {
//ConfigHashSet和AbstractConfigHashSet都是内置的set
ConfigHashSet/AbstractConfigHashSet
protected boolean readonly = false;
public AbstractConfigHashSet configLookup;
public final ArrayList<ATNConfig> configs = new ArrayList<ATNConfig>(7);
public int uniqueAlt;
protected BitSet conflictingAlts;
public boolean hasSemanticContext;
public boolean dipsIntoOuterContext;
public final boolean fullCtx;
private int cachedHashCode = -1;
...
public List<ATNConfig> elements() { return configs; }
public Set<ATNState> getStates() {
Set<ATNState> states = new HashSet<ATNState>();
for (ATNConfig c : configs) {
states.add(c.state);
}
return states;
}
public BitSet getAlts() {
BitSet alts = new BitSet();
for (ATNConfig config : configs) {
alts.set(config.alt);
}
return alts;
}
...
}
浙公网安备 33010602011771号