一个简单词法分析器的实现代码（java实现）

Main.java

/* * 主程序 */ import java.io.*; import lexer.*; public class Main { public static void main(String[] args) throws IOException { Lexer lexer = new Lexer(); while (lexer.getReaderState() == false) { lexer.scan(); } /* 保存相关信息 */ lexer.saveTokens(); lexer.saveSymbolsTable(); } }

Lexer.java

package lexer; import java.io.*; import java.util.*; import symbols.*; public class Lexer { public static int line = 1; /* 记录行号 */ char peek = ' '; /* 下一个读入字符 */ Hashtable<String, Word> words = new Hashtable<String, Word>(); /* 符号表 */ private Hashtable<Token, String> table = new Hashtable<Token, String>(); /* token序列 */ private List<String> tokens = new LinkedList<String> (); /* 读取文件变量 */ BufferedReader reader = null; /* 保存当前是否读取到了文件的结尾 */ private Boolean isEnd = false; /* 是否读取到文件的结尾 */ public Boolean getReaderState() { return this.isEnd; } /* 保存存储在table中的 */ public void saveSymbolsTable() throws IOException { FileWriter writer = new FileWriter("符号表.txt"); writer.write("[符号] [符号类型信息]\n"); writer.write("\r\n"); Enumeration<Token> e = table.keys(); while( e.hasMoreElements() ){ Token token = (Token)e.nextElement(); String desc = table.get(token); /* 写入文件 */ writer.write(token + "\t\t\t" + desc + "\r\n"); } writer.flush(); } /* 保存Tokens */ public void saveTokens() throws IOException { FileWriter writer = new FileWriter("Tokens表.txt"); writer.write("[符号] \n"); writer.write("\r\n"); for(int i = 0; i < tokens.size(); ++i) { String tok = (String)tokens.get(i); /* 写入文件 */ writer.write(tok + "\r\n"); } writer.flush(); } void reserve(Word w) { words.put(w.lexme, w); } /* * 构造函数中将关键字和类型添加到hashtable words中 */ public Lexer() { /* 初始化读取文件变量 */ try { reader = new BufferedReader(new FileReader("输入.txt")); } catch(IOException e) { System.out.print(e); } /* 关键字 */ this.reserve(new Word("if", Tag.IF)); this.reserve(new Word("then", Tag.THEN)); this.reserve(new Word("else", Tag.ELSE)); this.reserve(new Word("while", Tag.WHILE)); this.reserve(new Word("do", Tag.DO)); /* 类型 */ this.reserve(Word.True); this.reserve(Word.False); this.reserve(Type.Int); this.reserve(Type.Char); this.reserve(Type.Bool); this.reserve(Type.Float); } public void readch() throws IOException { /* 这里应该是使用的是 */ peek = (char)reader.read(); if((int)peek == 0xffff){ this.isEnd = true; } // peek = (char)System.in.read(); } public Boolean readch(char ch) throws IOException { readch(); if (this.peek != ch) { return false; } this.peek = ' '; return true; } public Token scan() throws IOException { /* 消除空白 */ for( ; ; readch() ) { if(peek == ' ' || peek == '\t') continue; else if (peek == '\n') line = line + 1; else break; } /* 下面开始分割关键字，标识符等信息 */ switch (peek) { /* 对于 ==, >=, <=, !=的区分使用状态机实现 */ case '=' : if (readch('=')) { tokens.add("=="); return Word.eq; } else { tokens.add("="); return new Token('='); } case '>' : if (readch('=')) { tokens.add(">="); return Word.ge; } else { tokens.add(">"); return new Token('>'); } case '<' : if (readch('=')) { tokens.add("<="); return Word.le; } else { tokens.add("<"); return new Token('<'); } case '!' : if (readch('=')) { tokens.add("!="); return Word.ne; } else { tokens.add("!"); return new Token('!'); } } /* 下面是对数字的识别，根据文法的规定的话，这里的 * 数字只要是能够识别整数就行. */ if(Character.isDigit(peek)) { int value = 0; do { value = 10 * value + Character.digit(peek, 10); readch(); } while (Character.isDigit(peek)); Num n = new Num(value); tokens.add(n.toString()); //table.put(n, "Num"); return n; } /* * 关键字或者是标识符的识别 */ if(Character.isLetter(peek)) { StringBuffer sb = new StringBuffer(); /* 首先得到整个的一个分割 */ do { sb.append(peek); readch(); } while (Character.isLetterOrDigit(peek)); /* 判断是关键字还是标识符 */ String s = sb.toString(); Word w = (Word)words.get(s); /* 如果是关键字或者是类型的话，w不应该是空的 */ if(w != null) { // table.put(w, "KeyWord or Type"); tokens.add(w.toString()); return w; /* 说明是关键字 或者是类型名 */ } /* 否则就是一个标识符id */ w = new Word(s, Tag.ID); tokens.add(w.toString()); table.put(w, "id"); words.put(s, w); return w; } /* peek中的任意字符都被认为是词法单元返回 */ Token tok = new Token(peek); // table.put(tok, "Token or Seprator"); if ((int)peek != 0xffff ) tokens.add(tok.toString()); peek = ' '; return tok; } }

Num.java

package lexer; public class Num extends Token{ public final int value; public Num(int v) { super(Tag.NUM); this.value = v; } public String toString() { return "" + value; } }

Tag.java

package lexer; public class Tag { public final static int AND = 256, BASIC = 257, BREAK = 258, DO = 259, ELSE = 260, EQ = 261, /* == */ FALSE = 262, GE = 263, ID = 264, IF = 265, INDEX = 266, LE = 267, MINUS = 268, NE = 269, NUM = 270, OR = 271, REAL = 272, TEMP = 273, TRUE = 274, WHILE = 275, /* 后面添加 */ THEN = 276; }

Token.java

package lexer; public class Token { public final int tag; public Token(int t) { this.tag = t; } public String toString() { return "" + (char)tag; } public static void main(String[] args) { Token tok = new Token('a'); System.out.println(tok); } }

Word.java

/* * 类word用于管理保留字，标识符以及像&&这样的复合单词元素 。 */ package lexer; public class Word extends Token { public String lexme = ""; public Word (String s, int t) { super(t); this.lexme = s; } public String toString() { return this.lexme; } public static final Word and = new Word("&&", Tag.AND), or = new Word("||", Tag.OR), eq = new Word ("==", Tag.EQ), ne = new Word("!=", Tag.NE), le = new Word("<=", Tag.LE), ge = new Word(">=", Tag.GE), minus = new Word("minus", Tag.MINUS), True = new Word("true", Tag.TRUE), False = new Word("false", Tag.FALSE), temp = new Word("t", Tag.TEMP); }

Type.java

/* * 说明数据类型 */ package symbols; import lexer.*; public class Type extends Word{ public Type(String s, int tag) { super(s, tag); } public static final Type Int = new Type("int", Tag.BASIC), Float = new Type("float", Tag.BASIC), Char = new Type ("char", Tag.BASIC), Bool = new Type("bool", Tag.BASIC); }