javac的词法分析，转载自：百度文库

public class Mess {

	/*-------------------ScannerFactory class-----------------------*/
	final Names names;
	final Source source;
	final Keywords keywords;

	/** Create a new scanner factory. */
	protected ScannerFactory(Context context) {
		context.put(scannerFactoryKey, this);
		this.log = Log.instance(context);
		this.names = Names.instance(context);/*---first, add name in the Names to table---*/
		this.source = Source.instance(context);
		this.keywords = Keywords.instance(context);/*---then, add Token constant to table---*/
	}

	/*----------------Scanner class---------------------------------*/
	/** The name table. */
	private final Names names;

	/** The keyword table. */
	private final Keywords keywords;

	/** Common code for constructors. */
	private Scanner(ScannerFactory fac) {
		log = fac.log;
		names = fac.names;
		keywords = fac.keywords;
		source = fac.source;
		allowBinaryLiterals = source.allowBinaryLiterals();
		allowHexFloats = source.allowHexFloats();
		allowUnderscoresInLiterals = source.allowUnderscoresInLiterals();
	}

	/*----------------Names class----------------------------------*/
	public static final Context.Key<Names> namesKey = new Context.Key<Names>();

	public static Names instance(Context context) {
		Names instance = context.get(namesKey);
		if (instance == null) {
			instance = new Names(context);
			context.put(namesKey, instance);
		}
		return instance;
	}

	public final Name slash;
	public final Name hyphen;
	public final Name T;
	public final Name slashequals;
	public final Name deprecated;
	public final Name init;
	public final Name clinit;
	public final Name error;
	public final Name any;
	public final Name empty;
	public final Name one;
	public final Name period;
	public final Name comma;
	public final Name semicolon;
	public final Name asterisk;
	public final Name _this;
	public final Name _super;
	public final Name _default;
	public final Name _class;
	public final Name java_lang;
	public final Name java_lang_Object;
	public final Name java_lang_Class;
	public final Name java_lang_Cloneable;
	public final Name java_io_Serializable;
	public final Name serialVersionUID;
	public final Name java_lang_Enum;
	public final Name java_lang_invoke_MethodHandle;
	public final Name package_info;
	public final Name ConstantValue;
	public final Name LineNumberTable;
	public final Name LocalVariableTable;
	public final Name LocalVariableTypeTable;
	public final Name CharacterRangeTable;
	public final Name StackMap;
	public final Name StackMapTable;
	public final Name SourceID;
	public final Name CompilationID;
	public final Name Code;
	public final Name Exceptions;
	public final Name SourceFile;
	public final Name InnerClasses;
	public final Name Synthetic;
	public final Name Bridge;
	public final Name Deprecated;
	public final Name Enum;
	public final Name _name;
	public final Name Signature;
	public final Name Varargs;
	public final Name Annotation;
	public final Name RuntimeVisibleAnnotations;
	public final Name RuntimeInvisibleAnnotations;
	public final Name RuntimeVisibleTypeAnnotations;
	public final Name RuntimeInvisibleTypeAnnotations;
	public final Name RuntimeVisibleParameterAnnotations;
	public final Name RuntimeInvisibleParameterAnnotations;
	public final Name Value;
	public final Name EnclosingMethod;
	public final Name desiredAssertionStatus;
	public final Name append;
	public final Name family;
	public final Name forName;
	public final Name toString;
	public final Name length;
	public final Name valueOf;
	public final Name value;
	public final Name getMessage;
	public final Name getClass;
	public final Name TYPE;
	public final Name TYPE_USE;
	public final Name TYPE_PARAMETER;
	public final Name FIELD;
	public final Name METHOD;
	public final Name PARAMETER;
	public final Name CONSTRUCTOR;
	public final Name LOCAL_VARIABLE;
	public final Name ANNOTATION_TYPE;
	public final Name PACKAGE;
	public final Name SOURCE;
	public final Name CLASS;
	public final Name RUNTIME;
	public final Name Array;
	public final Name Method;
	public final Name Bound;
	public final Name clone;
	public final Name getComponentType;
	public final Name getClassLoader;
	public final Name initCause;
	public final Name values;
	public final Name iterator;
	public final Name hasNext;
	public final Name next;
	public final Name AnnotationDefault;
	public final Name ordinal;
	public final Name equals;
	public final Name hashCode;
	public final Name compareTo;
	public final Name getDeclaringClass;
	public final Name ex;
	public final Name finalize;
	public final Name java_lang_AutoCloseable;
	public final Name close;
	public final Name addSuppressed;

	public final Name.Table table;

	public Names(Context context) {
		Options options = Options.instance(context);
		table = createTable(options);

		slash = fromString("/");
		hyphen = fromString("-");
		T = fromString("T");
		slashequals = fromString("/=");
		deprecated = fromString("deprecated");

		init = fromString("<init>");
		clinit = fromString("<clinit>");
		error = fromString("<error>");
		any = fromString("<any>");
		empty = fromString("");
		one = fromString("1");
		period = fromString(".");
		comma = fromString(",");
		semicolon = fromString(";");
		asterisk = fromString("*");
		_this = fromString("this");
		_super = fromString("super");
		_default = fromString("default");

		_class = fromString("class");
		java_lang = fromString("java.lang");
		java_lang_Object = fromString("java.lang.Object");
		java_lang_Class = fromString("java.lang.Class");
		java_lang_Cloneable = fromString("java.lang.Cloneable");
		java_io_Serializable = fromString("java.io.Serializable");
		java_lang_Enum = fromString("java.lang.Enum");
		java_lang_invoke_MethodHandle = fromString("java.lang.invoke.MethodHandle");
		package_info = fromString("package-info");
		serialVersionUID = fromString("serialVersionUID");
		ConstantValue = fromString("ConstantValue");
		LineNumberTable = fromString("LineNumberTable");
		LocalVariableTable = fromString("LocalVariableTable");
		LocalVariableTypeTable = fromString("LocalVariableTypeTable");
		CharacterRangeTable = fromString("CharacterRangeTable");
		StackMap = fromString("StackMap");
		StackMapTable = fromString("StackMapTable");
		SourceID = fromString("SourceID");
		CompilationID = fromString("CompilationID");
		Code = fromString("Code");
		Exceptions = fromString("Exceptions");
		SourceFile = fromString("SourceFile");
		InnerClasses = fromString("InnerClasses");
		Synthetic = fromString("Synthetic");
		Bridge = fromString("Bridge");
		Deprecated = fromString("Deprecated");
		Enum = fromString("Enum");
		_name = fromString("name");
		Signature = fromString("Signature");
		Varargs = fromString("Varargs");
		Annotation = fromString("Annotation");
		RuntimeVisibleAnnotations = fromString("RuntimeVisibleAnnotations");
		RuntimeInvisibleAnnotations = fromString("RuntimeInvisibleAnnotations");
		RuntimeVisibleTypeAnnotations = fromString("RuntimeVisibleTypeAnnotations");
		RuntimeInvisibleTypeAnnotations = fromString("RuntimeInvisibleTypeAnnotations");
		RuntimeVisibleParameterAnnotations = fromString("RuntimeVisibleParameterAnnotations");
		RuntimeInvisibleParameterAnnotations = fromString("RuntimeInvisibleParameterAnnotations");
		Value = fromString("Value");
		EnclosingMethod = fromString("EnclosingMethod");

		desiredAssertionStatus = fromString("desiredAssertionStatus");

		append = fromString("append");
		family = fromString("family");
		forName = fromString("forName");
		toString = fromString("toString");
		length = fromString("length");
		valueOf = fromString("valueOf");
		value = fromString("value");
		getMessage = fromString("getMessage");
		getClass = fromString("getClass");

		TYPE = fromString("TYPE");
		TYPE_USE = fromString("TYPE_USE");
		TYPE_PARAMETER = fromString("TYPE_PARAMETER");
		FIELD = fromString("FIELD");
		METHOD = fromString("METHOD");
		PARAMETER = fromString("PARAMETER");
		CONSTRUCTOR = fromString("CONSTRUCTOR");
		LOCAL_VARIABLE = fromString("LOCAL_VARIABLE");
		ANNOTATION_TYPE = fromString("ANNOTATION_TYPE");
		PACKAGE = fromString("PACKAGE");

		SOURCE = fromString("SOURCE");
		CLASS = fromString("CLASS");
		RUNTIME = fromString("RUNTIME");

		Array = fromString("Array");
		Method = fromString("Method");
		Bound = fromString("Bound");
		clone = fromString("clone");
		getComponentType = fromString("getComponentType");
		getClassLoader = fromString("getClassLoader");
		initCause = fromString("initCause");
		values = fromString("values");
		iterator = fromString("iterator");
		hasNext = fromString("hasNext");
		next = fromString("next");
		AnnotationDefault = fromString("AnnotationDefault");
		ordinal = fromString("ordinal");
		equals = fromString("equals");
		hashCode = fromString("hashCode");
		compareTo = fromString("compareTo");
		getDeclaringClass = fromString("getDeclaringClass");
		ex = fromString("ex");
		finalize = fromString("finalize");

		java_lang_AutoCloseable = fromString("java.lang.AutoCloseable");
		close = fromString("close");
		addSuppressed = fromString("addSuppressed");
	}

	/*---------------Keywords class------------------------*/
	public static final Context.Key<Keywords> keywordsKey = new Context.Key<Keywords>();

	public static Keywords instance(Context context) {
		Keywords instance = context.get(keywordsKey);
		if (instance == null)
			instance = new Keywords(context);
		return instance;
	}

	private final Names names;

	protected Keywords(Context context) {
		context.put(keywordsKey, this);
		names = Names.instance(context);

		for (Token t : Token.values()) {
			if (t.name != null)
				enterKeyword(t.name, t);
			else
				tokenName[t.ordinal()] = null;
		}

		key = new Token[maxKey + 1];
		for (int i = 0; i <= maxKey; i++)
			key[i] = IDENTIFIER;
		for (Token t : Token.values()) {
			if (t.name != null)
				key[tokenName[t.ordinal()].getIndex()] = t;
		}
	}

	public Token key(Name name) {
		return (name.getIndex() > maxKey) ? IDENTIFIER : key[name.getIndex()];
	}

	/**
	 * Keyword array. Maps name indices to Token.
	 */
	private final Token[] key;

	/**
	 * The number of the last entered keyword.
	 */
	private int maxKey = 0;

	/**
	 * The names of all tokens.
	 */
	private Name[] tokenName = new Name[Token.values().length];

	private void enterKeyword(String s, Token token) {
		Name n = names.fromString(s);
		tokenName[token.ordinal()] = n;
		if (n.getIndex() > maxKey)
			maxKey = n.getIndex();
	}

	/*-------------------Token enum-----------------------*/
	public enum Token implements Formattable {
		EOF, ERROR, IDENTIFIER, ABSTRACT("abstract"), ASSERT("assert"), 
		BOOLEAN("boolean"), BREAK("break"), BYTE("byte"), CASE("case"), 
		CATCH("catch"), CHAR("char"), CLASS("class"), CONST("const"), 
		CONTINUE("continue"), DEFAULT("default"), DO("do"), 
		DOUBLE("double"), ELSE("else"), ENUM("enum"), EXTENDS("extends"), 
		FINAL("final"), FINALLY("finally"), FLOAT("float"), FOR("for"), 
		GOTO("goto"), IF("if"), IMPLEMENTS("implements"), IMPORT("import"), 
		INSTANCEOF("instanceof"), INT("int"), INTERFACE("interface"), 
		LONG("long"), NATIVE("native"), NEW("new"), PACKAGE("package"), 
		PRIVATE("private"), PROTECTED("protected"), PUBLIC("public"), 
		RETURN("return"), SHORT("short"), STATIC("static"), 
		STRICTFP("strictfp"), SUPER("super"), SWITCH("switch"), 
		SYNCHRONIZED("synchronized"), THIS("this"), THROW("throw"), 
		THROWS("throws"), TRANSIENT("transient"), TRY("try"), 
		VOID("void"), VOLATILE("volatile"), WHILE("while"), INTLITERAL, 
		LONGLITERAL, FLOATLITERAL, DOUBLELITERAL, CHARLITERAL, 
		STRINGLITERAL, TRUE("true"), FALSE("false"), NULL("null"), 
		LPAREN("("), RPAREN(")"), LBRACE("{"), RBRACE("}"), LBRACKET("["), 
		RBRACKET("]"), SEMI(";"), COMMA(","), DOT("."), ELLIPSIS("..."), 
		EQ("="), GT(">"), LT("<"), BANG("!"), TILDE("~"), QUES("?"), 
		COLON(":"), EQEQ("=="), LTEQ("<="), GTEQ(">="), BANGEQ("!="), 
		AMPAMP("&&"), BARBAR("||"), PLUSPLUS("++"), SUBSUB("--"), 
		PLUS("+"), SUB("-"), STAR("*"), SLASH("/"), AMP("&"), BAR("|"), 
		CARET("^"), PERCENT("%"), LTLT("<<"), GTGT(">>"), GTGTGT(">>>"), 
		PLUSEQ("+="), SUBEQ("-="), STAREQ("*="), SLASHEQ("/="), 
		AMPEQ("&="), BAREQ("|="), CARETEQ("^="), PERCENTEQ("%="), 
		LTLTEQ("<<="), GTGTEQ(">>="), GTGTGTEQ(">>>="), MONKEYS_AT("@"), 
		CUSTOM;
	}

	/*----------------SharedNameTable class--------------------*/
	public class SharedNameTable extends Name.Table {

		/**
		 * The hash table for names.
		 */
		private NameImpl[] hashes;

		/**
		 * The number of filled bytes in `names'.
		 */
		private int nc = 0;

		@Override
		public Name fromChars(char[] cs, int start, int len) {
			int nc = this.nc;
			byte[] bytes = this.bytes;
			while (nc + len * 3 >= bytes.length) {
				// System.err.println("doubling name buffer of length " + names.length + " to fit " + len + " chars");//DEBUG
				byte[] newnames = new byte[bytes.length * 2];
				System.arraycopy(bytes, 0, newnames, 0, bytes.length);
				bytes = this.bytes = newnames;
			}
			int nbytes = Convert.chars2utf(cs, start, bytes, nc, len) - nc;
			int h = hashValue(bytes, nc, nbytes) & hashMask;
			NameImpl n = hashes[h];
			while (n != null && (n.getByteLength() != nbytes || !equals(bytes, n.index, bytes, nc, nbytes))) {
				n = n.next;
			}
			if (n == null) {
				n = new NameImpl(this);
				n.index = nc;/*---this is the index of name---*/
				n.length = nbytes;
				n.next = hashes[h];
				hashes[h] = n;/*---save the name in hashes array---*/
				this.nc = nc + nbytes;
				if (nbytes == 0) {
					this.nc++;/*---index increase by 1---*/
				}
			}
			return n;
		}

		static class NameImpl extends Name {
			int index;

			@Override
			public int getIndex() {
				return index;
			}

		}

	}

}

转载地址：http://wenku.baidu.com/view/6f5fa9718e9951e79b892736.html
http://www.docjar.com/html/api/com/sun/tools/javac/parser/ScannerFactory.java.html
http://www.docjar.com/html/api/com/sun/tools/javac/parser/Scanner.java.html
http://www.docjar.com/html/api/com/sun/tools/javac/parser/Keywords.java.html
http://www.docjar.com/html/api/com/sun/tools/javac/util/Names.java.html
http://www.docjar.com/html/api/com/sun/tools/javac/parser/Token.java.html
http://www.docjar.com/html/api/com/sun/tools/javac/util/SharedNameTable.java.html