beagle r4526 - in branches/beagle-lucene2_1/beagled/Lucene.Net: . Analysis/Standard



Author: dbera
Date: Sat Feb 23 06:42:55 2008
New Revision: 4526
URL: http://svn.gnome.org/viewvc/beagle?rev=4526&view=rev

Log:
Backport the JFlex based StandardAnalyzer from https://issues.apache.org/jira/browse/LUCENE-966


Added:
   branches/beagle-lucene2_1/beagled/Lucene.Net/Analysis/Standard/StandardTokenizerImpl.cs   (contents, props changed)
Removed:
   branches/beagle-lucene2_1/beagled/Lucene.Net/Analysis/Standard/CharStream.cs
   branches/beagle-lucene2_1/beagled/Lucene.Net/Analysis/Standard/FastCharStream.cs
   branches/beagle-lucene2_1/beagled/Lucene.Net/Analysis/Standard/ParseException.cs
   branches/beagle-lucene2_1/beagled/Lucene.Net/Analysis/Standard/StandardTokenizer.jj
   branches/beagle-lucene2_1/beagled/Lucene.Net/Analysis/Standard/StandardTokenizerConstants.cs
   branches/beagle-lucene2_1/beagled/Lucene.Net/Analysis/Standard/StandardTokenizerTokenManager.cs
   branches/beagle-lucene2_1/beagled/Lucene.Net/Analysis/Standard/Token.cs
   branches/beagle-lucene2_1/beagled/Lucene.Net/Analysis/Standard/TokenMgrError.cs
Modified:
   branches/beagle-lucene2_1/beagled/Lucene.Net/Analysis/Standard/StandardFilter.cs
   branches/beagle-lucene2_1/beagled/Lucene.Net/Analysis/Standard/StandardTokenizer.cs
   branches/beagle-lucene2_1/beagled/Lucene.Net/Makefile.include

Modified: branches/beagle-lucene2_1/beagled/Lucene.Net/Analysis/Standard/StandardFilter.cs
==============================================================================
--- branches/beagle-lucene2_1/beagled/Lucene.Net/Analysis/Standard/StandardFilter.cs	(original)
+++ branches/beagle-lucene2_1/beagled/Lucene.Net/Analysis/Standard/StandardFilter.cs	Sat Feb 23 06:42:55 2008
@@ -32,8 +32,8 @@
         {
         }
 		
-        private static readonly System.String APOSTROPHE_TYPE = Lucene.Net.Analysis.Standard.StandardTokenizerConstants.tokenImage[Lucene.Net.Analysis.Standard.StandardTokenizerConstants.APOSTROPHE];
-        private static readonly System.String ACRONYM_TYPE = Lucene.Net.Analysis.Standard.StandardTokenizerConstants.tokenImage[Lucene.Net.Analysis.Standard.StandardTokenizerConstants.ACRONYM];
+        private static readonly System.String APOSTROPHE_TYPE = Lucene.Net.Analysis.Standard.StandardTokenizerImpl.TOKEN_TYPES[Lucene.Net.Analysis.Standard.StandardTokenizerImpl.APOSTROPHE];
+        private static readonly System.String ACRONYM_TYPE = Lucene.Net.Analysis.Standard.StandardTokenizerImpl.TOKEN_TYPES[Lucene.Net.Analysis.Standard.StandardTokenizerImpl.ACRONYM];
 		
         /// <summary>Returns the next token in the stream, or null at EOS.
         /// <p>Removes <tt>'s</tt> from the end of words.
@@ -71,4 +71,4 @@
             }
         }
     }
-}
\ No newline at end of file
+}

Modified: branches/beagle-lucene2_1/beagled/Lucene.Net/Analysis/Standard/StandardTokenizer.cs
==============================================================================
--- branches/beagle-lucene2_1/beagled/Lucene.Net/Analysis/Standard/StandardTokenizer.cs	(original)
+++ branches/beagle-lucene2_1/beagled/Lucene.Net/Analysis/Standard/StandardTokenizer.cs	Sat Feb 23 06:42:55 2008
@@ -40,10 +40,22 @@
     public class StandardTokenizer : Lucene.Net.Analysis.Tokenizer
     {
 		
+        private StandardTokenizerImpl scanner;
+
+	public const int ALPHANUM          = 0;
+	public const int APOSTROPHE        = 1;
+	public const int ACRONYM           = 2;
+	public const int COMPANY           = 3;
+	public const int EMAIL             = 4;
+	public const int HOST              = 5;
+	public const int NUM               = 6;
+	public const int CJ                = 7;
+	public const int ACRONYM_DEP       = 8; /* deprecated */
+
         /// <summary>Constructs a tokenizer for this Reader. </summary>
-        public StandardTokenizer(System.IO.TextReader reader) : this(new FastCharStream(reader))
+        public StandardTokenizer(System.IO.TextReader reader) : base(reader)
         {
-            this.input = reader;
+            this.scanner = new StandardTokenizerImpl(reader);
         }
 		
         /// <summary>Returns the next token in the stream, or null at EOS.
@@ -52,236 +64,25 @@
         /// </summary>
         public override Lucene.Net.Analysis.Token Next()
         {
-            Token token = null;
-            switch ((jj_ntk == - 1) ? Jj_ntk() : jj_ntk)
-            {
-				
-                case Lucene.Net.Analysis.Standard.StandardTokenizerConstants.ALPHANUM: 
-                    token = Jj_consume_token(Lucene.Net.Analysis.Standard.StandardTokenizerConstants.ALPHANUM);
-                    break;
-				
-                case Lucene.Net.Analysis.Standard.StandardTokenizerConstants.APOSTROPHE: 
-                    token = Jj_consume_token(Lucene.Net.Analysis.Standard.StandardTokenizerConstants.APOSTROPHE);
-                    break;
-				
-                case Lucene.Net.Analysis.Standard.StandardTokenizerConstants.ACRONYM: 
-                    token = Jj_consume_token(Lucene.Net.Analysis.Standard.StandardTokenizerConstants.ACRONYM);
-                    break;
-				
-                case Lucene.Net.Analysis.Standard.StandardTokenizerConstants.COMPANY: 
-                    token = Jj_consume_token(Lucene.Net.Analysis.Standard.StandardTokenizerConstants.COMPANY);
-                    break;
-				
-                case Lucene.Net.Analysis.Standard.StandardTokenizerConstants.EMAIL: 
-                    token = Jj_consume_token(Lucene.Net.Analysis.Standard.StandardTokenizerConstants.EMAIL);
-                    break;
-				
-                case Lucene.Net.Analysis.Standard.StandardTokenizerConstants.HOST: 
-                    token = Jj_consume_token(Lucene.Net.Analysis.Standard.StandardTokenizerConstants.HOST);
-                    break;
-				
-                case Lucene.Net.Analysis.Standard.StandardTokenizerConstants.NUM: 
-                    token = Jj_consume_token(Lucene.Net.Analysis.Standard.StandardTokenizerConstants.NUM);
-                    break;
-				
-                case Lucene.Net.Analysis.Standard.StandardTokenizerConstants.CJ: 
-                    token = Jj_consume_token(Lucene.Net.Analysis.Standard.StandardTokenizerConstants.CJ);
-                    break;
-				
-                case 0: 
-                    token = Jj_consume_token(0);
-                    break;
-				
-                default: 
-                    jj_la1[0] = jj_gen;
-                    Jj_consume_token(- 1);
-                    throw new ParseException();
-				
-            }
-            if (token.kind == Lucene.Net.Analysis.Standard.StandardTokenizerConstants.EOF)
-            {
-            {
-                if (true)
-                    return null;
-            }
-            }
-            else
-            {
-            {
-                if (true)
-                    return new Lucene.Net.Analysis.Token(token.image, token.beginColumn, token.endColumn, Lucene.Net.Analysis.Standard.StandardTokenizerConstants.tokenImage[token.kind]);
-            }
-            }
-            throw new System.ApplicationException("Missing return statement in function");
+	    int tokenType = scanner.GetNextToken();
+
+	    if (tokenType == StandardTokenizerImpl.YYEOF) {
+	        return null;
+	    }
+
+	    int startPosition = scanner.yychar();
+
+	    string tokenImage = scanner.yytext();
+	    return new Token(tokenImage, startPosition, startPosition
+	    	+ tokenImage.Length,
+	    	StandardTokenizerImpl.TOKEN_TYPES[tokenType]);
         }
 
         /// <summary>By default, closes the input Reader. </summary>
         public override void Close() 
         { 
-            token_source.Close(); 
+            scanner.yyclose(); 
             base.Close(); 
         }
-
-        public StandardTokenizerTokenManager token_source;
-        public Token token, jj_nt;
-        private int jj_ntk;
-        private int jj_gen;
-        private int[] jj_la1 = new int[1];
-        private static int[] jj_la1_0_Renamed_Field;
-        private static void  jj_la1_0()
-        {
-            jj_la1_0_Renamed_Field = new int[]{0x10ff};
-        }
-		
-        public StandardTokenizer(CharStream stream)
-        {
-            token_source = new StandardTokenizerTokenManager(stream);
-            token = new Token();
-            jj_ntk = - 1;
-            jj_gen = 0;
-            for (int i = 0; i < 1; i++)
-                jj_la1[i] = - 1;
-        }
-		
-        public virtual void  ReInit(CharStream stream)
-        {
-            token_source.ReInit(stream);
-            token = new Token();
-            jj_ntk = - 1;
-            jj_gen = 0;
-            for (int i = 0; i < 1; i++)
-                jj_la1[i] = - 1;
-        }
-		
-        public StandardTokenizer(StandardTokenizerTokenManager tm)
-        {
-            token_source = tm;
-            token = new Token();
-            jj_ntk = - 1;
-            jj_gen = 0;
-            for (int i = 0; i < 1; i++)
-                jj_la1[i] = - 1;
-        }
-		
-        public virtual void  ReInit(StandardTokenizerTokenManager tm)
-        {
-            token_source = tm;
-            token = new Token();
-            jj_ntk = - 1;
-            jj_gen = 0;
-            for (int i = 0; i < 1; i++)
-                jj_la1[i] = - 1;
-        }
-		
-        private Token Jj_consume_token(int kind)
-        {
-            Token oldToken;
-            if ((oldToken = token).next != null)
-                token = token.next;
-            else
-                token = token.next = token_source.GetNextToken();
-            jj_ntk = - 1;
-            if (token.kind == kind)
-            {
-                jj_gen++;
-                return token;
-            }
-            token = oldToken;
-            jj_kind = kind;
-            throw GenerateParseException();
-        }
-		
-        public Token GetNextToken()
-        {
-            if (token.next != null)
-                token = token.next;
-            else
-                token = token.next = token_source.GetNextToken();
-            jj_ntk = - 1;
-            jj_gen++;
-            return token;
-        }
-		
-        public Token GetToken(int index)
-        {
-            Token t = token;
-            for (int i = 0; i < index; i++)
-            {
-                if (t.next != null)
-                    t = t.next;
-                else
-                    t = t.next = token_source.GetNextToken();
-            }
-            return t;
-        }
-		
-        private int Jj_ntk()
-        {
-            if ((jj_nt = token.next) == null)
-                return (jj_ntk = (token.next = token_source.GetNextToken()).kind);
-            else
-                return (jj_ntk = jj_nt.kind);
-        }
-		
-        private System.Collections.ArrayList jj_expentries = System.Collections.ArrayList.Synchronized(new System.Collections.ArrayList(10));
-        private int[] jj_expentry;
-        private int jj_kind = - 1;
-		
-        public virtual ParseException GenerateParseException()
-        {
-            jj_expentries.Clear();
-            bool[] la1tokens = new bool[16];
-            for (int i = 0; i < 16; i++)
-            {
-                la1tokens[i] = false;
-            }
-            if (jj_kind >= 0)
-            {
-                la1tokens[jj_kind] = true;
-                jj_kind = - 1;
-            }
-            for (int i = 0; i < 1; i++)
-            {
-                if (jj_la1[i] == jj_gen)
-                {
-                    for (int j = 0; j < 32; j++)
-                    {
-                        if ((jj_la1_0_Renamed_Field[i] & (1 << j)) != 0)
-                        {
-                            la1tokens[j] = true;
-                        }
-                    }
-                }
-            }
-            for (int i = 0; i < 16; i++)
-            {
-                if (la1tokens[i])
-                {
-                    jj_expentry = new int[1];
-                    jj_expentry[0] = i;
-                    jj_expentries.Add(jj_expentry);
-                }
-            }
-            int[][] exptokseq = new int[jj_expentries.Count][];
-            for (int i = 0; i < jj_expentries.Count; i++)
-            {
-                exptokseq[i] = (int[]) jj_expentries[i];
-            }
-            return new ParseException(token, exptokseq, Lucene.Net.Analysis.Standard.StandardTokenizerConstants.tokenImage);
-        }
-		
-        public void  Enable_tracing()
-        {
-        }
-		
-        public void  Disable_tracing()
-        {
-        }
-        static StandardTokenizer()
-        {
-        {
-            jj_la1_0();
-        }
-        }
     }
-}
\ No newline at end of file
+}

Added: branches/beagle-lucene2_1/beagled/Lucene.Net/Analysis/Standard/StandardTokenizerImpl.cs
==============================================================================
--- (empty file)
+++ branches/beagle-lucene2_1/beagled/Lucene.Net/Analysis/Standard/StandardTokenizerImpl.cs	Sat Feb 23 06:42:55 2008
@@ -0,0 +1,680 @@
+ï/* The following code was generated by JFlex 1.4.1 on 1/25/08 6:54 PM */
+/// <summary> Licensed to the Apache Software Foundation (ASF) under one or more
+/// contributor license agreements.  See the NOTICE file distributed with
+/// this work for additional information regarding copyright ownership.
+/// The ASF licenses this file to You under the Apache License, Version 2.0
+/// (the "License"); you may not use this file except in compliance with
+/// the License.  You may obtain a copy of the License at
+/// 
+/// http://www.apache.org/licenses/LICENSE-2.0
+/// 
+/// Unless required by applicable law or agreed to in writing, software
+/// distributed under the License is distributed on an "AS IS" BASIS,
+/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+/// See the License for the specific language governing permissions and
+/// limitations under the License.
+/// </summary>
+using System;
+using Token = Lucene.Net.Analysis.Token;
+
+namespace Lucene.Net.Analysis.Standard
+{
+	
+	
+	/// <summary> This class is a scanner generated by 
+	/// <a href="http://www.jflex.de/";>JFlex</a> 1.4.1
+	/// on 1/25/08 6:54 PM from the specification file
+	/// <tt>/tango/mike/src/lucene.clean/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex</tt>
+	/// </summary>
+	public class StandardTokenizerImpl
+	{
+		/// <summary> Resumes scanning until the next regular expression is matched,
+		/// the end of input is encountered or an I/O-Error occurs.
+		/// 
+		/// </summary>
+		/// <returns>      the next token
+		/// </returns>
+		/// <exception cref="java.io.IOException"> if any I/O-Error occurs
+		/// </exception>
+		public int GetNextToken()
+		{
+			int zzInput;
+			int zzAction;
+			
+			// cached fields:
+			int zzCurrentPosL;
+			int zzMarkedPosL;
+			int zzEndReadL = zzEndRead;
+			char[] zzBufferL = zzBuffer;
+			char[] zzCMapL = ZZ_CMAP;
+			
+			int[] zzTransL = ZZ_TRANS;
+			int[] zzRowMapL = ZZ_ROWMAP;
+			int[] zzAttrL = ZZ_ATTRIBUTE;
+			
+			while (true)
+			{
+				zzMarkedPosL = zzMarkedPos;
+				
+				yychar_Renamed_Field += zzMarkedPosL - zzStartRead;
+				
+				zzAction = - 1;
+				
+				zzCurrentPosL = zzCurrentPos = zzStartRead = zzMarkedPosL;
+				
+				zzState = zzLexicalState;
+				
+				
+				{
+					while (true)
+					{
+						
+						if (zzCurrentPosL < zzEndReadL)
+							zzInput = zzBufferL[zzCurrentPosL++];
+						else if (zzAtEOF)
+						{
+							zzInput = YYEOF;
+							//UPGRADE_NOTE: Labeled break statement was changed to a goto statement. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1012'"
+							goto zzForAction_brk;
+						}
+						else
+						{
+							// store back cached positions
+							zzCurrentPos = zzCurrentPosL;
+							zzMarkedPos = zzMarkedPosL;
+							bool eof = zzRefill();
+							// get translated positions and possibly new buffer
+							zzCurrentPosL = zzCurrentPos;
+							zzMarkedPosL = zzMarkedPos;
+							zzBufferL = zzBuffer;
+							zzEndReadL = zzEndRead;
+							if (eof)
+							{
+								zzInput = YYEOF;
+								//UPGRADE_NOTE: Labeled break statement was changed to a goto statement. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1012'"
+								goto zzForAction_brk;
+							}
+							else
+							{
+								zzInput = zzBufferL[zzCurrentPosL++];
+							}
+						}
+						int zzNext = zzTransL[zzRowMapL[zzState] + zzCMapL[zzInput]];
+						if (zzNext == - 1)
+						{
+							//UPGRADE_NOTE: Labeled break statement was changed to a goto statement. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1012'"
+							goto zzForAction_brk;
+						}
+						zzState = zzNext;
+						
+						int zzAttributes = zzAttrL[zzState];
+						if ((zzAttributes & 1) == 1)
+						{
+							zzAction = zzState;
+							zzMarkedPosL = zzCurrentPosL;
+							if ((zzAttributes & 8) == 8)
+							{
+								//UPGRADE_NOTE: Labeled break statement was changed to a goto statement. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1012'"
+								goto zzForAction_brk;
+							}
+						}
+					}
+				}
+				//UPGRADE_NOTE: Label 'zzForAction_brk' was added. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1011'"
+
+zzForAction_brk: ;
+				
+				
+				// store back cached position
+				zzMarkedPos = zzMarkedPosL;
+				
+				switch (zzAction < 0?zzAction:ZZ_ACTION[zzAction])
+				{
+					
+					case 5: 
+					{
+						return HOST;
+					}
+					
+					case 11:  break;
+					
+					case 9: 
+					{
+						return ACRONYM_DEP;
+					}
+					
+					case 12:  break;
+					
+					case 8: 
+					{
+						return ACRONYM;
+					}
+					
+					case 13:  break;
+					
+					case 1: 
+						{
+							/* ignore */
+						}
+						goto case 14;
+					
+					case 14:  break;
+					
+					case 7: 
+					{
+						return NUM;
+					}
+					
+					case 15:  break;
+					
+					case 3: 
+					{
+						return CJ;
+					}
+					
+					case 16:  break;
+					
+					case 2: 
+					{
+						return ALPHANUM;
+					}
+					
+					case 17:  break;
+					
+					case 6: 
+					{
+						return COMPANY;
+					}
+					
+					case 18:  break;
+					
+					case 4: 
+					{
+						return APOSTROPHE;
+					}
+					
+					case 19:  break;
+					
+					case 10: 
+					{
+						return EMAIL;
+					}
+					
+					case 20:  break;
+					
+					default: 
+						if (zzInput == YYEOF && zzStartRead == zzCurrentPos)
+						{
+							zzAtEOF = true;
+							return YYEOF;
+						}
+						else
+						{
+							zzScanError(ZZ_NO_MATCH);
+						}
+						break;
+					
+				}
+			}
+		}
+		
+		/// <summary>This character denotes the end of file </summary>
+		public const int YYEOF = - 1;
+		
+		/// <summary>initial size of the lookahead buffer </summary>
+		private const int ZZ_BUFFERSIZE = 16384;
+		
+		/// <summary>lexical states </summary>
+		public const int YYINITIAL = 0;
+		
+		/// <summary> Translates characters to character classes</summary>
+		private const System.String ZZ_CMAP_PACKED = "\x0009\x0000\x0001\x0000\x0001\x000E\x0001\x0000\x0001\x0000\x0001\x000D\x0012\x0000\x0001\x0000\x0005\x0000\x0001\x0003" + "\x0001\x0001\x0004\x0000\x0001\x0007\x0001\x0005\x0001\x0002\x0001\x0007\x000A\x0009\x0006\x0000\x0001\x0004\x001A\x0008" + "\x0004\x0000\x0001\x0006\x0001\x0000\x001A\x0008\x0045\x0000\x0017\x0008\x0001\x0000\x001F\x0008\x0001\x0000\u0568\x0008" + "\x000A\x000A\x0086\x0008\x000A\x000A\u026c\x0008\x000A\x000A\x0076\x0008\x000A\x000A\x0076\x0008\x000A\x000A\x0076\x0008" + "\x000A\x000A\x0076\x0008\x000A\x000A\x0077\x0008\x0009\x000A\x0076\x0008\x000A\x000A\x0076\x0008\x000A\x000A\x0076\x0008" + "\x000A\x000A\x00E0\x0008\x000A\x000A\x0076\x0008\x000A\x000A\u0166\x0008\x000A\x000A\x00B6\x0008\u0100\x0008\u0e00\x0008" + "\u1040\x0000\u0150\x000C\x0060\x0000\x0010\x000C\u0100\x0000\x0080\x000C\x0080\x0000\u19c0\x000C\x0040\x0000\u5200\x000C" + "\u0c00\x0000\u2bb0\x000B\u2150\x0000\u0200\x000C\u0465\x0000\x003B
 \x000C\x003D\x0008\x0023\x0000";
+		
+		/// <summary> Translates characters to character classes</summary>
+		//UPGRADE_NOTE: Final was removed from the declaration of 'ZZ_CMAP '. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1003'"
+		private static readonly char[] ZZ_CMAP = zzUnpackCMap(ZZ_CMAP_PACKED);
+		
+		/// <summary> Translates DFA states to action switch labels.</summary>
+		//UPGRADE_NOTE: Final was removed from the declaration of 'ZZ_ACTION '. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1003'"
+		private static readonly int[] ZZ_ACTION = zzUnpackAction();
+		
+		private const System.String ZZ_ACTION_PACKED_0 = "\x0001\x0000\x0001\x0001\x0004\x0002\x0001\x0003\x0001\x0001\x0006\x0000\x0002\x0002\x0006\x0000" + "\x0001\x0004\x0004\x0005\x0002\x0006\x0002\x0000\x0001\x0007\x0001\x0000\x0001\x0007\x0003\x0005" + "\x0006\x0007\x0003\x0005\x0001\x0008\x0001\x0000\x0001\x0009\x0002\x0000\x0001\x0008\x0001\x0009" + "\x0001\x0000\x0002\x0009\x0002\x0008\x0002\x0005\x0001\x000A";
+		
+		private static int[] zzUnpackAction()
+		{
+			int[] result = new int[61];
+			int offset = 0;
+			offset = zzUnpackAction(ZZ_ACTION_PACKED_0, offset, result);
+			return result;
+		}
+		
+		private static int zzUnpackAction(System.String packed, int offset, int[] result)
+		{
+			int i = 0; /* index in packed string  */
+			int j = offset; /* index in unpacked array */
+			int l = packed.Length;
+			while (i < l)
+			{
+				int count = packed[i++];
+				int value_Renamed = packed[i++];
+				do 
+					result[j++] = value_Renamed;
+				while (--count > 0);
+			}
+			return j;
+		}
+		
+		
+		/// <summary> Translates a state to a row index in the transition table</summary>
+		//UPGRADE_NOTE: Final was removed from the declaration of 'ZZ_ROWMAP '. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1003'"
+		private static readonly int[] ZZ_ROWMAP = zzUnpackRowMap();
+		
+		private const System.String ZZ_ROWMAP_PACKED_0 = "\x0000\x0000\x0000\x000F\x0000\x001E\x0000\x002D\x0000\x003C\x0000\x004B\x0000\x000F\x0000\x005A" + "\x0000\x0069\x0000\x0078\x0000\x0087\x0000\x0096\x0000\x00A5\x0000\x00B4\x0000\x00C3\x0000\x00D2" + "\x0000\x00E1\x0000\x00F0\x0000\x00FF\x0000\u010e\x0000\u011d\x0000\u012c\x0000\u013b\x0000\u014a" + "\x0000\u0159\x0000\u0168\x0000\u0177\x0000\x0087\x0000\u0186\x0000\u0195\x0000\u01a4\x0000\u01b3" + "\x0000\u01c2\x0000\u01d1\x0000\u01e0\x0000\u01ef\x0000\u01fe\x0000\u020d\x0000\u021c\x0000\u022b" + "\x0000\u023a\x0000\u0249\x0000\u0258\x0000\u0267\x0000\u0276\x0000\u0285\x0000\u0294\x0000\u02a3" + "\x0000\u02b2\x0000\u02c1\x0000\u02d0\x0000\u02df\x0000\u02ee\x0000\u02fd\x0000\u012c\x0000\x00E1" + "\x0000\x0078\x0000\u011d\x0000\u030c\x0000\u031b\x0000\u032a";
+		
+		private static int[] zzUnpackRowMap()
+		{
+			int[] result = new int[61];
+			int offset = 0;
+			offset = zzUnpackRowMap(ZZ_ROWMAP_PACKED_0, offset, result);
+			return result;
+		}
+		
+		private static int zzUnpackRowMap(System.String packed, int offset, int[] result)
+		{
+			int i = 0; /* index in packed string  */
+			int j = offset; /* index in unpacked array */
+			int l = packed.Length;
+			while (i < l)
+			{
+				int high = packed[i++] << 16;
+				result[j++] = high | packed[i++];
+			}
+			return j;
+		}
+		
+		/// <summary> The transition table of the DFA</summary>
+		//UPGRADE_NOTE: Final was removed from the declaration of 'ZZ_TRANS '. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1003'"
+		private static readonly int[] ZZ_TRANS = zzUnpackTrans();
+		
+		private const System.String ZZ_TRANS_PACKED_0 = "\x0008\x0002\x0001\x0003\x0001\x0004\x0001\x0005\x0001\x0006\x0001\x0007\x0001\x0008\x0001\x0002" + "\x0010\x0000\x0001\x0009\x0001\x000A\x0001\x000B\x0001\x000C\x0002\x000D\x0001\x000E\x0001\x000F" + "\x0001\x0004\x0001\x0010\x0001\x0006\x0005\x0000\x0001\x0011\x0001\x0000\x0001\x0012\x0002\x0013" + "\x0001\x0014\x0003\x0004\x0001\x0006\x0004\x0000\x0001\x0009\x0001\x0015\x0001\x000B\x0001\x000C" + "\x0002\x0013\x0001\x0014\x0001\x0010\x0001\x0004\x0001\x0010\x0001\x0006\x0005\x0000\x0001\x0016" + "\x0001\x0000\x0001\x0012\x0002\x000D\x0001\x000E\x0004\x0006\x0011\x0000\x0001\x0002\x0008\x0000" + "\x0001\x0017\x0001\x0000\x0001\x0017\x000C\x0000\x0001\x0018\x0001\x0019\x0001\x001A\x0001\x001B" + "\x000B\x0000\x0001\x001C\x0001\x0000\x0001\x001C\x000C\x0000\x0001\x001D\x0001\x001E\x0001\x001D" + "\x0001\x001E\x000B\x0000\x0001\x001F\x0002\x0020\x0001\x0021\x000B\x0000\x0001\x000E\x0002\x0022" + "\x0005\x0000\x0001\x0009\x000
 1\x0016\x0001\x000B\x0001\x000C\x0002\x000D\x0001\x000E\x0001\x000F" + "\x0001\x0004\x0001\x0010\x0001\x0006\x0004\x0000\x0001\x0009\x0001\x0011\x0001\x000B\x0001\x000C" + "\x0002\x0013\x0001\x0014\x0001\x0010\x0001\x0004\x0001\x0010\x0001\x0006\x000B\x0000\x0001\x0023" + "\x0002\x0024\x0001\x0025\x000B\x0000\x0004\x001E\x000B\x0000\x0001\x0026\x0002\x0027\x0001\x0028" + "\x000B\x0000\x0001\x0029\x0002\x002A\x0001\x002B\x000B\x0000\x0001\x002C\x0001\x0024\x0001\x002D" + "\x0001\x0025\x000B\x0000\x0001\x002E\x0002\x0019\x0001\x001B\x0004\x0000\x0001\x0009\x0006\x0000" + "\x0001\x0017\x0001\x0000\x0001\x0017\x0006\x0000\x0001\x002F\x0001\x0000\x0001\x0012\x0002\x0030" + "\x0001\x0000\x0001\x002E\x0002\x0019\x0001\x001B\x0005\x0000\x0001\x0031\x0001\x0000\x0001\x0012" + "\x0002\x0032\x0001\x0033\x0003\x0019\x0001\x001B\x0005\x0000\x0001\x0034\x0001\x0000\x0001\x0012" + "\x0002\x0032\x0001\x0033\x0003\x0019\x0001\x001B\x0005\x0000\x0001\x0035\x0001\x0000\x0001\x0012" + 
+			"\x0002\x0030\x0001\x0000\x0004\x001B\x0005\x0000\x0001\x0036\x0002\x0000\x0001\x0036\x0002\x0000" + "\x0001\x001D\x0001\x001E\x0001\x001D\x0001\x001E\x0005\x0000\x0001\x0036\x0002\x0000\x0001\x0036" + "\x0002\x0000\x0004\x001E\x0005\x0000\x0001\x0030\x0001\x0000\x0001\x0012\x0002\x0030\x0001\x0000" + "\x0001\x001F\x0002\x0020\x0001\x0021\x0005\x0000\x0001\x0032\x0001\x0000\x0001\x0012\x0002\x0032" + "\x0001\x0033\x0003\x0020\x0001\x0021\x0005\x0000\x0001\x0030\x0001\x0000\x0001\x0012\x0002\x0030" + "\x0001\x0000\x0004\x0021\x0005\x0000\x0001\x0033\x0002\x0000\x0003\x0033\x0003\x0022\x0006\x0000" + "\x0001\x0037\x0001\x0000\x0001\x0012\x0002\x000D\x0001\x000E\x0001\x0023\x0002\x0024\x0001\x0025" + "\x0005\x0000\x0001\x0038\x0001\x0000\x0001\x0012\x0002\x0013\x0001\x0014\x0003\x0024\x0001\x0025" + "\x0005\x0000\x0001\x0037\x0001\x0000\x0001\x0012\x0002\x000D\x0001\x000E\x0004\x0025\x0005\x0000" + "\x0001\x000D\x0001\x0000\x0001\x0012\x0002\x000D\x0001\x000E\x0001\x0026\x00
 02\x0027\x0001\x0028" + "\x0005\x0000\x0001\x0013\x0001\x0000\x0001\x0012\x0002\x0013\x0001\x0014\x0003\x0027\x0001\x0028" + "\x0005\x0000\x0001\x000D\x0001\x0000\x0001\x0012\x0002\x000D\x0001\x000E\x0004\x0028\x0005\x0000" + "\x0001\x000E\x0002\x0000\x0003\x000E\x0001\x0029\x0002\x002A\x0001\x002B\x0005\x0000\x0001\x0014" + "\x0002\x0000\x0003\x0014\x0003\x002A\x0001\x002B\x0005\x0000\x0001\x000E\x0002\x0000\x0003\x000E" + "\x0004\x002B\x0005\x0000\x0001\x0039\x0001\x0000\x0001\x0012\x0002\x000D\x0001\x000E\x0001\x0023" + "\x0002\x0024\x0001\x0025\x0005\x0000\x0001\x003A\x0001\x0000\x0001\x0012\x0002\x0013\x0001\x0014" + "\x0003\x0024\x0001\x0025\x0005\x0000\x0001\x0035\x0001\x0000\x0001\x0012\x0002\x0030\x0001\x0000" + "\x0001\x002E\x0002\x0019\x0001\x001B\x000B\x0000\x0001\x003B\x0001\x001B\x0001\x003B\x0001\x001B" + "\x000B\x0000\x0004\x0021\x000B\x0000\x0004\x0025\x000B\x0000\x0004\x0028\x000B\x0000\x0004\x002B" + "\x000B\x0000\x0001\x003C\x0001\x0025\x0001\x003C\x0001\
 x0025\x000B\x0000\x0004\x001B\x000B\x0000" + 
+			"\x0004\x003D\x0005\x0000\x0001\x002F\x0001\x0000\x0001\x0012\x0002\x0030\x0001\x0000\x0004\x001B" + "\x0005\x0000\x0001\x0039\x0001\x0000\x0001\x0012\x0002\x000D\x0001\x000E\x0004\x0025\x0005\x0000" + "\x0001\x0036\x0002\x0000\x0001\x0036\x0002\x0000\x0004\x003D\x0003\x0000";
+		
+		private static int[] zzUnpackTrans()
+		{
+			int[] result = new int[825];
+			int offset = 0;
+			offset = zzUnpackTrans(ZZ_TRANS_PACKED_0, offset, result);
+			return result;
+		}
+		
+		private static int zzUnpackTrans(System.String packed, int offset, int[] result)
+		{
+			int i = 0; /* index in packed string  */
+			int j = offset; /* index in unpacked array */
+			int l = packed.Length;
+			while (i < l)
+			{
+				int count = packed[i++];
+				int value_Renamed = packed[i++];
+				value_Renamed--;
+				do 
+					result[j++] = value_Renamed;
+				while (--count > 0);
+			}
+			return j;
+		}
+		
+		
+		/* error codes */
+		private const int ZZ_UNKNOWN_ERROR = 0;
+		private const int ZZ_NO_MATCH = 1;
+		private const int ZZ_PUSHBACK_2BIG = 2;
+		
+		/* error messages for the codes above */
+		//UPGRADE_NOTE: Final was removed from the declaration of 'ZZ_ERROR_MSG'. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1003'"
+		private static readonly System.String[] ZZ_ERROR_MSG = new System.String[]{"Unkown internal scanner error", "Error: could not match input", "Error: pushback value was too large"};
+		
+		/// <summary> ZZ_ATTRIBUTE[aState] contains the attributes of state <code>aState</code></summary>
+		//UPGRADE_NOTE: Final was removed from the declaration of 'ZZ_ATTRIBUTE '. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1003'"
+		private static readonly int[] ZZ_ATTRIBUTE = zzUnpackAttribute();
+		
+		private const System.String ZZ_ATTRIBUTE_PACKED_0 = "\x0001\x0000\x0001\x0009\x0004\x0001\x0001\x0009\x0001\x0001\x0006\x0000\x0002\x0001\x0006\x0000" + "\x0007\x0001\x0002\x0000\x0001\x0001\x0001\x0000\x000E\x0001\x0001\x0000\x0001\x0001\x0002\x0000" + "\x0002\x0001\x0001\x0000\x0007\x0001";
+		
+		private static int[] zzUnpackAttribute()
+		{
+			int[] result = new int[61];
+			int offset = 0;
+			offset = zzUnpackAttribute(ZZ_ATTRIBUTE_PACKED_0, offset, result);
+			return result;
+		}
+		
+		private static int zzUnpackAttribute(System.String packed, int offset, int[] result)
+		{
+			int i = 0; /* index in packed string  */
+			int j = offset; /* index in unpacked array */
+			int l = packed.Length;
+			while (i < l)
+			{
+				int count = packed[i++];
+				int value_Renamed = packed[i++];
+				do 
+					result[j++] = value_Renamed;
+				while (--count > 0);
+			}
+			return j;
+		}
+		
+		/// <summary>the input device </summary>
+		//UPGRADE_ISSUE: Class hierarchy differences between 'java.io.Reader' and 'System.IO.StreamReader' may cause compilation errors. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1186'"
+		private System.IO.TextReader zzReader;
+		
+		/// <summary>the current state of the DFA </summary>
+		private int zzState;
+		
+		/// <summary>the current lexical state </summary>
+		private int zzLexicalState = YYINITIAL;
+		
+		/// <summary>this buffer contains the current text to be matched and is
+		/// the source of the yytext() string 
+		/// </summary>
+		private char[] zzBuffer = new char[ZZ_BUFFERSIZE];
+		
+		/// <summary>the textposition at the last accepting state </summary>
+		private int zzMarkedPos;
+		
+		/// <summary>the textposition at the last state to be included in yytext </summary>
+		private int zzPushbackPos;
+		
+		/// <summary>the current text position in the buffer </summary>
+		private int zzCurrentPos;
+		
+		/// <summary>startRead marks the beginning of the yytext() string in the buffer </summary>
+		private int zzStartRead;
+		
+		/// <summary>endRead marks the last character in the buffer, that has been read
+		/// from input 
+		/// </summary>
+		private int zzEndRead;
+		
+		/// <summary>number of newlines encountered up to the start of the matched text </summary>
+		private int yyline;
+		
+		/// <summary>the number of characters up to the start of the matched text </summary>
+		private int yychar_Renamed_Field;
+		
+		/// <summary> the number of characters from the last newline up to the start of the 
+		/// matched text
+		/// </summary>
+		private int yycolumn;
+		
+		/// <summary> zzAtBOL == true <=> the scanner is currently at the beginning of a line</summary>
+		private bool zzAtBOL = true;
+		
+		/// <summary>zzAtEOF == true <=> the scanner is at the EOF </summary>
+		private bool zzAtEOF;
+		
+		/* user code: */
+		
+		//UPGRADE_NOTE: Final was removed from the declaration of 'ALPHANUM '. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1003'"
+		public static readonly int ALPHANUM = StandardTokenizer.ALPHANUM;
+		//UPGRADE_NOTE: Final was removed from the declaration of 'APOSTROPHE '. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1003'"
+		public static readonly int APOSTROPHE = StandardTokenizer.APOSTROPHE;
+		//UPGRADE_NOTE: Final was removed from the declaration of 'ACRONYM '. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1003'"
+		public static readonly int ACRONYM = StandardTokenizer.ACRONYM;
+		//UPGRADE_NOTE: Final was removed from the declaration of 'COMPANY '. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1003'"
+		public static readonly int COMPANY = StandardTokenizer.COMPANY;
+		//UPGRADE_NOTE: Final was removed from the declaration of 'EMAIL '. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1003'"
+		public static readonly int EMAIL = StandardTokenizer.EMAIL;
+		//UPGRADE_NOTE: Final was removed from the declaration of 'HOST '. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1003'"
+		public static readonly int HOST = StandardTokenizer.HOST;
+		//UPGRADE_NOTE: Final was removed from the declaration of 'NUM '. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1003'"
+		public static readonly int NUM = StandardTokenizer.NUM;
+		//UPGRADE_NOTE: Final was removed from the declaration of 'CJ '. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1003'"
+		public static readonly int CJ = StandardTokenizer.CJ;
+		/// <deprecated> this solves a bug where HOSTs that end with '.' are identified
+		/// as ACRONYMs. It is deprecated and will be removed in the next
+		/// release.
+		/// </deprecated>
+		//UPGRADE_NOTE: Final was removed from the declaration of 'ACRONYM_DEP '. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1003'"
+		public static readonly int ACRONYM_DEP = StandardTokenizer.ACRONYM_DEP;
+		
+		//UPGRADE_NOTE: Final was removed from the declaration of 'TOKEN_TYPES '. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1003'"
+		public static readonly System.String[] TOKEN_TYPES = new System.String[]{"<ALPHANUM>", "<APOSTROPHE>", "<ACRONYM>", "<COMPANY>", "<EMAIL>", "<HOST>", "<NUM>", "<CJ>", "<ACRONYM_DEP>"};
+		
+		public int yychar()
+		{
+			return yychar_Renamed_Field;
+		}
+		
+		
+		/// <summary> Creates a new scanner
+		/// There is also a java.io.InputStream version of this constructor.
+		/// 
+		/// </summary>
+		/// <param name="in"> the java.io.Reader to read input from.
+		/// </param>
+		//UPGRADE_ISSUE: Class hierarchy differences between 'java.io.Reader' and 'System.IO.StreamReader' may cause compilation errors. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1186'"
+		internal StandardTokenizerImpl(System.IO.TextReader in_Renamed)
+		{
+			this.zzReader = in_Renamed;
+		}
+		
+		/// <summary> Creates a new scanner.
+		/// There is also java.io.Reader version of this constructor.
+		/// 
+		/// </summary>
+		/// <param name="in"> the java.io.Inputstream to read input from.
+		/// </param>
+		internal StandardTokenizerImpl(System.IO.Stream in_Renamed):this(new System.IO.StreamReader(in_Renamed, System.Text.Encoding.Default))
+		{
+		}
+		
+		/// <summary> Unpacks the compressed character translation table.
+		/// 
+		/// </summary>
+		/// <param name="packed">  the packed character translation table
+		/// </param>
+		/// <returns>         the unpacked character translation table
+		/// </returns>
+		private static char[] zzUnpackCMap(System.String packed)
+		{
+			char[] map = new char[0x10000];
+			int i = 0; /* index in packed string  */
+			int j = 0; /* index in unpacked array */
+			while (i < 156)
+			{
+				int count = packed[i++];
+				char value_Renamed = packed[i++];
+				do 
+					map[j++] = value_Renamed;
+				while (--count > 0);
+			}
+			return map;
+		}
+		
+		
+		/// <summary> Refills the input buffer.
+		/// 
+		/// </summary>
+		/// <returns>      <code>false</code>, iff there was new input.
+		/// 
+		/// </returns>
+		/// <exception cref="java.io.IOException"> if any I/O-Error occurs
+		/// </exception>
+		private bool zzRefill()
+		{
+			
+			/* first: make room (if you can) */
+			if (zzStartRead > 0)
+			{
+				Array.Copy(zzBuffer, zzStartRead, zzBuffer, 0, zzEndRead - zzStartRead);
+				
+				/* translate stored positions */
+				zzEndRead -= zzStartRead;
+				zzCurrentPos -= zzStartRead;
+				zzMarkedPos -= zzStartRead;
+				zzPushbackPos -= zzStartRead;
+				zzStartRead = 0;
+			}
+			
+			/* is the buffer big enough? */
+			if (zzCurrentPos >= zzBuffer.Length)
+			{
+				/* if not: blow it up */
+				char[] newBuffer = new char[zzCurrentPos * 2];
+				Array.Copy(zzBuffer, 0, newBuffer, 0, zzBuffer.Length);
+				zzBuffer = newBuffer;
+			}
+			
+			/* finally: fill the buffer with new input */
+			//UPGRADE_TODO: Method 'java.io.Reader.read' was converted to 'System.IO.StreamReader.Read' which has a different behavior. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1073_javaioReaderread_char[]_int_int'"
+			int lengthToRead = zzBuffer.Length - zzEndRead;
+			int numRead = zzReader.Read(zzBuffer, zzEndRead, lengthToRead);
+			
+			if (numRead == 0)
+			{
+				return true;
+			}
+			else
+			{
+				// dBera: set EOF is fewer chars were read
+				zzAtEOF = (numRead < lengthToRead);
+
+				zzEndRead += numRead;
+				return false;
+			}
+		}
+		
+		
+		/// <summary> Closes the input stream.</summary>
+		public void  yyclose()
+		{
+			zzAtEOF = true; /* indicate end of file */
+			zzEndRead = zzStartRead; /* invalidate buffer    */
+			
+			if (zzReader != null)
+				zzReader.Close();
+		}
+		
+		
+		/// <summary> Resets the scanner to read from a new input stream.
+		/// Does not close the old reader.
+		/// 
+		/// All internal variables are reset, the old input stream 
+		/// <b>cannot</b> be reused (internal buffer is discarded and lost).
+		/// Lexical state is set to <tt>ZZ_INITIAL</tt>.
+		/// 
+		/// </summary>
+		/// <param name="reader">  the new input stream 
+		/// </param>
+		//UPGRADE_ISSUE: Class hierarchy differences between 'java.io.Reader' and 'System.IO.StreamReader' may cause compilation errors. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1186'"
+		public void  yyreset(System.IO.StreamReader reader)
+		{
+			zzReader = reader;
+			zzAtBOL = true;
+			zzAtEOF = false;
+			zzEndRead = zzStartRead = 0;
+			zzCurrentPos = zzMarkedPos = zzPushbackPos = 0;
+			yyline = yychar_Renamed_Field = yycolumn = 0;
+			zzLexicalState = YYINITIAL;
+		}
+		
+		
+		/// <summary> Returns the current lexical state.</summary>
+		public int yystate()
+		{
+			return zzLexicalState;
+		}
+		
+		
+		/// <summary> Enters a new lexical state
+		/// 
+		/// </summary>
+		/// <param name="newState">the new lexical state
+		/// </param>
+		public void  yybegin(int newState)
+		{
+			zzLexicalState = newState;
+		}
+		
+		
+		/// <summary> Returns the text matched by the current regular expression.</summary>
+		public System.String yytext()
+		{
+			return new System.String(zzBuffer, zzStartRead, zzMarkedPos - zzStartRead);
+		}
+		
+		
+		/// <summary> Returns the character at position <tt>pos</tt> from the 
+		/// matched text. 
+		/// 
+		/// It is equivalent to yytext().charAt(pos), but faster
+		/// 
+		/// </summary>
+		/// <param name="pos">the position of the character to fetch. 
+		/// A value from 0 to yylength()-1.
+		/// 
+		/// </param>
+		/// <returns> the character at position pos
+		/// </returns>
+		public char yycharat(int pos)
+		{
+			return zzBuffer[zzStartRead + pos];
+		}
+		
+		
+		/// <summary> Returns the length of the matched text region.</summary>
+		public int yylength()
+		{
+			return zzMarkedPos - zzStartRead;
+		}
+		
+		
+		/// <summary> Reports an error that occured while scanning.
+		/// 
+		/// In a wellformed scanner (no or only correct usage of 
+		/// yypushback(int) and a match-all fallback rule) this method 
+		/// will only be called with things that "Can't Possibly Happen".
+		/// If this method is called, something is seriously wrong
+		/// (e.g. a JFlex bug producing a faulty scanner etc.).
+		/// 
+		/// Usual syntax/scanner level error handling should be done
+		/// in error fallback rules.
+		/// 
+		/// </summary>
+		/// <param name="errorCode"> the code of the errormessage to display
+		/// </param>
+		private void  zzScanError(int errorCode)
+		{
+			System.String message;
+			try
+			{
+				message = ZZ_ERROR_MSG[errorCode];
+			}
+			catch (System.IndexOutOfRangeException e)
+			{
+				message = ZZ_ERROR_MSG[ZZ_UNKNOWN_ERROR];
+			}
+			
+			throw new System.ApplicationException(message);
+		}
+		
+		
+		/// <summary> Pushes the specified amount of characters back into the input stream.
+		/// 
+		/// They will be read again by then next call of the scanning method
+		/// 
+		/// </summary>
+		/// <param name="number"> the number of characters to be read again.
+		/// This number must not be greater than yylength()!
+		/// </param>
+		public virtual void  yypushback(int number)
+		{
+			if (number > yylength())
+				zzScanError(ZZ_PUSHBACK_2BIG);
+			
+			zzMarkedPos -= number;
+		}
+	}
+}

Modified: branches/beagle-lucene2_1/beagled/Lucene.Net/Makefile.include
==============================================================================
--- branches/beagle-lucene2_1/beagled/Lucene.Net/Makefile.include	(original)
+++ branches/beagle-lucene2_1/beagled/Lucene.Net/Makefile.include	Sat Feb 23 06:42:55 2008
@@ -24,15 +24,9 @@
 	$(lucenedir)/Analysis/StopFilter.cs			\
 	$(lucenedir)/Analysis/StopAnalyzer.cs			\
 	$(lucenedir)/Analysis/Standard/StandardTokenizer.cs	\
-	$(lucenedir)/Analysis/Standard/Token.cs			\
-	$(lucenedir)/Analysis/Standard/TokenMgrError.cs		\
 	$(lucenedir)/Analysis/Standard/StandardAnalyzer.cs	\
 	$(lucenedir)/Analysis/Standard/StandardFilter.cs	\
-	$(lucenedir)/Analysis/Standard/FastCharStream.cs	\
-	$(lucenedir)/Analysis/Standard/StandardTokenizerTokenManager.cs		\
-	$(lucenedir)/Analysis/Standard/StandardTokenizerConstants.cs		\
-	$(lucenedir)/Analysis/Standard/CharStream.cs		\
-	$(lucenedir)/Analysis/Standard/ParseException.cs	\
+	$(lucenedir)/Analysis/Standard/StandardTokenizerImpl.cs	\
 	$(lucenedir)/Analysis/TokenStream.cs			\
 	$(lucenedir)/Analysis/WordlistLoader.cs			\
 	$(lucenedir)/Analysis/SimpleAnalyzer.cs			\



[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]