C++ Program

Write a tokenizer. The tokenizer should input a stream of ASCII characters and output the token, its line number in the stream, its type, and its value. You must write the code from scratch without the use of any lexicographical parsing libraries or utilities.

Tokenizer should recognize the following:

1. A few keywords: "if", "else", "for", "while"

2. A few single-character symbols: '&', '|', '+', '*', ':', ';'

3. Labels (alpha-numerical strings)

4. Integers (including negative integers)

5. Floating-point numbers in radix notation (such as pi, i.e. 3.14159265)

6. Floating-point numbers in exponential notation (such as Avogadro's number, i.e. 6.022140857E23)

Max 100 lines of code and I/O example, I/O has to be read from file!

10 // 2: word (lowercase only) 3: number (unsigned decimal integer) 12 #define STATEERROR #define STATEWHITESPACE 1 #define STATE WORD #define STATE NUMBER - 14 - 2 18 char szwhiteSpacel]-" Itrin" 19 char szword[]="abcdefghijklmnopqrstuvwxyz"; 20 char szNumber[]- "9123456789"; 21 char *szStates[]-"ERROR", "WHITESPACE", "WORD", "NUMBER" 23 int main(void){ char C 25 char szToken [256]; 26 27 int nTokenSize-0; int nChars-0, nTokens-0, nLines-1; int nCurState-STATE ERROR, nNextState-STATE ERROR; while((c-getc(stdin))) 32 if(c--EOF) nChars++; : break; 34 : if(strchr (szWord, c)) nNextstate-STATE_WORD; else if(strchr (szNumber, c)) nNextState-STATE_NUMBER; else nNextstate-STATE ERROR if(nChars-1) 37 : else if (strchr(szwhiteSpace, c)) nNextState-STATE WHITESPACE; 40 : nCurState-nNextState; // uncomment the following line to debug [Xo2X) 42 43 // printf("line %4d, char %4d : %c (%d-> %d) ", nLines, nChars, c, c, nCurState, nNextState); if(nNextstate-nCurState) 45 46 szToken[nTokenSize++]-c; if(c=-' ') continue; | nLines++, 48 49 50 52 : :: if((nCurstate-STATE-WORD)!! (nCurState-STATE line NUMBER)) ++nTokens, printf("token %2d, %2d : %10s (%s) ", nLines, szToken, szStates [nCurState)); 56 57 58 nTokenSize-0; szToken[nTokensize++]=c; nCurState-nNextState if(c=-' ') :: : : nLines++; 60 62 return(nTokens); 10 // 2: word (lowercase only) 3: number (unsigned decimal integer) 12 #define STATEERROR #define STATEWHITESPACE 1 #define STATE WORD #define STATE NUMBER - 14 - 2 18 char szwhiteSpacel]-" Itrin" 19 char szword[]="abcdefghijklmnopqrstuvwxyz"; 20 char szNumber[]- "9123456789"; 21 char *szStates[]-"ERROR", "WHITESPACE", "WORD", "NUMBER" 23 int main(void){ char C 25 char szToken [256]; 26 27 int nTokenSize-0; int nChars-0, nTokens-0, nLines-1; int nCurState-STATE ERROR, nNextState-STATE ERROR; while((c-getc(stdin))) 32 if(c--EOF) nChars++; : break; 34 : if(strchr (szWord, c)) nNextstate-STATE_WORD; else if(strchr (szNumber, c)) nNextState-STATE_NUMBER; else nNextstate-STATE ERROR if(nChars-1) 37 : else if (strchr(szwhiteSpace, c)) nNextState-STATE WHITESPACE; 40 : nCurState-nNextState; // uncomment the following line to debug [Xo2X) 42 43 // printf("line %4d, char %4d : %c (%d-> %d) ", nLines, nChars, c, c, nCurState, nNextState); if(nNextstate-nCurState) 45 46 szToken[nTokenSize++]-c; if(c=-' ') continue; | nLines++, 48 49 50 52 : :: if((nCurstate-STATE-WORD)!! (nCurState-STATE line NUMBER)) ++nTokens, printf("token %2d, %2d : %10s (%s) ", nLines, szToken, szStates [nCurState)); 56 57 58 nTokenSize-0; szToken[nTokensize++]=c; nCurState-nNextState if(c=-' ') :: : : nLines++; 60 62 return(nTokens)