![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to lex.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [Research Unix] / researchv10dc / cmd / icon / src / tran|
Return to lex.c CVS log | Up to [Research Unix] / researchv10dc / cmd / icon / src / tran |
1.1 root 1: /*
2: * The lexical analyzer.
3: */
4:
5: #include "itran.h"
6: #include "token.h"
7: #include "lex.h"
8: #include "char.h"
9: #include "tree.h"
10:
11: int tline;
12: int tcol;
13:
14: /*
15: * yylex - find the next token in the input stream, and return its token
16: * type and value to the parser.
17: *
18: * Variables of interest:
19: *
20: * cc - character following last token.
21: * comflag - set if in a comment.
22: * nlflag - set if a newline was between the last token and the current token
23: * lastend - set if the last token was an Ender.
24: * lastval - when a semicolon is inserted and returned, lastval gets the
25: * token value that would have been returned if the semicolon hadn't
26: * been inserted.
27: */
28:
29: yylex()
30: {
31: register struct toktab *t;
32: register int c;
33: int nlflag;
34: int comflag;
35: static struct toktab *lasttok = NULL;
36: static nodeptr lastval;
37: static int lastend = 0;
38: static int eofflag = 0;
39: static int lastline = 0;
40: static int cc = '\n';
41: extern struct toktab *getident(), *getnum(), *getstring(), *getop();
42:
43: if (lasttok != NULL) {
44: /*
45: * A semicolon was inserted and returned on the last call to yylex,
46: * instead of going to the input, return lasttok and set the
47: * appropriate variables.
48: */
49: yylval = lastval;
50: tline = Line(lastval);
51: tcol = Col(lastval);
52: t = lasttok;
53: goto ret;
54: }
55: nlflag = 0;
56: comflag = 0;
57: loop:
58: c = cc;
59: /*
60: * Skip whitespace and comments.
61: */
62: while (c != EOF && (comflag || c == Comment || isspace(c))) {
63: if (c == '\n') {
64: nlflag++;
65: comflag = 0;
66: }
67: else if (c == Comment)
68: comflag++;
69: c = NextChar;
70: }
71: /*
72: * A token is the next thing in the input. Record the last line number
73: * and set tline and tcol to the current line and column.
74: */
75: lastline = tline;
76: tline = inline;
77: tcol = incol;
78:
79: if (c == EOF) {
80: /*
81: * End of file has been reached. Set eofflag, return T_Eof, and
82: * set cc to EOF so that any subsequent scans also return T_Eof.
83: */
84: if (eofflag++) {
85: eofflag = 0;
86: cc = '\n';
87: return (int) (yylval = 0);
88: }
89: cc = EOF;
90: t = T_Eof;
91: yylval = 0;
92: goto ret;
93: }
94:
95: /*
96: * Look at current input character to determine what class of token
97: * is next and take the appropriate action. Note that the various
98: * token gathering routines write a value into cc.
99: */
100: c = ctran[c];
101: if (isalpha(c)) { /* gather ident or reserved word */
102: if ((t = getident(c, &cc)) == NULL)
103: goto loop;
104: }
105: else if (isdigit(c)) { /* gather numeric literal */
106: if ((t = getnum(c, &cc)) == NULL)
107: goto loop;
108: }
109: else if (c == '"' || c == '\'') { /* gather string or cset literal */
110: if ((t = getstring(c, &cc)) == NULL)
111: goto loop;
112: }
113: else { /* gather longest legal operator */
114: if ((t = getop(c, &cc)) == NULL)
115: goto loop;
116: yylval = OpNode(t->t_type);
117: }
118: if (nlflag && lastend && (t->t_flags & Beginner)) {
119: /*
120: * A newline was encountered between the current token and the last,
121: * the last token was an Ender, and the current token is a Beginner.
122: * Return a semicolon and save the current token in lastval.
123: */
124: lastval = yylval;
125: lasttok = t;
126: tline = lastline;
127: tcol = 0;
128: yylval = OpNode(SEMICOL);
129: return SEMICOL;
130: }
131: ret:
132: /*
133: * Clear lasttok, set lastend if the token being returned is an
134: * Ender, and return the token.
135: */
136: lasttok = 0;
137: lastend = t->t_flags & Ender;
138: return (t->t_type);
139: }
140:
141: /*
142: * getident - gather an identifier beginning with ac. The character
143: * following identifier goes in cc.
144: */
145:
146: struct toktab *getident(ac, cc)
147: char ac;
148: int *cc;
149: {
150: register c;
151: register char *p;
152: register struct toktab *t;
153: extern char *putident();
154: extern struct toktab *findres();
155:
156: c = ac;
157: p = strfree;
158: /*
159: * Copy characters into string space until a non-alphanumeric character
160: * is found.
161: */
162: do {
163: if (p >= send)
164: syserr("out of string space");
165: *p++ = c;
166: c = ctran[NextChar];
167: } while (isalnum(c));
168: if (p >= send)
169: syserr("out of string space");
170: *p++ = 0;
171: *cc = c;
172: /*
173: * If the identifier is a reserved word, make a ResNode for it and return
174: * the token value. Otherwise, install it with putident, make an
175: * IdNode for it, and return.
176: */
177: if ((t = findres()) != NULL) {
178: yylval = ResNode(t->t_type);
179: return t;
180: }
181: else {
182: yylval = IdNode((int)putident(p-strfree));
183: return T_Ident;
184: }
185: }
186:
187: /*
188: * findres - if the string just copied into the string space by getident
189: * is a reserved word, return a pointer to its entry in the token table.
190: * Return NULL if the string isn't a reserved word.
191: */
192:
193: struct toktab *findres()
194: {
195: register struct toktab *t;
196: register char c, *p;
197:
198: p = strfree;
199: c = *p;
200: if (!islower(c))
201: return NULL;
202: /*
203: * Point t at first reserved word that starts with c (if any).
204: */
205: if ((t = restab[c - '_']) == NULL)
206: return NULL;
207: /*
208: * Search through reserved words, stopping when a match is found
209: * or when the current reserved word doesn't start with c.
210: */
211: while (t->t_word[0] == c) {
212: if (strcmp(t->t_word, p) == 0)
213: return t;
214: t++;
215: }
216: return NULL;
217: }
218:
219: /*
220: * getnum - gather a numeric literal starting with ac and put the
221: * character following the literal into *cc.
222: */
223:
224: struct toktab *getnum(ac, cc)
225: char ac;
226: int *cc;
227: {
228: register c;
229: register r;
230: register state;
231: char *p;
232: int realflag;
233: extern char *putident();
234:
235: c = ac;
236: r = tonum(c);
237: p = strfree;
238: state = 0;
239: realflag = 0;
240: for (;;) {
241: if (p >= send)
242: syserr("out of string space");
243: *p++ = c;
244: c = ctran[NextChar];
245: switch (state) {
246: case 0: /* integer part */
247: if (isdigit(c)) { r = r * 10 + tonum(c); continue; }
248: if (c == '.') { state = 1; realflag++; continue; }
249: if (tolower(c) == 'e') { state = 2; realflag++; continue; }
250: if (tolower(c) == 'r') {
251: state = 5;
252: if (r < 2 || r > 36)
253: err("invalid radix for integer literal", 0);
254: continue;
255: }
256: break;
257: case 1: /* fractional part */
258: if (isdigit(c)) continue;
259: if (tolower(c) == 'e') { state = 2; continue; }
260: break;
261: case 2: /* optional exponent sign */
262: if (c == '+' || c == '-') { state = 3; continue; }
263: case 3: /* first digit after e, e+, or e- */
264: if (isdigit(c)) { state = 4; continue; }
265: err("invalid real literal", 0);
266: break;
267: case 4: /* remaining digits after e */
268: if (isdigit(c)) continue;
269: break;
270: case 5: /* first digit after r */
271: if ((isdigit(c) || isletter(c)) && tonum(c) < r)
272: { state = 6; continue; }
273: err("invalid integer literal", 0);
274: break;
275: case 6: /* remaining digits after r */
276: if (isdigit(c) || isletter(c)) {
277: if (tonum(c) >= r) { /* illegal digit for radix r */
278: err("invalid digit in integer literal", 0);
279: r = tonum('z'); /* prevent more messages */
280: }
281: continue;
282: }
283: break;
284: }
285: break;
286: }
287: if (p >= send)
288: syserr("out of string space");
289: *p++ = 0;
290: *cc = c;
291: if (realflag) {
292: yylval = RealNode((int)putident(p-strfree));
293: return T_Real;
294: }
295: yylval = IntNode((int)putident(p-strfree));
296: return T_Int;
297: }
298:
299: /*
300: * getstring - gather a string literal starting with ac and place the
301: * character following the literal in *cc.
302: */
303:
304: struct toktab *getstring(ac, cc)
305: char ac;
306: int *cc;
307: {
308: register c, sc;
309: register char *p;
310: char *lc;
311: extern char *putident();
312: int len;
313:
314: sc = c = ac;
315: p = strfree;
316: lc = 0;
317: while ((c = NextChar) != sc && c != '\n' && c != EOF) {
318: contin:
319: if (c == '_')
320: lc = p;
321: else if (!isspace(c))
322: lc = 0;
323: if (ctran[c] == Escape) {
324: c = NextChar;
325: if (isoctal(c))
326: c = octesc(c);
327: else if (ctran[c] == 'x')
328: c = hexesc();
329: else if (ctran[c] == '^')
330: c = ctlesc();
331: else
332: c = esctab[c];
333: if (c == EOF)
334: goto noquote;
335: }
336: if (p >= send)
337: syserr("out of string space");
338: *p++ = c;
339: }
340: if (p >= send)
341: syserr("out of string space");
342: *p++ = 0;
343: if (c == sc)
344: *cc = ' ';
345: else {
346: if (c == '\n' && lc) {
347: p = lc;
348: while ((c = NextChar) != EOF && isspace(c)) ;
349: if (c != EOF)
350: goto contin;
351: }
352: noquote:
353: err("unclosed quote", 0);
354: *cc = c;
355: }
356: if (ac == '"') { /* a string literal */
357: len = p - strfree;
358: yylval = StrNode((int)putident(len), len);
359: return T_String;
360: }
361: else { /* a cset literal */
362: len = p - strfree;
363: yylval = CsetNode((int)putident(len), len);
364: return T_Cset;
365: }
366: }
367:
368: /*
369: * ctlesc - translate a control escape -- backslash followed by
370: * caret and one character.
371: */
372:
373: ctlesc()
374: {
375: register c;
376:
377: c = NextChar;
378: if (c == EOF)
379: return EOF;
380: return (c & 037);
381: }
382:
383: /*
384: * octesc - translate an octal escape -- backslash followed by
385: * one, two, or three octal digits.
386: */
387:
388: octesc(ac)
389: char ac;
390: {
391: register c, nc, i;
392:
393: c = 0;
394: nc = ac;
395: i = 1;
396: do {
397: c = (c << 3) | (nc - '0');
398: nc = NextChar;
399: if (nc == EOF)
400: return EOF;
401: } while (isoctal(nc) && i++ < 3);
402: PushChar(nc);
403: return (c & 0377);
404: }
405:
406: /*
407: * hexesc - translate a hexadecimal escape -- backslash-x
408: * followed by one or two hexadecimal digits.
409: */
410:
411: hexesc()
412: {
413: register c, nc, i;
414:
415: c = 0;
416: i = 0;
417: while (i++ < 2) {
418: nc = NextChar;
419: if (nc == EOF)
420: return EOF;
421: if (nc >= 'a' && nc <= 'f')
422: nc -= 'a' - 10;
423: else if (nc >= 'A' && nc <= 'F')
424: nc -= 'A' - 10;
425: else if (isdigit(nc))
426: nc -= '0';
427: else {
428: PushChar(nc);
429: break;
430: }
431: c = (c << 4) | nc;
432: }
433: return c;
434: }
435:
436: /*
437: * getop - find the longest legal operator and return a pointer
438: * to its entry in the token table. The tour describes the
439: * operator recognition process in detail.
440: */
441:
442: struct toktab *getop(ac, cc)
443: char ac;
444: int *cc;
445: {
446: register struct optab *state;
447: register char c, i;
448:
449: state = state0;
450: c = ac;
451: for (;;) {
452: while ((i = state->o_input) && c != i)
453: state++;
454: switch (state->o_action) {
455: case A_Goto:
456: state = (struct optab *) state->o_val;
457: c = ctran[NextChar];
458: continue;
459: case A_Error:
460: err("invalid character", 0);
461: *cc = ' ';
462: return NULL;
463: case A_Return:
464: *cc = c;
465: return (struct toktab *) (state->o_val);
466: case A_Immret:
467: *cc = ' ';
468: return (struct toktab *) (state->o_val);
469: }
470: }
471: }
472:
473: /*
474: * nextchar - return the next character in the input.
475: */
476:
477: nextchar()
478: {
479: register int c;
480:
481: if (c = peekc) {
482: peekc = 0;
483: return c;
484: }
485: c = getc(infile);
486: switch (c) {
487: case EOF:
488: if (incol) {
489: c = '\n';
490: inline++;
491: incol = 0;
492: peekc = EOF;
493: break;
494: }
495: else {
496: inline = 0;
497: incol = 0;
498: break;
499: }
500: case '\n':
501: inline++;
502: incol = 0;
503: break;
504: case '\t':
505: incol = (incol | 7) + 1;
506: break;
507: case '\b':
508: if (incol)
509: incol--;
510: break;
511: default:
512: incol++;
513: }
514: return c;
515: }
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.