Return to analyzer.lex CVS log

Up to [Research Unix] / researchv10no / cmd / sml / lib / twig

Annotation of researchv10no/cmd/sml/lib/twig/analyzer.lex, revision 1.1.1.1

1.1       root        1: 
                      2: (* November 1988, Jussi Rintanen, Helsinki University of Technology *)
                      3: 
                      4: (* This is the specification of ML-Twig lexical analyzer.
                      5: 
                      6:    This lexer correctly recognizes all Standard ML tokens, as specified
                      7:    in [Harper, MacQueen and Milner, 1986].
                      8: *)
                      9: 
                     10: datatype lexresult =
                     11:       IDENTIFIER of string
                     12:       | INT of string
                     13:       | EQ
                     14:       | RPAREN
                     15:       | LPAREN
                     16:       | COLON
                     17:       | SEMICOLON
                     18:       | COMMA
                     19:       | TREEREF of int list
                     20:       | OTHER of string
                     21:       | SPACE of string
                     22:       | EOF
                     23: 
                     24: local
                     25:   fun digit c = ("0" <= c) andalso (c <= "9")
                     26:   fun str2 (a,c::r) =
                     27:     if digit c
                     28:       then str2 (a*10 + ord c - ord "0" ,r)
                     29:     else (a,c::r)
                     30:     | str2 r = r
                     31: in
                     32:   fun str0int s = str2 (0,s)
                     33: end
                     34: 
                     35: local
                     36:   fun parse_treeref' nil = nil
                     37:     | parse_treeref' ["$"] = nil
                     38:     | parse_treeref' s =
                     39:       let val (i,r) = str0int s
                     40:       in
                     41:        i :: parse_treeref' (tl r)
                     42:       end
                     43: in  
                     44:   val parse_treeref = (parse_treeref' o tl o explode)
                     45: end
                     46: 
                     47: val current_line_number : int ref = ref 1
                     48: val commentlevel : int ref = ref 0
                     49: 
                     50: fun current_line () = !current_line_number
                     51: 
                     52: fun eof() = EOF
                     53: 
                     54: %%
                     55: 
                     56: %structure TwigLexer
                     57: 
                     58: %s C;
                     59: 
                     60: idbegin=[a-zA-Z'_];
                     61: idchar=[a-zA-Z0-9'_];
                     62: 
                     63: sidbegin=[!%&$+/:<=>?@~'|*\\^-];
                     64: sidchar=[!%&$+/:<=>?@~'|#*\\^-];
                     65: 
                     66: digit=[0-9];
                     67: whitespace=[\ \t];
                     68: newline=[\n];
                     69: controllable=[a-zA-Z];
                     70: 
                     71: %%
                     72: 
                     73: <INITIAL>"$" ({digit}+("."{digit}+)*)? "$" => (TREEREF (parse_treeref yytext));
                     74: 
                     75: <INITIAL>{digit}+ => (INT yytext);
                     76: 
                     77: <INITIAL>{digit}+("."{digit}+)?(E"~"?{digit}+)? => (OTHER yytext);
                     78: 
                     79: <INITIAL>{sidbegin}{sidchar}* => (case yytext of
                     80:                          ":" => COLON
                     81:                          | "=" => EQ
                     82:                          | _ => OTHER yytext);
                     83: 
                     84: <INITIAL>{idbegin}{idchar}* => (IDENTIFIER yytext);
                     85: 
                     86: <INITIAL>(({idbegin}{idchar}*) | ({sidbegin}{sidchar}*))
                     87: ("."(({idbegin}{idchar}*) | ({sidchar}+)))+
                     88:                     => (OTHER yytext);
                     89: 
                     90: <INITIAL>"(" => (LPAREN);
                     91: <INITIAL>")" => (RPAREN);
                     92: <INITIAL>";" => (SEMICOLON);
                     93: <INITIAL>"," => (COMMA);
                     94: <INITIAL>"]" => (OTHER yytext);
                     95: <INITIAL>"[" => (OTHER yytext);    
                     96: <INITIAL>"..." => (OTHER yytext);
                     97: <INITIAL>"#"{digit}+ => (OTHER yytext);
                     98: 
                     99: <INITIAL>\"((\\([nt\\"]|([0-9]{3})|("^"[a-zA-Z])|([\ \t\n]+\\)))|[^\n\\"])*\"
                    100:  => (let val dummy = (current_line_number :=
                    101:                      fold (fn (a,b) => b+(if a="\n" then 1 else 0))
                    102:                      (explode yytext) (!current_line_number))
                    103:        in
                    104:          OTHER yytext
                    105:        end);
                    106: 
                    107: <INITIAL>"{"|"}"|"_" => (OTHER yytext);
                    108: 
                    109: <INITIAL>{whitespace}+ => (SPACE yytext);
                    110: 
                    111: <INITIAL>{newline} => ( inc current_line_number; SPACE yytext);
                    112:                                                           
                    113: <INITIAL>"(*" => ( YYBEGIN C; inc commentlevel; SPACE yytext );
                    114: 
                    115: <C>"(*" => ( inc commentlevel; SPACE yytext );
                    116: <C>"*)" => ( dec commentlevel; if !commentlevel = 0 then YYBEGIN INITIAL else (); SPACE yytext );
                    117: <C>{newline} => ( inc current_line_number; SPACE yytext );
                    118: <C>"*"{newline} => ( inc current_line_number; SPACE yytext );
                    119: <C>(("("[^*\n])|[^(*\n]|("*"[^)\n]))+ => ( SPACE yytext );

This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.