Return to lex.sml CVS log

Up to [Research Unix] / researchv10no / cmd / sml / doc / examples

Annotation of researchv10no/cmd/sml/doc/examples/lex.sml, revision 1.1

1.1     ! root        1: (* lex.sml *)
        !             2: 
        !             3: (* two versions of rudimentary lexical analyzers that break a string (representing
        !             4: and input line,say) into a list of "words" defined as sequences of nonwhile-space
        !             5: characters *)
        !             6: 
        !             7: signature LEX =
        !             8: sig
        !             9:   val words: string -> string list
        !            10: end
        !            11: 
        !            12: structure Lex1 : LEX =
        !            13: struct
        !            14: 
        !            15:   fun separator " " = true
        !            16:     | separator "\t" = true
        !            17:     | separator "\n" = true
        !            18:     | separator _ = false
        !            19: 
        !            20:   fun words s =
        !            21:       let fun getword(w,[])  = [implode(rev w)]
        !            22:            | getword(w,c::rest) = 
        !            23:                if separator(c)
        !            24:                then implode(rev w) :: skip rest
        !            25:                else getword(c::w,rest)
        !            26:          and skip [] = []
        !            27:            | skip(c::rest) =
        !            28:                if separator c
        !            29:                then skip rest
        !            30:                else getword([c],rest)
        !            31:        in skip(explode s)
        !            32:       end
        !            33: 
        !            34: end (* Lex1 *)
        !            35: 
        !            36: structure Lex2 : LEX =
        !            37: struct
        !            38: 
        !            39:   fun separator " " = true
        !            40:     | separator "\t" = true
        !            41:     | separator "\n" = true
        !            42:     | separator _ = false
        !            43: 
        !            44:   fun words(s: string) =
        !            45:       let val len  = length s
        !            46:          fun skip n = 
        !            47:              let fun getword m =
        !            48:                  if m>=len orelse separator(substring(s,m,1))
        !            49:                  then substring(s,n,(m-n))::skip(m+1)
        !            50:                  else getword(m+1)
        !            51:               in if n>=len
        !            52:                  then []
        !            53:                  else if separator(substring(s,n,1))
        !            54:                  then skip(n+1)
        !            55:                  else getword(n+1)
        !            56:              end
        !            57:        in skip 0
        !            58:       end
        !            59: 
        !            60: end (* Lex2 *)

This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.