1 people like it.

Remove block comments from a list of tokens

Cleaning up source code for a Forth like language before the compiling stage

 1: 
 2: 
 3: 
 4: 
 5: 
 6: 
 7: 
 8: 
 9: 
10: 
11: 
12: 
13: 
14: 
15: 
16: 
17: 
18: 
19: 
20: 
21: 
22: 
23: 
24: 
// Remove block comment(s) from a list of tokens
let removeComments listIn start stop =
    let rec loop listIn listOut depth =   
        match listIn with
        | [] -> match depth with
                | 0 -> listOut
                | depth when depth > 0 -> failwith (sprintf "Too many '%s' tokens\n" stop)
                | _ -> failwith (sprintf "Too many '%s' tokens\n" start)
        | head::tail when depth > 0    -> loop [] listOut depth
        | head::tail when head = start -> loop tail listOut (depth - 1)
        | head::tail when head = stop  -> loop tail listOut (depth + 1)
        | head::tail when depth = 0    -> loop tail (head::listOut) depth
        | head::tail                   -> loop tail listOut depth
    loop listIn [] 0 |> List.rev

 // Tokenize string by splitting at whitespace and removing empty cells
let tokenize (t:string) = t.Split () |> Array.toList |> List.filter (fun x -> x <> "")

let source = ": CLSB ( n -- n ) dup 1 - and ; ( Clear least significant bit )"
printfn "Source: %A" source

let tokens = tokenize source
let noCommentTokens = removeComments tokens "(" ")"
printfn "Clean source %A" (List.fold (fun a t -> a + t + " ") " " noCommentTokens)
val removeComments : listIn:string list -> start:string -> stop:string -> string list

Full name: Script.removeComments
val listIn : string list
val start : string
val stop : string
val loop : (string list -> string list -> int -> string list)
val listOut : string list
val depth : int
val failwith : message:string -> 'T

Full name: Microsoft.FSharp.Core.Operators.failwith
val sprintf : format:Printf.StringFormat<'T> -> 'T

Full name: Microsoft.FSharp.Core.ExtraTopLevelOperators.sprintf
val head : string
val tail : string list
Multiple items
module List

from Microsoft.FSharp.Collections

--------------------
type List<'T> =
  | ( [] )
  | ( :: ) of Head: 'T * Tail: 'T list
  interface IEnumerable
  interface IEnumerable<'T>
  member Head : 'T
  member IsEmpty : bool
  member Item : index:int -> 'T with get
  member Length : int
  member Tail : 'T list
  static member Cons : head:'T * tail:'T list -> 'T list
  static member Empty : 'T list

Full name: Microsoft.FSharp.Collections.List<_>
val rev : list:'T list -> 'T list

Full name: Microsoft.FSharp.Collections.List.rev
val tokenize : t:string -> string list

Full name: Script.tokenize
val t : string
Multiple items
val string : value:'T -> string

Full name: Microsoft.FSharp.Core.Operators.string

--------------------
type string = System.String

Full name: Microsoft.FSharp.Core.string
System.String.Split([<System.ParamArray>] separator: char []) : string []
System.String.Split(separator: string [], options: System.StringSplitOptions) : string []
System.String.Split(separator: char [], options: System.StringSplitOptions) : string []
System.String.Split(separator: char [], count: int) : string []
System.String.Split(separator: string [], count: int, options: System.StringSplitOptions) : string []
System.String.Split(separator: char [], count: int, options: System.StringSplitOptions) : string []
module Array

from Microsoft.FSharp.Collections
val toList : array:'T [] -> 'T list

Full name: Microsoft.FSharp.Collections.Array.toList
val filter : predicate:('T -> bool) -> list:'T list -> 'T list

Full name: Microsoft.FSharp.Collections.List.filter
val x : string
val source : string

Full name: Script.source
val printfn : format:Printf.TextWriterFormat<'T> -> 'T

Full name: Microsoft.FSharp.Core.ExtraTopLevelOperators.printfn
val tokens : string list

Full name: Script.tokens
val noCommentTokens : string list

Full name: Script.noCommentTokens
val fold : folder:('State -> 'T -> 'State) -> state:'State -> list:'T list -> 'State

Full name: Microsoft.FSharp.Collections.List.fold
val a : string
Raw view New version

More information

Link:http://fssnip.net/ih
Posted:6 years ago
Author:Bjørn Bæverfjord
Tags: forth , block comments