Skip to content

Instantly share code, notes, and snippets.

@rgchris

rgchris/sql-lexer.bnf

Last active December 6, 2020 19:41
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save rgchris/f9f070c15afb64022d1361f191baa16c to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save rgchris/f9f070c15afb64022d1361f191baa16c to your computer and use it in GitHub Desktop.
Download ZIP

Revisions

  1. rgchris revised this gist Dec 6, 2020. 2 changed files with 97 additions and 1 deletion.
    2 changes: 1 addition & 1 deletion sql-lexer.reb
    View file
    Original file line number Diff line number Diff line change
    @@ -97,4 +97,4 @@ parse-sql: func [
    head is-sql
    ]

    probe parse-sql {Select * From `something` Where x < 10; @Foo -- Test}
    probe parse-sql {Select * From `something` Where x < 10; @Foo @bar -- Test}
    96 changes: 96 additions & 0 deletions sql-lexer.red
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,96 @@
    Red [
    Title: "SQL Lexer"
    Date: 24-Jul-2020
    Author: "Christopher Ross-Gill"
    ]

    lexer: make object! [
    whitespace: charset " ^-"
    delimiters: charset {"'(),.;`}
    chars: complement union union whitespace delimiters charset "^/^M"
    digit: charset "0123456789"
    lower-alpha: charset [#"a" - #"z"]
    upper-alpha: charset [#"A" - #"Z"]
    alpha: union lower-alpha upper-alpha
    word-literal: complement charset "`"
    comment-line: complement charset "^/^M"
    comment: complement charset "*"
    quoted-single: complement charset {'\}
    quoted-double: complement charset {"\}

    type: none
    part: none
    delimiter: ";"

    value: [
    "--" any comment-line
    (type: 'comment-line)
    |
    "/*" any [some comment | #"*" not #"/"] "*/"
    (type: 'comment)
    |
    #"#" any comment-line ; Used by MySQL
    (type: 'comment-hash)
    |
    #"'" any [some quoted-single | "\\" | "\'" | "''"] #"'"
    (type: 'string-single)
    |
    #"^"" any [some quoted-double | "\\" | {\"} | {""}] #"^""
    (type: 'string-double)
    |
    #"`" some word-literal #"`"
    (type: 'literal)
    |
    #"@" opt #"@" some [some lower-alpha | #"." | #"_"] ; include "$" here?
    (type: 'variable)
    |
    alpha any [some alpha | digit | #"_"] ; include #"$" here?
    (type: 'word)
    |
    opt #"-" some digit
    opt ["." some digit]
    opt [[#"e" | #"E"] opt #"-" some digit]
    (type: 'number)
    |
    copy part some chars ; misc
    (
    type: switch part [
    "*" ['star]
    "+" "-" "/" ['math]
    "=" "!=" ">=" ">" "<=" "<>" "<" ['operator]
    default ['other]
    ]
    )
    ]

    statement: [
    any [
    #"^/" | change [#"^M" opt #"^/"] #"^/"
    |
    change copy part some whitespace (detab part)
    |
    delimiter
    |
    #"(" | #")"
    |
    #"," | #"."
    |
    change copy part value (rejoin ["«" form any [type '_] "»«" part "»"])
    |
    end
    |
    (print "SKIP: Should Not Happen") ?? skip
    ]
    ]
    ]

    parse-sql: func [
    statement [string!]
    ][
    all [
    parse/case statement: copy statement lexer/statement
    statement
    ]
    ]

    probe parse-sql {Select * From `something` Where x < 10; @Foo @bar -- Test}
  2. rgchris renamed this gist Jul 25, 2020. 1 changed file with 0 additions and 0 deletions.
    0 sql-lexer.ebnf → sql-lexer.bnf
    View file
    File renamed without changes.
  3. rgchris revised this gist Jul 25, 2020. 1 changed file with 25 additions and 0 deletions.
    25 changes: 25 additions & 0 deletions sql-lexer.ebnf
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,25 @@
    Statement ::= (Newline | Whitespace | '(' | ')' | ',' | '.' | Value | ';')*

    Value ::= Comment-Line | Comment | String-Single | String-Double | Literal | Variable | Word | Number | Misc

    Comment-Line ::= ('--' | '#') [^#xA#xD]*

    Comment ::= '/*' ( [^*] | '*'+ [^*/] )* '*'* '*/'

    String-Single ::= "'" ([^'\]+ | '\\' | "\'" | "''")* "'"

    String-Double ::= '"' ([^"\]+ | '\\' | '\"' | '""')* '"'

    Literal ::= '`' [^`]+ '`'

    Variable ::= '@' '@'? ([a-z] | '.' | '_')+ /* include '$' here? */

    Word ::= [A-Za-z] [0-9_A-Za-z]* /* include '$' here? */

    Number ::= '-'? [0-9]+ ('.' [0-9]+)? ([Ee] '-'? [0-9]+)?

    Misc ::= [^#x9#xA#xD "'(),.;`]+
    Newline ::= #xA | #xD #xA?

    Whitespace ::= (#x9 | #x20)+
  4. rgchris revised this gist Jul 24, 2020. 1 changed file with 3 additions and 0 deletions.
    3 changes: 3 additions & 0 deletions sql-lexer.reb
    View file
    Original file line number Diff line number Diff line change
    @@ -29,6 +29,9 @@ lexer: make object! [
    "/*" any [some comment | #"*" not #"/"] "*/"
    (type: 'comment)
    |
    #"#" any comment-line ; Used by MySQL
    (type: 'comment-hash)
    |
    #"'" any [some quoted-single | "\\" | "\'" | "''"] #"'"
    (type: 'string-single)
    |
  5. rgchris revised this gist Jul 24, 2020. 1 changed file with 8 additions and 6 deletions.
    14 changes: 8 additions & 6 deletions sql-lexer.reb
    View file
    Original file line number Diff line number Diff line change
    @@ -26,13 +26,13 @@ lexer: make object! [
    "--" any comment-line
    (type: 'comment-line)
    |
    "/*" any [comment | #"*" not #"/"] "*/"
    "/*" any [some comment | #"*" not #"/"] "*/"
    (type: 'comment)
    |
    #"'" any ["\\" | "\'" | "''" | quoted-single] #"'"
    #"'" any [some quoted-single | "\\" | "\'" | "''"] #"'"
    (type: 'string-single)
    |
    #"^"" any ["\\" | {\"} | {""} | quoted-double] #"^""
    #"^"" any [some quoted-double | "\\" | {\"} | {""}] #"^""
    (type: 'string-double)
    |
    #"`" some word-literal #"`"
    @@ -41,10 +41,12 @@ lexer: make object! [
    #"@" opt #"@" some [some lower-alpha | #"." | #"_"] ; include "$" here?
    (type: 'variable)
    |
    alpha any [alpha | digit | #"_"] ; include #"$" here?
    alpha any [some alpha | digit | #"_"] ; include #"$" here?
    (type: 'word)
    |
    opt #"-" some digit opt ["." some digit] opt [[#"e" | #"E"] opt #"-" some digit]
    opt #"-" some digit
    opt ["." some digit]
    opt [[#"e" | #"E"] opt #"-" some digit]
    (type: 'number)
    |
    copy part some chars ; misc
    @@ -92,4 +94,4 @@ parse-sql: func [
    head is-sql
    ]

    probe parse-sql {Select * From `something` Where x < 10; @Foo}
    probe parse-sql {Select * From `something` Where x < 10; @Foo -- Test}
  6. rgchris revised this gist Jul 24, 2020. 1 changed file with 13 additions and 6 deletions.
    19 changes: 13 additions & 6 deletions sql-lexer.reb
    View file
    Original file line number Diff line number Diff line change
    @@ -10,7 +10,7 @@ lexer: make object! [
    chars: complement union union whitespace delimiters charset "^/^M"
    digit: charset "0123456789"
    lower-alpha: charset [#"a" - #"z"]
    upper-alpha: charset [#"a" - #"z"]
    upper-alpha: charset [#"A" - #"Z"]
    alpha: union lower-alpha upper-alpha
    word-literal: complement charset "`"
    comment-line: complement charset "^/^M"
    @@ -26,7 +26,7 @@ lexer: make object! [
    "--" any comment-line
    (type: 'comment-line)
    |
    "/*" any [comment | "*" not "/"] "*/"
    "/*" any [comment | #"*" not #"/"] "*/"
    (type: 'comment)
    |
    #"'" any ["\\" | "\'" | "''" | quoted-single] #"'"
    @@ -47,8 +47,15 @@ lexer: make object! [
    opt #"-" some digit opt ["." some digit] opt [[#"e" | #"E"] opt #"-" some digit]
    (type: 'number)
    |
    some chars ; misc
    (type: 'other)
    copy part some chars ; misc
    (
    type: switch part [
    "*" ['star]
    "+" "-" "/" ['math]
    "=" "!=" ">=" ">" "<=" "<>" "<" ['operator]
    default ['other]
    ]
    )
    ]

    statement: [
    @@ -78,11 +85,11 @@ parse-sql: func [
    ][
    assert [
    all [
    did is-sql: parse copy statement lexer/statement
    is-sql: parse/case copy statement lexer/statement
    tail? is-sql
    ]
    ]
    head is-sql
    ]

    probe parse-sql {Select * From `something`}
    probe parse-sql {Select * From `something` Where x < 10; @Foo}
  7. rgchris revised this gist Jul 24, 2020. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion sql-lexer.reb
    View file
    Original file line number Diff line number Diff line change
    @@ -67,7 +67,7 @@ lexer: make object! [
    |
    end
    |
    (info "SKIP: Should Not Happen") ?? skip
    (print "SKIP: Should Not Happen") ?? skip
    ]
    ]
    ]
  8. rgchris revised this gist Jul 24, 2020. 1 changed file with 5 additions and 5 deletions.
    10 changes: 5 additions & 5 deletions sql-lexer.reb
    View file
    Original file line number Diff line number Diff line change
    @@ -38,10 +38,10 @@ lexer: make object! [
    #"`" some word-literal #"`"
    (type: 'literal)
    |
    "@" opt "@" some [some lower-alpha | "." | "_"] ; include "$" here?
    (type: 'word)
    #"@" opt #"@" some [some lower-alpha | #"." | #"_"] ; include "$" here?
    (type: 'variable)
    |
    alpha any [alpha | digit | "_"] ; include "$" here?
    alpha any [alpha | digit | #"_"] ; include #"$" here?
    (type: 'word)
    |
    opt #"-" some digit opt ["." some digit] opt [[#"e" | #"E"] opt #"-" some digit]
    @@ -59,9 +59,9 @@ lexer: make object! [
    |
    delimiter
    |
    "(" | ")"
    #"(" | #")"
    |
    "," | "."
    #"," | #"."
    |
    change copy part value (rejoin ["«" form type "»«" part "»"])
    |
  9. rgchris created this gist Jul 24, 2020.
    88 changes: 88 additions & 0 deletions sql-lexer.reb
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,88 @@
    Rebol [
    Title: "SQL Lexer"
    Date: 24-Jul-2020
    Author: "Christopher Ross-Gill"
    ]

    lexer: make object! [
    whitespace: charset " ^-"
    delimiters: charset {"'(),.;`}
    chars: complement union union whitespace delimiters charset "^/^M"
    digit: charset "0123456789"
    lower-alpha: charset [#"a" - #"z"]
    upper-alpha: charset [#"a" - #"z"]
    alpha: union lower-alpha upper-alpha
    word-literal: complement charset "`"
    comment-line: complement charset "^/^M"
    comment: complement charset "*"
    quoted-single: complement charset {'\}
    quoted-double: complement charset {"\}

    type: _
    part: _
    delimiter: ";"

    value: [
    "--" any comment-line
    (type: 'comment-line)
    |
    "/*" any [comment | "*" not "/"] "*/"
    (type: 'comment)
    |
    #"'" any ["\\" | "\'" | "''" | quoted-single] #"'"
    (type: 'string-single)
    |
    #"^"" any ["\\" | {\"} | {""} | quoted-double] #"^""
    (type: 'string-double)
    |
    #"`" some word-literal #"`"
    (type: 'literal)
    |
    "@" opt "@" some [some lower-alpha | "." | "_"] ; include "$" here?
    (type: 'word)
    |
    alpha any [alpha | digit | "_"] ; include "$" here?
    (type: 'word)
    |
    opt #"-" some digit opt ["." some digit] opt [[#"e" | #"E"] opt #"-" some digit]
    (type: 'number)
    |
    some chars ; misc
    (type: 'other)
    ]

    statement: [
    any [
    #"^/" | change [#"^M" opt #"^/"] #"^/"
    |
    change copy part some whitespace (detab part)
    |
    delimiter
    |
    "(" | ")"
    |
    "," | "."
    |
    change copy part value (rejoin ["«" form type "»«" part "»"])
    |
    end
    |
    (info "SKIP: Should Not Happen") ?? skip
    ]
    ]
    ]

    parse-sql: func [
    statement [text!]
    <local> is-sql
    ][
    assert [
    all [
    did is-sql: parse copy statement lexer/statement
    tail? is-sql
    ]
    ]
    head is-sql
    ]

    probe parse-sql {Select * From `something`}