falsepattern/ZigBrains
1
0
Fork 0
Code Issues Pull requests Projects Releases 2 Packages Wiki Activity

fix: improve comment and multiline string tokenizer

Browse source
This commit is contained in:
FalsePattern 2025-03-23 13:49:36 +01:00
parent e33c96d658
commit 868c37c567
Signed by: falsepattern
GPG key ID: E930CDEC50C50E23
3 changed files with 59 additions and 44 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
CHANGELOG.md
View file

@ -17,6 +17,11 @@ Changelog structure reference:
## [Unreleased] ## [Unreleased]
### Fixed
- Zig
- Documentation comment after regular comment was being highlighted as regular comment
## [23.0.1] ## [23.0.1]
### Fixed ### Fixed

64
core/src/main/grammar/Zig.flex
View file

@ -36,8 +36,10 @@ import static com.falsepattern.zigbrains.zig.psi.ZigTypes.*;
%type IElementType %type IElementType
%unicode %unicode
CRLF=\R WHITE_SPACE=\s+
WHITE_SPACE=[\s]+
// visual studio parity
LF=\r\n?|[\n\u0085\u2028\u2029]
bin=[01] bin=[01]
bin_="_"? {bin} bin_="_"? {bin}
@ -54,13 +56,13 @@ dec_int={dec} {dec_}*
hex_int={hex} {hex_}* hex_int={hex} {hex_}*
char_char= \\ . char_char= \\ .
| [^\'\n] | [^\'\r\n\u0085\u2028\u2029]
string_char= \\ . string_char= \\ .
| [^\"\n] | [^\"\r\n\u0085\u2028\u2029]
all_nl_wrap=[^\n]* [ \n]* nl_wrap={LF} (\s|{LF})*
all_no_nl=[^\n]+ all_no_nl=[^\r\n\u0085\u2028\u2029]+
FLOAT= "0x" {hex_int} "." {hex_int} ([pP] [-+]? {dec_int})? FLOAT= "0x" {hex_int} "." {hex_int} ([pP] [-+]? {dec_int})?
@ -84,30 +86,35 @@ BUILTINIDENTIFIER="@"[A-Za-z_][A-Za-z0-9_]*
%state UNT_SQUOT %state UNT_SQUOT
%state UNT_DQUOT %state UNT_DQUOT
%state CDOC_CMT %state CMT_LINE
%state DOC_CMT %state CMT_DOC
%state LINE_CMT %state CMT_CDOC
%% %%
//Comments //Comments
<YYINITIAL> "//!" { yybegin(CDOC_CMT); } <YYINITIAL> "//!" { yybegin(CMT_CDOC); }
<CDOC_CMT> {all_nl_wrap} "//!" { } <YYINITIAL> "////" { yybegin(CMT_LINE); }
<CDOC_CMT> {all_no_nl} { } <YYINITIAL> "///" { yybegin(CMT_DOC); }
<CDOC_CMT> \n { yybegin(YYINITIAL); return CONTAINER_DOC_COMMENT; } <YYINITIAL> "//" { yybegin(CMT_LINE); }
<CDOC_CMT> <<EOF>> { yybegin(YYINITIAL); return CONTAINER_DOC_COMMENT; }
<YYINITIAL> "///" { yybegin(DOC_CMT); } <CMT_LINE> {all_no_nl} { }
<DOC_CMT> {all_nl_wrap} "///" { } <CMT_LINE> {nl_wrap} "////" { }
<DOC_CMT> {all_no_nl} { } <CMT_LINE> {nl_wrap} "///" { yypushback(yylength()); yybegin(YYINITIAL); return LINE_COMMENT; }
<DOC_CMT> \n { yybegin(YYINITIAL); return DOC_COMMENT; } <CMT_LINE> {nl_wrap} "//" { }
<DOC_CMT> <<EOF>> { yybegin(YYINITIAL); return DOC_COMMENT; } <CMT_LINE> {LF} { yybegin(YYINITIAL); return LINE_COMMENT; }
<CMT_LINE> <<EOF>> { yybegin(YYINITIAL); return LINE_COMMENT; }
<YYINITIAL> "//" { yybegin(LINE_CMT); } <CMT_DOC> {all_no_nl} { }
<LINE_CMT> {all_nl_wrap} "//" { } <CMT_DOC> {nl_wrap} "////" { yypushback(yylength()); yybegin(YYINITIAL); return DOC_COMMENT; }
<LINE_CMT> {all_no_nl} { } <CMT_DOC> {nl_wrap} "///" { }
<LINE_CMT> \n { yybegin(YYINITIAL); return LINE_COMMENT; } <CMT_DOC> {LF} { yybegin(YYINITIAL); return DOC_COMMENT; }
<LINE_CMT> <<EOF>> { yybegin(YYINITIAL); return LINE_COMMENT; } <CMT_DOC> <<EOF>> { yybegin(YYINITIAL); return DOC_COMMENT; }
<CMT_CDOC> {all_no_nl} { }
<CMT_CDOC> {nl_wrap} "//!" { }
<CMT_CDOC> {LF} { yybegin(YYINITIAL); return CONTAINER_DOC_COMMENT; }
<CMT_CDOC> <<EOF>> { yybegin(YYINITIAL); return CONTAINER_DOC_COMMENT; }
//Symbols //Symbols
<YYINITIAL> "&" { return AMPERSAND; } <YYINITIAL> "&" { return AMPERSAND; }
@ -239,10 +246,11 @@ BUILTINIDENTIFIER="@"[A-Za-z_][A-Za-z0-9_]*
<STR_LIT> {string_char}*"\"" { yybegin(YYINITIAL); return STRING_LITERAL_SINGLE; } <STR_LIT> {string_char}*"\"" { yybegin(YYINITIAL); return STRING_LITERAL_SINGLE; }
<STR_LIT> <<EOF>> { yybegin(YYINITIAL); return BAD_DQUOT; } <STR_LIT> <<EOF>> { yybegin(YYINITIAL); return BAD_DQUOT; }
<STR_LIT> [^] { yypushback(1); yybegin(UNT_DQUOT); } <STR_LIT> [^] { yypushback(1); yybegin(UNT_DQUOT); }
<YYINITIAL> "\\\\" { yybegin(STR_MULT_LINE); } <YYINITIAL> "\\\\" { yybegin(STR_MULT_LINE); }
<STR_MULT_LINE> {all_nl_wrap} "\\\\" { }
<STR_MULT_LINE> {all_no_nl} { } <STR_MULT_LINE> {all_no_nl} { }
<STR_MULT_LINE> \n { yybegin(YYINITIAL); return STRING_LITERAL_MULTI; } <STR_MULT_LINE> {nl_wrap} "\\\\" { }
<STR_MULT_LINE> {LF} { yybegin(YYINITIAL); return STRING_LITERAL_MULTI; }
<STR_MULT_LINE> <<EOF>> { yybegin(YYINITIAL); return STRING_LITERAL_MULTI; } <STR_MULT_LINE> <<EOF>> { yybegin(YYINITIAL); return STRING_LITERAL_MULTI; }
<YYINITIAL> {IDENTIFIER_PLAIN} { return IDENTIFIER; } <YYINITIAL> {IDENTIFIER_PLAIN} { return IDENTIFIER; }
@ -254,10 +262,10 @@ BUILTINIDENTIFIER="@"[A-Za-z_][A-Za-z0-9_]*
<YYINITIAL> {BUILTINIDENTIFIER} { return BUILTINIDENTIFIER; } <YYINITIAL> {BUILTINIDENTIFIER} { return BUILTINIDENTIFIER; }
<UNT_SQUOT> <<EOF>> { yybegin(YYINITIAL); return BAD_SQUOT; } <UNT_SQUOT> <<EOF>> { yybegin(YYINITIAL); return BAD_SQUOT; }
<UNT_SQUOT> {CRLF} { yybegin(YYINITIAL); return BAD_SQUOT; } <UNT_SQUOT> {LF} { yybegin(YYINITIAL); return BAD_SQUOT; }
<UNT_SQUOT> {all_no_nl} { } <UNT_SQUOT> {all_no_nl} { }
<UNT_DQUOT> <<EOF>> { yybegin(YYINITIAL); return BAD_DQUOT; } <UNT_DQUOT> <<EOF>> { yybegin(YYINITIAL); return BAD_DQUOT; }
<UNT_DQUOT> {CRLF} { yybegin(YYINITIAL); return BAD_DQUOT; } <UNT_DQUOT> {LF} { yybegin(YYINITIAL); return BAD_DQUOT; }
<UNT_DQUOT> {all_no_nl} { } <UNT_DQUOT> {all_no_nl} { }
<YYINITIAL> {WHITE_SPACE} { return WHITE_SPACE; } <YYINITIAL> {WHITE_SPACE} { return WHITE_SPACE; }

34
core/src/main/grammar/Zon.flex
View file

@ -36,8 +36,10 @@ import static com.falsepattern.zigbrains.zon.psi.ZonTypes.*;
%type IElementType %type IElementType
%unicode %unicode
CRLF=\R WHITE_SPACE=\s+
WHITE_SPACE=[\s]+
// visual studio parity
LF=\r\n?|[\n\u0085\u2028\u2029]
bin=[01] bin=[01]
bin_="_"? {bin} bin_="_"? {bin}
@ -54,13 +56,13 @@ dec_int={dec} {dec_}*
hex_int={hex} {hex_}* hex_int={hex} {hex_}*
char_char= \\ . char_char= \\ .
| [^\'\n] | [^\'\r\n\u0085\u2028\u2029]
string_char= \\ . string_char= \\ .
| [^\"\n] | [^\"\r\n\u0085\u2028\u2029]
all_nl_wrap=[^\n]* [ \n]* nl_wrap={LF} (\s|{LF})*
all_no_nl=[^\n]+ all_no_nl=[^\r\n\u0085\u2028\u2029]+
FLOAT= "0x" {hex_int} "." {hex_int} ([pP] [-+]? {dec_int})? FLOAT= "0x" {hex_int} "." {hex_int} ([pP] [-+]? {dec_int})?
@ -83,16 +85,16 @@ IDENTIFIER_PLAIN=[A-Za-z_][A-Za-z0-9_]*
%state UNT_SQUOT %state UNT_SQUOT
%state UNT_DQUOT %state UNT_DQUOT
%state LINE_CMT %state CMT_LINE
%% %%
//Comments //Comments
<YYINITIAL> "//" { yybegin(LINE_CMT); } <YYINITIAL> "//" { yybegin(CMT_LINE); }
<LINE_CMT> {all_nl_wrap} "//" { } <CMT_LINE> {all_no_nl} { }
<LINE_CMT> {all_no_nl} { } <CMT_LINE> {nl_wrap} "//" { }
<LINE_CMT> \n { yybegin(YYINITIAL); return LINE_COMMENT; } <CMT_LINE> \R { yybegin(YYINITIAL); return LINE_COMMENT; }
<LINE_CMT> <<EOF>> { yybegin(YYINITIAL); return LINE_COMMENT; } <CMT_LINE> <<EOF>> { yybegin(YYINITIAL); return LINE_COMMENT; }
//Symbols //Symbols
@ -123,9 +125,9 @@ IDENTIFIER_PLAIN=[A-Za-z_][A-Za-z0-9_]*
<STR_LIT> [^] { yypushback(1); yybegin(UNT_DQUOT); } <STR_LIT> [^] { yypushback(1); yybegin(UNT_DQUOT); }
<YYINITIAL> "\\\\" { yybegin(STR_MULT_LINE); } <YYINITIAL> "\\\\" { yybegin(STR_MULT_LINE); }
<STR_MULT_LINE> {all_nl_wrap} "\\\\" { }
<STR_MULT_LINE> {all_no_nl} { } <STR_MULT_LINE> {all_no_nl} { }
<STR_MULT_LINE> \n { yybegin(YYINITIAL); return STRING_LITERAL_MULTI; } <STR_MULT_LINE> {nl_wrap} "\\\\" { }
<STR_MULT_LINE> {LF} { yybegin(YYINITIAL); return STRING_LITERAL_MULTI; }
<STR_MULT_LINE> <<EOF>> { yybegin(YYINITIAL); return STRING_LITERAL_MULTI; } <STR_MULT_LINE> <<EOF>> { yybegin(YYINITIAL); return STRING_LITERAL_MULTI; }
//Numbers //Numbers
@ -144,10 +146,10 @@ IDENTIFIER_PLAIN=[A-Za-z_][A-Za-z0-9_]*
//Error handling //Error handling
<UNT_SQUOT> <<EOF>> { yybegin(YYINITIAL); return BAD_SQUOT; } <UNT_SQUOT> <<EOF>> { yybegin(YYINITIAL); return BAD_SQUOT; }
<UNT_SQUOT> {CRLF} { yybegin(YYINITIAL); return BAD_SQUOT; } <UNT_SQUOT> {LF} { yybegin(YYINITIAL); return BAD_SQUOT; }
<UNT_SQUOT> {all_no_nl} { } <UNT_SQUOT> {all_no_nl} { }
<UNT_DQUOT> <<EOF>> { yybegin(YYINITIAL); return BAD_DQUOT; } <UNT_DQUOT> <<EOF>> { yybegin(YYINITIAL); return BAD_DQUOT; }
<UNT_DQUOT> {CRLF} { yybegin(YYINITIAL); return BAD_DQUOT; } <UNT_DQUOT> {LF} { yybegin(YYINITIAL); return BAD_DQUOT; }
<UNT_DQUOT> {all_no_nl} { } <UNT_DQUOT> {all_no_nl} { }
//Misc //Misc