Sign Up
Log In
Log In
or
Sign Up
Places
All Projects
Status Monitor
Collapse sidebar
home:Ledest:erlang:25
erlang
3431-leex-column-number-support.patch
Overview
Repositories
Revisions
Requests
Users
Attributes
Meta
File 3431-leex-column-number-support.patch of Package erlang
From 23014493ea15eb7753777d2ca1f664096263a76b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Min=C4=91ek?= <marko.mindek@invariant.hr> Date: Mon, 30 Jan 2023 00:05:30 +0100 Subject: [PATCH] leex column number support Added: Support for including column number in token, end and error locations. Variables in Rules: TokenCol and TokenLoc Leex options: error_location and tab_size Fixed: Bug in testcase OTP14285 --- lib/parsetools/doc/src/leex.xml | 53 ++-- lib/parsetools/include/leexinc.hrl | 377 ++++++++++++++++------------- lib/parsetools/src/leex.erl | 190 ++++++++++++--- lib/parsetools/test/leex_SUITE.erl | 334 ++++++++++++++----------- 4 files changed, 592 insertions(+), 362 deletions(-) diff --git a/lib/parsetools/doc/src/leex.xml b/lib/parsetools/doc/src/leex.xml index d802e46b59..7714a7c27f 100644 --- a/lib/parsetools/doc/src/leex.xml +++ b/lib/parsetools/doc/src/leex.xml @@ -115,6 +115,22 @@ <p>Causes generated -file() attributes to only include the basename of the file path.</p> </item> + <tag><c>{error_location, line | column}</c></tag> + <item> + <p>If set to <c>column</c>, error location will be + <c>{Line,Column}</c> tuple instead of just <c>Line</c>. Also, <c>StartLoc</c> + and <c>EndLoc</c> in <c>string/2</c>, <c>token/3</c>, and <c>tokens/3</c> + functions will be <c>{Line,Column}</c> tuple instead of just <c>Line</c>. + Default is <c>line</c>. Note that you can use <c>TokenLoc</c> for token + location independently, even if the <c>error_location</c> is set to <c>line</c>.</p> + <p>Unicode characters are counted as many columns as they use bytes + to represent.</p> + </item> + <tag><c>{tab_size, pos_integer()}</c></tag> + <item> + <p>Sets the width of <c>\t</c> character (only relevant if <c>error_location</c> + is set to <c>column</c>). Default is <c>8</c>.</p> + </item> </taglist> <p>Any of the Boolean options can be set to <c>true</c> by stating the name of the option. For example, <c>verbose</c> @@ -147,17 +163,18 @@ </fsdescription> <func> <name since="">Module:string(String) -> StringRet</name> - <name since="">Module:string(String, StartLine) -> StringRet</name> + <name since="">Module:string(String, StartLoc) -> StringRet</name> <fsummary>Generated by Leex</fsummary> <type> <v>String = string()</v> - <v>StringRet = {ok,Tokens,EndLine} | ErrorInfo</v> + <v>StringRet = {ok,Tokens,EndLoc} | ErrorInfo</v> <v>Tokens = [Token]</v> - <v>EndLine = StartLine = erl_anno:line()</v> + <v>StartLoc = EndLoc = erl_anno:location()</v> </type> <desc> <p>Scans <c>String</c> and returns all the tokens in it, or an - error.</p> + error. <c>StartLoc</c> and <c>EndLoc</c> are either <c>erl_anno:line()</c> + or <c>erl_anno:location()</c>, depending on the <c>error_location</c> option.</p> <note><p>It is an error if not all of the characters in <c>String</c> are consumed.</p></note> </desc> @@ -166,7 +183,7 @@ <func> <name since="">Module:token(Cont, Chars) -> {more,Cont1} | {done,TokenRet,RestChars} </name> - <name since="">Module:token(Cont, Chars, StartLine) -> {more,Cont1} + <name since="">Module:token(Cont, Chars, StartLoc) -> {more,Cont1} | {done,TokenRet,RestChars} </name> <fsummary>Generated by Leex</fsummary> @@ -174,10 +191,10 @@ <v>Cont = [] | Cont1</v> <v>Cont1 = tuple()</v> <v>Chars = RestChars = string() | eof</v> - <v>TokenRet = {ok, Token, EndLine} - | {eof, EndLine} + <v>TokenRet = {ok, Token, EndLoc} + | {eof, EndLoc} | ErrorInfo</v> - <v>StartLine = EndLine = erl_anno:line()</v> + <v>StartLoc = EndLoc = erl_anno:location()</v> </type> <desc> <p>This is a re-entrant call to try and scan one token from @@ -193,7 +210,7 @@ but used through the i/o system where it can typically be called in an application by:</p> <code> -io:request(InFile, {get_until,unicode,Prompt,Module,token,[Line]}) +io:request(InFile, {get_until,unicode,Prompt,Module,token,[Loc]}) -> TokenRet</code> </desc> </func> @@ -201,7 +218,7 @@ io:request(InFile, {get_until,unicode,Prompt,Module,token,[Line]}) <func> <name since="">Module:tokens(Cont, Chars) -> {more,Cont1} | {done,TokensRet,RestChars} </name> - <name since="">Module:tokens(Cont, Chars, StartLine) -> + <name since="">Module:tokens(Cont, Chars, StartLoc) -> {more,Cont1} | {done,TokensRet,RestChars} </name> <fsummary>Generated by Leex</fsummary> @@ -209,11 +226,11 @@ io:request(InFile, {get_until,unicode,Prompt,Module,token,[Line]}) <v>Cont = [] | Cont1</v> <v>Cont1 = tuple()</v> <v>Chars = RestChars = string() | eof</v> - <v>TokensRet = {ok, Tokens, EndLine} - | {eof, EndLine} + <v>TokensRet = {ok, Tokens, EndLoc} + | {eof, EndLoc} | ErrorInfo</v> <v>Tokens = [Token]</v> - <v>StartLine = EndLine = erl_anno:line()</v> + <v>StartLoc = EndLoc = erl_anno:location()</v> </type> <desc> <p>This is a re-entrant call to try and scan tokens from @@ -240,7 +257,7 @@ io:request(InFile, {get_until,unicode,Prompt,Module,token,[Line]}) but used through the i/o system where it can typically be called in an application by:</p> <code> -io:request(InFile, {get_until,unicode,Prompt,Module,tokens,[Line]}) +io:request(InFile, {get_until,unicode,Prompt,Module,tokens,[Loc]}) -> TokensRet</code> </desc> </func> @@ -320,6 +337,14 @@ NAME = VALUE</code> <tag><c>TokenLine</c></tag> <item><p>The line number where the token occurred.</p> </item> + <tag><c>TokenCol</c></tag> + <item><p>The column number where the token occurred + (column of the first character included in the token).</p> + </item> + <tag><c>TokenLoc</c></tag> + <item><p>Token location. Expands to <c>{TokenLine,TokenCol}</c> (even + when <c>error_location</c> is set to <c>line</c>.</p> + </item> </taglist> <p>The code must return:</p> diff --git a/lib/parsetools/include/leexinc.hrl b/lib/parsetools/include/leexinc.hrl index 8dfc42f479..a06584ff79 100644 --- a/lib/parsetools/include/leexinc.hrl +++ b/lib/parsetools/include/leexinc.hrl @@ -16,261 +16,269 @@ format_error({illegal,S}) -> ["illegal characters ",io_lib:write_string(S)]; format_error({user,S}) -> S. -string(String) -> string(String, 1). - -string(String, Line) -> string(String, Line, String, []). - -%% string(InChars, Line, TokenChars, Tokens) -> -%% {ok,Tokens,Line} | {error,ErrorInfo,Line}. -%% Note the line number going into yystate, L0, is line of token -%% start while line number returned is line of token end. We want line -%% of token start. - -string([], L, [], Ts) -> % No partial tokens! - {ok,yyrev(Ts),L}; -string(Ics0, L0, Tcs, Ts) -> - case yystate(yystate(), Ics0, L0, 0, reject, 0) of - {A,Alen,Ics1,L1} -> % Accepting end state - string_cont(Ics1, L1, yyaction(A, Alen, Tcs, L0), Ts); - {A,Alen,Ics1,L1,_S1} -> % Accepting transition state - string_cont(Ics1, L1, yyaction(A, Alen, Tcs, L0), Ts); - {reject,_Alen,Tlen,_Ics1,L1,_S1} -> % After a non-accepting state - {error,{L0,?MODULE,{illegal,yypre(Tcs, Tlen+1)}},L1}; - {A,Alen,Tlen,_Ics1,L1,_S1} -> +%% string(InChars) -> +%% string(InChars, Loc) -> +%% {ok,Tokens,EndLoc} | {error,ErrorInfo,EndLoc}. +%% Loc is the starting location of the token, while EndLoc is the first not scanned +%% location. Location is either Line or {Line,Column}, depending on the "error_location" option. + +##str + +do_string([], L, C, [], Ts) -> % No partial tokens! + {ok,yyrev(Ts),{L,C}}; +do_string(Ics0, L0, C0, Tcs, Ts) -> + case yystate(yystate(), Ics0, L0, C0, 0, reject, 0) of + {A,Alen,Ics1,L1,_C1} -> % Accepting end state + C2 = adjust_col(Tcs, Alen, C0), + string_cont(Ics1, L1, C2, yyaction(A, Alen, Tcs, L0, C0), Ts); + {A,Alen,Ics1,L1,_C1,_S1} -> % Accepting transition state + C2 = adjust_col(Tcs, Alen, C0), + string_cont(Ics1, L1, C2, yyaction(A, Alen, Tcs, L0, C0), Ts); + {reject,_Alen,Tlen,_Ics1,_L1,_C1,_S1} -> % After a non-accepting state + {error,{{L0, C0} ,?MODULE,{illegal,yypre(Tcs, Tlen+1)}},{L0, C0}}; + {A,Alen,Tlen,_Ics1,L1, C1,_S1}-> Tcs1 = yysuf(Tcs, Alen), L2 = adjust_line(Tlen, Alen, Tcs1, L1), - string_cont(Tcs1, L2, yyaction(A, Alen, Tcs, L0), Ts) + C2 = adjust_col(Tcs, Alen, C1), + string_cont(Tcs1, L2, C2, yyaction(A, Alen, Tcs, L0,C0), Ts) end. -%% string_cont(RestChars, Line, Token, Tokens) +%% string_cont(RestChars, Line, Col, Token, Tokens) %% Test for and remove the end token wrapper. Push back characters %% are prepended to RestChars. --dialyzer({nowarn_function, string_cont/4}). +-dialyzer({nowarn_function, string_cont/5}). -string_cont(Rest, Line, {token,T}, Ts) -> - string(Rest, Line, Rest, [T|Ts]); -string_cont(Rest, Line, {token,T,Push}, Ts) -> +string_cont(Rest, Line, Col, {token,T}, Ts) -> + do_string(Rest, Line, Col, Rest, [T|Ts]); +string_cont(Rest, Line, Col, {token,T,Push}, Ts) -> NewRest = Push ++ Rest, - string(NewRest, Line, NewRest, [T|Ts]); -string_cont(Rest, Line, {end_token,T}, Ts) -> - string(Rest, Line, Rest, [T|Ts]); -string_cont(Rest, Line, {end_token,T,Push}, Ts) -> + do_string(NewRest, Line, Col, NewRest, [T|Ts]); +string_cont(Rest, Line, Col, {end_token,T}, Ts) -> + do_string(Rest, Line, Col, Rest, [T|Ts]); +string_cont(Rest, Line, Col, {end_token,T,Push}, Ts) -> NewRest = Push ++ Rest, - string(NewRest, Line, NewRest, [T|Ts]); -string_cont(Rest, Line, skip_token, Ts) -> - string(Rest, Line, Rest, Ts); -string_cont(Rest, Line, {skip_token,Push}, Ts) -> + do_string(NewRest, Line, Col, NewRest, [T|Ts]); +string_cont(Rest, Line, Col, skip_token, Ts) -> + do_string(Rest, Line, Col, Rest, Ts); +string_cont(Rest, Line, Col, {skip_token,Push}, Ts) -> NewRest = Push ++ Rest, - string(NewRest, Line, NewRest, Ts); -string_cont(_Rest, Line, {error,S}, _Ts) -> - {error,{Line,?MODULE,{user,S}},Line}. + do_string(NewRest, Line, Col, NewRest, Ts); +string_cont(_Rest, Line, Col, {error,S}, _Ts) -> + {error,{{Line, Col},?MODULE,{user,S}},{Line,Col}}. %% token(Continuation, Chars) -> -%% token(Continuation, Chars, Line) -> +%% token(Continuation, Chars, Loc) -> %% {more,Continuation} | {done,ReturnVal,RestChars}. %% Must be careful when re-entering to append the latest characters to the %% after characters in an accept. The continuation is: -%% {token,State,CurrLine,TokenChars,TokenLen,TokenLine,AccAction,AccLen} +%% {token,State,CurrLine,CurrCol,TokenChars,TokenLen,TokenLine,TokenCol,AccAction,AccLen} -token(Cont, Chars) -> token(Cont, Chars, 1). +##tkn -token([], Chars, Line) -> - token(yystate(), Chars, Line, Chars, 0, Line, reject, 0); -token({token,State,Line,Tcs,Tlen,Tline,Action,Alen}, Chars, _) -> - token(State, Chars, Line, Tcs ++ Chars, Tlen, Tline, Action, Alen). +do_token([], Chars, Line, Col) -> + token(yystate(), Chars, Line, Col, Chars, 0, Line, Col, reject, 0); +do_token({token,State,Line,Col,Tcs,Tlen,Tline,Tcol,Action,Alen}, Chars, _, _) -> + token(State, Chars, Line, Col, Tcs ++ Chars, Tlen, Tline, Tcol, Action, Alen). -%% token(State, InChars, Line, TokenChars, TokenLen, TokenLine, +%% token(State, InChars, Line, Col, TokenChars, TokenLen, TokenLine, TokenCol %% AcceptAction, AcceptLen) -> %% {more,Continuation} | {done,ReturnVal,RestChars}. %% The argument order is chosen to be more efficient. -token(S0, Ics0, L0, Tcs, Tlen0, Tline, A0, Alen0) -> - case yystate(S0, Ics0, L0, Tlen0, A0, Alen0) of +token(S0, Ics0, L0, C0, Tcs, Tlen0, Tline, Tcol, A0, Alen0) -> + case yystate(S0, Ics0, L0, C0, Tlen0, A0, Alen0) of %% Accepting end state, we have a token. - {A1,Alen1,Ics1,L1} -> - token_cont(Ics1, L1, yyaction(A1, Alen1, Tcs, Tline)); + {A1,Alen1,Ics1,L1,C1} -> + C2 = adjust_col(Tcs, Alen1, C1), + token_cont(Ics1, L1, C2, yyaction(A1, Alen1, Tcs, Tline,Tcol)); %% Accepting transition state, can take more chars. - {A1,Alen1,[],L1,S1} -> % Need more chars to check - {more,{token,S1,L1,Tcs,Alen1,Tline,A1,Alen1}}; - {A1,Alen1,Ics1,L1,_S1} -> % Take what we got - token_cont(Ics1, L1, yyaction(A1, Alen1, Tcs, Tline)); + {A1,Alen1,[],L1,C1,S1} -> % Need more chars to check + {more,{token,S1,L1,C1,Tcs,Alen1,Tline,Tcol,A1,Alen1}}; + {A1,Alen1,Ics1,L1,C1,_S1} -> % Take what we got + C2 = adjust_col(Tcs, Alen1, C1), + token_cont(Ics1, L1, C2, yyaction(A1, Alen1, Tcs, Tline,Tcol)); %% After a non-accepting state, maybe reach accept state later. - {A1,Alen1,Tlen1,[],L1,S1} -> % Need more chars to check - {more,{token,S1,L1,Tcs,Tlen1,Tline,A1,Alen1}}; - {reject,_Alen1,Tlen1,eof,L1,_S1} -> % No token match + {A1,Alen1,Tlen1,[],L1,C1,S1} -> % Need more chars to check + {more,{token,S1,L1,C1,Tcs,Tlen1,Tline,Tcol,A1,Alen1}}; + {reject,_Alen1,Tlen1,eof,L1,C1,_S1} -> % No token match %% Check for partial token which is error. - Ret = if Tlen1 > 0 -> {error,{Tline,?MODULE, + Ret = if Tlen1 > 0 -> {error,{{Tline,Tcol},?MODULE, %% Skip eof tail in Tcs. - {illegal,yypre(Tcs, Tlen1)}},L1}; - true -> {eof,L1} + {illegal,yypre(Tcs, Tlen1)}},{L1,C1}}; + true -> {eof,{L1,C1}} end, {done,Ret,eof}; - {reject,_Alen1,Tlen1,Ics1,L1,_S1} -> % No token match - Error = {Tline,?MODULE,{illegal,yypre(Tcs, Tlen1+1)}}, - {done,{error,Error,L1},Ics1}; - {A1,Alen1,Tlen1,_Ics1,L1,_S1} -> % Use last accept match + {reject,_Alen1,Tlen1,Ics1,_L1,_C1,_S1} -> % No token match + Error = {{Tline,Tcol},?MODULE,{illegal,yypre(Tcs, Tlen1+1)}}, + {done,{error,Error,{Tline,Tcol}},Ics1}; + {A1,Alen1,Tlen1,_Ics1,L1,_C1,_S1} -> % Use last accept match Tcs1 = yysuf(Tcs, Alen1), L2 = adjust_line(Tlen1, Alen1, Tcs1, L1), - token_cont(Tcs1, L2, yyaction(A1, Alen1, Tcs, Tline)) + C2 = C0 + Alen1, + token_cont(Tcs1, L2, C2, yyaction(A1, Alen1, Tcs, Tline, Tcol)) end. -%% token_cont(RestChars, Line, Token) +%% token_cont(RestChars, Line, Col, Token) %% If we have a token or error then return done, else if we have a %% skip_token then continue. --dialyzer({nowarn_function, token_cont/3}). +-dialyzer({nowarn_function, token_cont/4}). -token_cont(Rest, Line, {token,T}) -> - {done,{ok,T,Line},Rest}; -token_cont(Rest, Line, {token,T,Push}) -> +token_cont(Rest, Line, Col, {token,T}) -> + {done,{ok,T,{Line,Col}},Rest}; +token_cont(Rest, Line, Col, {token,T,Push}) -> NewRest = Push ++ Rest, - {done,{ok,T,Line},NewRest}; -token_cont(Rest, Line, {end_token,T}) -> - {done,{ok,T,Line},Rest}; -token_cont(Rest, Line, {end_token,T,Push}) -> + {done,{ok,T,{Line,Col}},NewRest}; +token_cont(Rest, Line, Col, {end_token,T}) -> + {done,{ok,T,{Line,Col}},Rest}; +token_cont(Rest, Line, Col, {end_token,T,Push}) -> NewRest = Push ++ Rest, - {done,{ok,T,Line},NewRest}; -token_cont(Rest, Line, skip_token) -> - token(yystate(), Rest, Line, Rest, 0, Line, reject, 0); -token_cont(Rest, Line, {skip_token,Push}) -> + {done,{ok,T,{Line,Col}},NewRest}; +token_cont(Rest, Line, Col, skip_token) -> + token(yystate(), Rest, Line, Col, Rest, 0, Line, Col, reject, 0); +token_cont(Rest, Line, Col, {skip_token,Push}) -> NewRest = Push ++ Rest, - token(yystate(), NewRest, Line, NewRest, 0, Line, reject, 0); -token_cont(Rest, Line, {error,S}) -> - {done,{error,{Line,?MODULE,{user,S}},Line},Rest}. + token(yystate(), NewRest, Line, Col, NewRest, 0, Line, Col, reject, 0); +token_cont(Rest, Line, Col, {error,S}) -> + {done,{error,{{Line, Col},?MODULE,{user,S}},{Line, Col}},Rest}. -%% tokens(Continuation, Chars, Line) -> +%% tokens(Continuation, Chars) -> +%% tokens(Continuation, Chars, Loc) -> %% {more,Continuation} | {done,ReturnVal,RestChars}. %% Must be careful when re-entering to append the latest characters to the %% after characters in an accept. The continuation is: -%% {tokens,State,CurrLine,TokenChars,TokenLen,TokenLine,Tokens,AccAction,AccLen} -%% {skip_tokens,State,CurrLine,TokenChars,TokenLen,TokenLine,Error,AccAction,AccLen} +%% {tokens,State,CurrLine,CurrCol,TokenChars,TokenLen,TokenLine,TokenCur,Tokens,AccAction,AccLen} +%% {skip_tokens,State,CurrLine,CurrCol,TokenChars,TokenLen,TokenLine,TokenCur,Error,AccAction,AccLen} -tokens(Cont, Chars) -> tokens(Cont, Chars, 1). +##tks -tokens([], Chars, Line) -> - tokens(yystate(), Chars, Line, Chars, 0, Line, [], reject, 0); -tokens({tokens,State,Line,Tcs,Tlen,Tline,Ts,Action,Alen}, Chars, _) -> - tokens(State, Chars, Line, Tcs ++ Chars, Tlen, Tline, Ts, Action, Alen); -tokens({skip_tokens,State,Line,Tcs,Tlen,Tline,Error,Action,Alen}, Chars, _) -> - skip_tokens(State, Chars, Line, Tcs ++ Chars, Tlen, Tline, Error, Action, Alen). +do_tokens([], Chars, Line, Col) -> + tokens(yystate(), Chars, Line, Col, Chars, 0, Line, Col, [], reject, 0); +do_tokens({tokens,State,Line,Col,Tcs,Tlen,Tline,Tcol,Ts,Action,Alen}, Chars, _,_) -> + tokens(State, Chars, Line, Col, Tcs ++ Chars, Tlen, Tline, Tcol, Ts, Action, Alen); +do_tokens({skip_tokens,State,Line, Col, Tcs,Tlen,Tline,Tcol,Error,Action,Alen}, Chars, _,_) -> + skip_tokens(State, Chars, Line, Col, Tcs ++ Chars, Tlen, Tline, Tcol, Error, Action, Alen). -%% tokens(State, InChars, Line, TokenChars, TokenLen, TokenLine, Tokens, +%% tokens(State, InChars, Line, Col, TokenChars, TokenLen, TokenLine, TokenCol,Tokens, %% AcceptAction, AcceptLen) -> %% {more,Continuation} | {done,ReturnVal,RestChars}. -tokens(S0, Ics0, L0, Tcs, Tlen0, Tline, Ts, A0, Alen0) -> - case yystate(S0, Ics0, L0, Tlen0, A0, Alen0) of +tokens(S0, Ics0, L0, C0, Tcs, Tlen0, Tline, Tcol, Ts, A0, Alen0) -> + case yystate(S0, Ics0, L0, C0, Tlen0, A0, Alen0) of %% Accepting end state, we have a token. - {A1,Alen1,Ics1,L1} -> - tokens_cont(Ics1, L1, yyaction(A1, Alen1, Tcs, Tline), Ts); + {A1,Alen1,Ics1,L1,C1} -> + C2 = adjust_col(Tcs, Alen1, C1), + tokens_cont(Ics1, L1, C2, yyaction(A1, Alen1, Tcs, Tline, Tcol), Ts); %% Accepting transition state, can take more chars. - {A1,Alen1,[],L1,S1} -> % Need more chars to check - {more,{tokens,S1,L1,Tcs,Alen1,Tline,Ts,A1,Alen1}}; - {A1,Alen1,Ics1,L1,_S1} -> % Take what we got - tokens_cont(Ics1, L1, yyaction(A1, Alen1, Tcs, Tline), Ts); + {A1,Alen1,[],L1,C1,S1} -> % Need more chars to check + {more,{tokens,S1,L1,C1,Tcs,Alen1,Tline,Tcol,Ts,A1,Alen1}}; + {A1,Alen1,Ics1,L1,C1,_S1} -> % Take what we got + C2 = adjust_col(Tcs, Alen1, C1), + tokens_cont(Ics1, L1, C2, yyaction(A1, Alen1, Tcs, Tline,Tcol), Ts); %% After a non-accepting state, maybe reach accept state later. - {A1,Alen1,Tlen1,[],L1,S1} -> % Need more chars to check - {more,{tokens,S1,L1,Tcs,Tlen1,Tline,Ts,A1,Alen1}}; - {reject,_Alen1,Tlen1,eof,L1,_S1} -> % No token match + {A1,Alen1,Tlen1,[],L1,C1,S1} -> % Need more chars to check + {more,{tokens,S1,L1,C1,Tcs,Tlen1,Tline,Tcol,Ts,A1,Alen1}}; + {reject,_Alen1,Tlen1,eof,L1,C1,_S1} -> % No token match %% Check for partial token which is error, no need to skip here. - Ret = if Tlen1 > 0 -> {error,{Tline,?MODULE, + Ret = if Tlen1 > 0 -> {error,{{Tline,Tcol},?MODULE, %% Skip eof tail in Tcs. - {illegal,yypre(Tcs, Tlen1)}},L1}; - Ts == [] -> {eof,L1}; - true -> {ok,yyrev(Ts),L1} + {illegal,yypre(Tcs, Tlen1)}},{L1,C1}}; + Ts == [] -> {eof,{L1,C1}}; + true -> {ok,yyrev(Ts),{L1,C1}} end, {done,Ret,eof}; - {reject,_Alen1,Tlen1,_Ics1,L1,_S1} -> + {reject,_Alen1,Tlen1,_Ics1,L1,C1,_S1} -> %% Skip rest of tokens. - Error = {L1,?MODULE,{illegal,yypre(Tcs, Tlen1+1)}}, - skip_tokens(yysuf(Tcs, Tlen1+1), L1, Error); - {A1,Alen1,Tlen1,_Ics1,L1,_S1} -> - Token = yyaction(A1, Alen1, Tcs, Tline), + Error = {{L1,C1},?MODULE,{illegal,yypre(Tcs, Tlen1+1)}}, + skip_tokens(yysuf(Tcs, Tlen1+1), L1, C1, Error); + {A1,Alen1,Tlen1,_Ics1,L1,_C1,_S1} -> + Token = yyaction(A1, Alen1, Tcs, Tline,Tcol), Tcs1 = yysuf(Tcs, Alen1), L2 = adjust_line(Tlen1, Alen1, Tcs1, L1), - tokens_cont(Tcs1, L2, Token, Ts) + C2 = C0 + Alen1, + tokens_cont(Tcs1, L2, C2, Token, Ts) end. -%% tokens_cont(RestChars, Line, Token, Tokens) +%% tokens_cont(RestChars, Line, Column, Token, Tokens) %% If we have an end_token or error then return done, else if we have %% a token then save it and continue, else if we have a skip_token %% just continue. --dialyzer({nowarn_function, tokens_cont/4}). +-dialyzer({nowarn_function, tokens_cont/5}). -tokens_cont(Rest, Line, {token,T}, Ts) -> - tokens(yystate(), Rest, Line, Rest, 0, Line, [T|Ts], reject, 0); -tokens_cont(Rest, Line, {token,T,Push}, Ts) -> +tokens_cont(Rest, Line, Col, {token,T}, Ts) -> + tokens(yystate(), Rest, Line, Col, Rest, 0, Line, Col, [T|Ts], reject, 0); +tokens_cont(Rest, Line, Col, {token,T,Push}, Ts) -> NewRest = Push ++ Rest, - tokens(yystate(), NewRest, Line, NewRest, 0, Line, [T|Ts], reject, 0); -tokens_cont(Rest, Line, {end_token,T}, Ts) -> - {done,{ok,yyrev(Ts, [T]),Line},Rest}; -tokens_cont(Rest, Line, {end_token,T,Push}, Ts) -> + tokens(yystate(), NewRest, Line, Col, NewRest, 0, Line, Col, [T|Ts], reject, 0); +tokens_cont(Rest, Line, Col, {end_token,T}, Ts) -> + {done,{ok,yyrev(Ts, [T]),{Line,Col}},Rest}; +tokens_cont(Rest, Line, Col, {end_token,T,Push}, Ts) -> NewRest = Push ++ Rest, - {done,{ok,yyrev(Ts, [T]),Line},NewRest}; -tokens_cont(Rest, Line, skip_token, Ts) -> - tokens(yystate(), Rest, Line, Rest, 0, Line, Ts, reject, 0); -tokens_cont(Rest, Line, {skip_token,Push}, Ts) -> + {done,{ok,yyrev(Ts, [T]),{Line, Col}},NewRest}; +tokens_cont(Rest, Line, Col, skip_token, Ts) -> + tokens(yystate(), Rest, Line, Col, Rest, 0, Line, Col, Ts, reject, 0); +tokens_cont(Rest, Line, Col, {skip_token,Push}, Ts) -> NewRest = Push ++ Rest, - tokens(yystate(), NewRest, Line, NewRest, 0, Line, Ts, reject, 0); -tokens_cont(Rest, Line, {error,S}, _Ts) -> - skip_tokens(Rest, Line, {Line,?MODULE,{user,S}}). + tokens(yystate(), NewRest, Line, Col, NewRest, 0, Line, Col, Ts, reject, 0); +tokens_cont(Rest, Line, Col, {error,S}, _Ts) -> + skip_tokens(Rest, Line, Col, {{Line,Col},?MODULE,{user,S}}). -%%skip_tokens(InChars, Line, Error) -> {done,{error,Error,Line},Ics}. +%% skip_tokens(InChars, Line, Col, Error) -> {done,{error,Error,{Line,Col}},Ics}. %% Skip tokens until an end token, junk everything and return the error. -skip_tokens(Ics, Line, Error) -> - skip_tokens(yystate(), Ics, Line, Ics, 0, Line, Error, reject, 0). +skip_tokens(Ics, Line, Col, Error) -> + skip_tokens(yystate(), Ics, Line, Col, Ics, 0, Line, Col, Error, reject, 0). -%% skip_tokens(State, InChars, Line, TokenChars, TokenLen, TokenLine, Tokens, +%% skip_tokens(State, InChars, Line, Col, TokenChars, TokenLen, TokenLine, TokenCol, Tokens, %% AcceptAction, AcceptLen) -> %% {more,Continuation} | {done,ReturnVal,RestChars}. -skip_tokens(S0, Ics0, L0, Tcs, Tlen0, Tline, Error, A0, Alen0) -> - case yystate(S0, Ics0, L0, Tlen0, A0, Alen0) of - {A1,Alen1,Ics1,L1} -> % Accepting end state - skip_cont(Ics1, L1, yyaction(A1, Alen1, Tcs, Tline), Error); - {A1,Alen1,[],L1,S1} -> % After an accepting state - {more,{skip_tokens,S1,L1,Tcs,Alen1,Tline,Error,A1,Alen1}}; - {A1,Alen1,Ics1,L1,_S1} -> - skip_cont(Ics1, L1, yyaction(A1, Alen1, Tcs, Tline), Error); - {A1,Alen1,Tlen1,[],L1,S1} -> % After a non-accepting state - {more,{skip_tokens,S1,L1,Tcs,Tlen1,Tline,Error,A1,Alen1}}; - {reject,_Alen1,_Tlen1,eof,L1,_S1} -> - {done,{error,Error,L1},eof}; - {reject,_Alen1,Tlen1,_Ics1,L1,_S1} -> - skip_tokens(yysuf(Tcs, Tlen1+1), L1, Error); - {A1,Alen1,Tlen1,_Ics1,L1,_S1} -> - Token = yyaction(A1, Alen1, Tcs, Tline), +skip_tokens(S0, Ics0, L0, C0, Tcs, Tlen0, Tline, Tcol, Error, A0, Alen0) -> + case yystate(S0, Ics0, L0, C0, Tlen0, A0, Alen0) of + {A1,Alen1,Ics1,L1, C1} -> % Accepting end state + skip_cont(Ics1, L1, C1, yyaction(A1, Alen1, Tcs, Tline, Tcol), Error); + {A1,Alen1,[],L1,C1, S1} -> % After an accepting state + {more,{skip_tokens,S1,L1,C1,Tcs,Alen1,Tline,Tcol,Error,A1,Alen1}}; + {A1,Alen1,Ics1,L1,C1,_S1} -> + skip_cont(Ics1, L1, C1, yyaction(A1, Alen1, Tcs, Tline, Tcol), Error); + {A1,Alen1,Tlen1,[],L1,C1,S1} -> % After a non-accepting state + {more,{skip_tokens,S1,L1,C1,Tcs,Tlen1,Tline,Tcol,Error,A1,Alen1}}; + {reject,_Alen1,_Tlen1,eof,L1,C1,_S1} -> + {done,{error,Error,{L1,C1}},eof}; + {reject,_Alen1,Tlen1,_Ics1,L1,C1,_S1} -> + skip_tokens(yysuf(Tcs, Tlen1+1), L1, C1,Error); + {A1,Alen1,Tlen1,_Ics1,L1,C1,_S1} -> + Token = yyaction(A1, Alen1, Tcs, Tline, Tcol), Tcs1 = yysuf(Tcs, Alen1), L2 = adjust_line(Tlen1, Alen1, Tcs1, L1), - skip_cont(Tcs1, L2, Token, Error) + skip_cont(Tcs1, L2, C1, Token, Error) end. -%% skip_cont(RestChars, Line, Token, Error) +%% skip_cont(RestChars, Line, Col, Token, Error) %% Skip tokens until we have an end_token or error then return done %% with the original rror. --dialyzer({nowarn_function, skip_cont/4}). +-dialyzer({nowarn_function, skip_cont/5}). -skip_cont(Rest, Line, {token,_T}, Error) -> - skip_tokens(yystate(), Rest, Line, Rest, 0, Line, Error, reject, 0); -skip_cont(Rest, Line, {token,_T,Push}, Error) -> +skip_cont(Rest, Line, Col, {token,_T}, Error) -> + skip_tokens(yystate(), Rest, Line, Col, Rest, 0, Line, Col, Error, reject, 0); +skip_cont(Rest, Line, Col, {token,_T,Push}, Error) -> NewRest = Push ++ Rest, - skip_tokens(yystate(), NewRest, Line, NewRest, 0, Line, Error, reject, 0); -skip_cont(Rest, Line, {end_token,_T}, Error) -> - {done,{error,Error,Line},Rest}; -skip_cont(Rest, Line, {end_token,_T,Push}, Error) -> + skip_tokens(yystate(), NewRest, Line, Col, NewRest, 0, Line, Col, Error, reject, 0); +skip_cont(Rest, Line, Col, {end_token,_T}, Error) -> + {done,{error,Error,{Line,Col}},Rest}; +skip_cont(Rest, Line, Col, {end_token,_T,Push}, Error) -> NewRest = Push ++ Rest, - {done,{error,Error,Line},NewRest}; -skip_cont(Rest, Line, skip_token, Error) -> - skip_tokens(yystate(), Rest, Line, Rest, 0, Line, Error, reject, 0); -skip_cont(Rest, Line, {skip_token,Push}, Error) -> + {done,{error,Error,{Line,Col}},NewRest}; +skip_cont(Rest, Line, Col, skip_token, Error) -> + skip_tokens(yystate(), Rest, Line, Col, Rest, 0, Line, Col, Error, reject, 0); +skip_cont(Rest, Line, Col, {skip_token,Push}, Error) -> NewRest = Push ++ Rest, - skip_tokens(yystate(), NewRest, Line, NewRest, 0, Line, Error, reject, 0); -skip_cont(Rest, Line, {error,_S}, Error) -> - skip_tokens(yystate(), Rest, Line, Rest, 0, Line, Error, reject, 0). + skip_tokens(yystate(), NewRest, Line, Col, NewRest, 0, Line, Col, Error, reject, 0); +skip_cont(Rest, Line, Col, {error,_S}, Error) -> + skip_tokens(yystate(), Rest, Line, Col, Rest, 0, Line, Col, Error, reject, 0). -compile({nowarn_unused_function, [yyrev/1, yyrev/2, yypre/2, yysuf/2]}). @@ -292,21 +300,44 @@ adjust_line(T, A, [$\n|Cs], L) -> adjust_line(T, A, [_|Cs], L) -> adjust_line(T-1, A, Cs, L). +%% adjust_col(Chars, AcceptLength, Col) -> NewCol +%% Handle newlines, tabs and unicode chars. +adjust_col(_, 0, Col) -> + Col; +adjust_col([$\n | R], L, _) -> + adjust_col(R, L-1, 1); +adjust_col([$\t | R], L, Col) -> + adjust_col(R, L-1, tab_forward(Col)+1); +adjust_col([C | R], L, Col) when C>=0 andalso C=< 16#7F -> + adjust_col(R, L-1, Col+1); +adjust_col([C | R], L, Col) when C>= 16#80 andalso C=< 16#7FF -> + adjust_col(R, L-1, Col+2); +adjust_col([C | R], L, Col) when C>= 16#800 andalso C=< 16#FFFF -> + adjust_col(R, L-1, Col+3); +adjust_col([C | R], L, Col) when C>= 16#10000 andalso C=< 16#10FFFF -> + adjust_col(R, L-1, Col+4). + +tab_forward(C) -> + D = C rem tab_size(), + A = tab_size()-D, + C+A. + +##tab_size + %% yystate() -> InitialState. -%% yystate(State, InChars, Line, CurrTokLen, AcceptAction, AcceptLen) -> -%% {Action, AcceptLen, RestChars, Line} | -%% {Action, AcceptLen, RestChars, Line, State} | -%% {reject, AcceptLen, CurrTokLen, RestChars, Line, State} | -%% {Action, AcceptLen, CurrTokLen, RestChars, Line, State}. +%% yystate(State, InChars, Line, Col, CurrTokLen, AcceptAction, AcceptLen) -> +%% {Action, AcceptLen, RestChars, Line, Col} | +%% {Action, AcceptLen, RestChars, Line, Col, State} | +%% {reject, AcceptLen, CurrTokLen, RestChars, Line, Col, State} | +%% {Action, AcceptLen, CurrTokLen, RestChars, Line, Col, State}. %% Generated state transition functions. The non-accepting end state %% return signal either an unrecognised character or end of current %% input. ##dfa -%% yyaction(Action, TokenLength, TokenChars, TokenLine) -> +%% yyaction(Action, TokenLength, TokenChars, TokenLine, TokenCol) -> %% {token,Token} | {end_token, Token} | skip_token | {error,String}. %% Generated action function. ##actions - diff --git a/lib/parsetools/src/leex.erl b/lib/parsetools/src/leex.erl index b764678516..c5e61b8149 100644 --- a/lib/parsetools/src/leex.erl +++ b/lib/parsetools/src/leex.erl @@ -119,6 +119,8 @@ file(File) -> file(File, []). | {'verbose', boolean()} | {'warnings_as_errors', boolean()} | {'deterministic', boolean()} + | {'error_location', line | column} + | {'tab_size', pos_integer()} | 'dfa_graph' | 'report_errors' | 'report_warnings' | 'report' | 'return_errors' | 'return_warnings' | 'return' @@ -281,6 +283,12 @@ check_options([{Option, Boolean} | Options], AllOptions, L) false -> badarg end; +check_options([{error_location, Loc}=O | Options], AllOptions, L) + when Loc =:= line; Loc =:= column -> + check_options(Options, AllOptions, [O | L]); +check_options([{tab_size, S}=O | Options], AllOptions, L) + when is_integer(S) andalso S>0 -> + check_options(Options, AllOptions, [O | L]); check_options([], _AllOptions, L) -> L; check_options(_Options, _, _L) -> @@ -289,7 +297,7 @@ check_options(_Options, _, _L) -> all_options() -> [dfa_graph,includefile,report_errors,report_warnings, return_errors,return_warnings,scannerfile,verbose, - warnings_as_errors, deterministic]. + warnings_as_errors,deterministic,error_location,tab_size]. default_option(dfa_graph) -> false; default_option(includefile) -> []; @@ -300,7 +308,9 @@ default_option(return_warnings) -> false; default_option(scannerfile) -> []; default_option(verbose) -> false; default_option(warnings_as_errors) -> false; -default_option(deterministic) -> false. +default_option(deterministic) -> false; +default_option(error_location) -> line; +default_option(tab_size) -> 8. atom_option(dfa_graph) -> {dfa_graph,true}; atom_option(report_errors) -> {report_errors,true}; @@ -596,7 +606,9 @@ parse_rule(S, Line, Atoks, Ms, N, St) -> TokenChars = var_used('TokenChars', Atoks), TokenLen = var_used('TokenLen', Atoks), TokenLine = var_used('TokenLine', Atoks), - {ok,{R,N},{N,Atoks,TokenChars,TokenLen,TokenLine},St}; + TokenCol = var_used('TokenCol', Atoks), + TokenLoc = var_used('TokenLoc', Atoks), + {ok,{R,N},{N,Atoks,TokenChars,TokenLen,TokenLine,TokenCol,TokenLoc},St}; {error,E} -> add_error({Line,leex,E}, St) end. @@ -1415,6 +1427,10 @@ out_file(Ifile, Ofile, St, DFA, DF, Actions, Code, L) -> case string:slice(Line, 0, 5) of "##mod" -> out_module(Ofile, St); "##cod" -> out_erlang_code(Ofile, St, Code, L); + "##str" -> out_string(Ofile, St#leex.opts); + "##tkn" -> out_token(Ofile, St#leex.opts); + "##tks" -> out_tokens(Ofile, St#leex.opts); + "##tab" -> out_tab_size(Ofile, St#leex.opts); "##dfa" -> out_dfa(Ofile, St, DFA, Code, DF, L); "##act" -> out_actions(Ofile, St#leex.xfile, Deterministic, Actions); _ -> io:put_chars(Ofile, Line) @@ -1440,6 +1456,92 @@ out_erlang_code(File, St, Code, L) -> io:nl(File), output_file_directive(File, St#leex.ifile, Deterministic, L). +out_tab_size(File, Opts) -> + Size = proplists:get_value(tab_size, Opts), + io:fwrite(File, "tab_size() -> ~p.\n", [Size]). + +%% Exclude column number if needed +out_string(File, Opts) -> + out_string_1(File, Opts), + out_string_2(File, Opts), + Vars = lists:join(", ",["Ics","L0","C0","Tcs","Ts"]), + out_head(File,string,Vars), + EL = proplists:get_value(error_location, Opts), + case EL of + column -> + io:fwrite(File," do_string(~s).\n",[Vars]); + line -> + io:fwrite(File," case do_string(~s) of\n",[Vars]), + io:fwrite(File," {ok, T, {L,_}} -> {ok, T, L};\n",[]), + io:fwrite(File," {error, {{EL,_},M,D}, {L,_}} ->\n",[]), + io:fwrite(File," EI = {EL,M,D},\n",[]), + io:fwrite(File," {error, EI, L}\n",[]), + io:fwrite(File," end.\n",[]) + end. + +out_string_1(File, Opts) -> + out_head(File,string,"Ics"), + EL = proplists:get_value(error_location, Opts), + DefLoc = case EL of + column -> "{1,1}"; + line -> "1" + end, + io:fwrite(File," string(~s).\n",["Ics,"++DefLoc]). + +out_string_2(File, Opts) -> + EL = proplists:get_value(error_location, Opts), + case EL of + column -> + out_head(File,string,"Ics,{L0,C0}"), + CallVars = lists:join(", ", ["Ics","L0","C0","Ics","[]"]), + io:fwrite(File," string(~s).\n",[CallVars]); + line -> + out_head(File,string,"Ics,L0"), + CallVars = lists:join(", ", ["Ics","L0","1","Ics","[]"]), + io:fwrite(File," string(~s).\n",[CallVars]) + end. + +out_token(File, Opts) -> + out_tokens_wrapper(File, Opts, token). + +out_tokens(File, Opts) -> + out_tokens_wrapper(File, Opts, tokens). + +out_tokens_wrapper(File, Opts, Fun) -> + out_token_2(File, Opts, Fun), + EL = proplists:get_value(error_location, Opts), + case EL of + column -> + VarsCol = lists:join(", ",["Cont","Chars","{Line,Col}"]), + out_head(File, Fun, VarsCol), + io:fwrite(File," do_~s(~s).\n",[Fun,"Cont,Chars,Line,Col"]); + line -> + VarsCol = lists:join(", ",["Cont","Chars","Line"]), + out_head(File, Fun, VarsCol), + io:fwrite(File," case do_~s(~s) of\n",[Fun,"Cont,Chars,Line,1"]), + io:fwrite(File," {more, _} = C -> C;\n",[]), + io:fwrite(File," {done, Ret0, R} ->\n",[]), + io:fwrite(File," Ret1 = case Ret0 of\n",[]), + io:fwrite(File," {ok, T, {L,_}} -> {ok, T, L};\n",[]), + io:fwrite(File," {eof, {L,_}} -> {eof, L};\n",[]), + io:fwrite(File," {error, {{EL,_},M,D},{L,_}} -> {error, {EL,M,D},L}\n",[]), + io:fwrite(File," end,\n",[]), + io:fwrite(File," {done, Ret1, R}\n",[]), + io:fwrite(File," end.\n",[]) + end. + +out_token_2(File, Opts, Fun) -> + out_head(File, Fun, "Cont,Chars"), + EL = proplists:get_value(error_location, Opts), + DefLoc = case EL of + column -> "{1,1}"; + line -> "1" + end, + io:fwrite(File," ~s(~s).\n",[Fun,"Cont,Chars,"++DefLoc]). + +out_head(File, Fun, Vars) -> + io:fwrite(File, "~s(~s) -> \n",[Fun,Vars]). + file_copy(From, To) -> case io:get_line(From, leex) of eof -> ok; @@ -1455,36 +1557,36 @@ out_dfa(File, St, DFA, Code, DF, L) -> output_file_directive(File, St#leex.efile, Deterministic, L+(NCodeLines-1)+3), io:fwrite(File, "yystate() -> ~w.~n~n", [DF]), foreach(fun (S) -> out_trans(File, S) end, DFA), - io:fwrite(File, "yystate(S, Ics, Line, Tlen, Action, Alen) ->~n", []), - io:fwrite(File, " {Action,Alen,Tlen,Ics,Line,S}.~n", []). + io:fwrite(File, "yystate(S, Ics, Line, Col, Tlen, Action, Alen) ->~n", []), + io:fwrite(File, " {Action,Alen,Tlen,Ics,Line,Col,S}.~n", []). out_trans(File, #dfa_state{no=N,trans=[],accept={accept,A}}) -> %% Accepting end state, guaranteed done. - io:fwrite(File, "yystate(~w, Ics, Line, Tlen, _, _) ->~n", [N]), - io:fwrite(File, " {~w,Tlen,Ics,Line};~n", [A]); + io:fwrite(File, "yystate(~w, Ics, Line, Col, Tlen, _, _) ->~n", [N]), + io:fwrite(File, " {~w,Tlen,Ics,Line,Col};~n", [A]); out_trans(File, #dfa_state{no=N,trans=Tr,accept={accept,A}}) -> %% Accepting state, but there maybe more. foreach(fun (T) -> out_accept_tran(File, N, A, T) end, pack_trans(Tr)), - io:fwrite(File, "yystate(~w, Ics, Line, Tlen, _, _) ->~n", [N]), - io:fwrite(File, " {~w,Tlen,Ics,Line,~w};~n", [A,N]); + io:fwrite(File, "yystate(~w, Ics, Line, Col, Tlen, _, _) ->~n", [N]), + io:fwrite(File, " {~w,Tlen,Ics,Line,Col,~w};~n", [A,N]); out_trans(File, #dfa_state{no=N,trans=Tr,accept=noaccept}) -> %% Non-accepting transition state. foreach(fun (T) -> out_noaccept_tran(File, N, T) end, pack_trans(Tr)), - io:fwrite(File, "yystate(~w, Ics, Line, Tlen, Action, Alen) ->~n", [N]), - io:fwrite(File, " {Action,Alen,Tlen,Ics,Line,~w};~n", [N]). + io:fwrite(File, "yystate(~w, Ics, Line, Col, Tlen, Action, Alen) ->~n", [N]), + io:fwrite(File, " {Action,Alen,Tlen,Ics,Line,Col,~w};~n", [N]). out_accept_tran(File, N, A, {{Cf,maxchar},S}) -> out_accept_head_max(File, N, Cf), - out_accept_body(File, S, "Line", A); + out_accept_body(File, S, "Line", "Col", A); out_accept_tran(File, N, A, {{Cf,Cl},S}) -> out_accept_head_range(File, N, Cf, Cl), - out_accept_body(File, S, "Line", A); + out_accept_body(File, S, "Line", "Col", A); out_accept_tran(File, N, A, {$\n,S}) -> out_accept_head_1(File, N, $\n), - out_accept_body(File, S, "Line+1", A); + out_accept_body(File, S, "Line+1", "1", A); out_accept_tran(File, N, A, {C,S}) -> out_accept_head_1(File, N, C), - out_accept_body(File, S, "Line", A). + out_accept_body(File, S, "Line", "Col", A). out_accept_head_1(File, State, Char) -> out_head_1(File, State, Char, "_", "_"). @@ -1495,21 +1597,21 @@ out_accept_head_max(File, State, Min) -> out_accept_head_range(File, State, Min, Max) -> out_head_range(File, State, Min, Max, "_", "_"). -out_accept_body(File, Next, Line, Action) -> - out_body(File, Next, Line, io_lib:write(Action), "Tlen"). +out_accept_body(File, Next, Line, Col, Action) -> + out_body(File, Next, Line, Col, io_lib:write(Action), "Tlen"). out_noaccept_tran(File, N, {{Cf,maxchar},S}) -> out_noaccept_head_max(File, N, Cf), - out_noaccept_body(File, S, "Line"); + out_noaccept_body(File, S, "Line", "Col"); out_noaccept_tran(File, N, {{Cf,Cl},S}) -> out_noaccept_head_range(File, N, Cf, Cl), - out_noaccept_body(File, S, "Line"); + out_noaccept_body(File, S, "Line", "Col"); out_noaccept_tran(File, N, {$\n,S}) -> out_noaccept_head_1(File, N, $\n), - out_noaccept_body(File, S, "Line+1"); + out_noaccept_body(File, S, "Line+1", "1"); out_noaccept_tran(File, N, {C,S}) -> out_noaccept_head_1(File, N, C), - out_noaccept_body(File, S, "Line"). + out_noaccept_body(File, S, "Line", "Col"). out_noaccept_head_1(File, State, Char) -> out_head_1(File, State, Char, "Action", "Alen"). @@ -1520,24 +1622,27 @@ out_noaccept_head_max(File, State, Min) -> out_noaccept_head_range(File, State, Min, Max) -> out_head_range(File, State, Min, Max, "Action", "Alen"). -out_noaccept_body(File, Next, Line) -> - out_body(File, Next, Line, "Action", "Alen"). +out_noaccept_body(File, Next, Line, Col) -> + out_body(File, Next, Line, Col, "Action", "Alen"). +out_head_1(File, State, Char = $\n, Action, Alen) -> + io:fwrite(File, "yystate(~w, [~w|Ics], Line, _, Tlen, ~s, ~s) ->\n", + [State,Char,Action,Alen]); out_head_1(File, State, Char, Action, Alen) -> - io:fwrite(File, "yystate(~w, [~w|Ics], Line, Tlen, ~s, ~s) ->\n", + io:fwrite(File, "yystate(~w, [~w|Ics], Line, Col, Tlen, ~s, ~s) ->\n", [State,Char,Action,Alen]). out_head_max(File, State, Min, Action, Alen) -> - io:fwrite(File, "yystate(~w, [C|Ics], Line, Tlen, ~s, ~s) when C >= ~w ->\n", + io:fwrite(File, "yystate(~w, [C|Ics], Line, Col, Tlen, ~s, ~s) when C >= ~w ->\n", [State,Action,Alen,Min]). out_head_range(File, State, Min, Max, Action, Alen) -> - io:fwrite(File, "yystate(~w, [C|Ics], Line, Tlen, ~s, ~s) when C >= ~w, C =< ~w ->\n", + io:fwrite(File, "yystate(~w, [C|Ics], Line, Col, Tlen, ~s, ~s) when C >= ~w, C =< ~w ->\n", [State,Action,Alen,Min,Max]). -out_body(File, Next, Line, Action, Alen) -> - io:fwrite(File, " yystate(~w, Ics, ~s, Tlen+1, ~s, ~s);\n", - [Next,Line,Action,Alen]). +out_body(File, Next, Line, Col, Action, Alen) -> + io:fwrite(File, " yystate(~w, Ics, ~s, ~s, Tlen+1, ~s, ~s);\n", + [Next,Line,Col,Action,Alen]). %% pack_trans([{Crange,State}]) -> [{Crange,State}] when %% Crange = {Char,Char} | Char. @@ -1581,31 +1686,32 @@ pack_trans([], Pt) -> Pt. out_actions(File, XrlFile, Deterministic, As) -> As1 = prep_out_actions(As), foreach(fun (A) -> out_action(File, A) end, As1), - io:fwrite(File, "yyaction(_, _, _, _) -> error.~n", []), + io:fwrite(File, "yyaction(_, _, _, _, _) -> error.~n", []), foreach(fun (A) -> out_action_code(File, XrlFile, Deterministic, A) end, As1). prep_out_actions(As) -> map(fun ({A,empty_action}) -> {A,empty_action}; - ({A,Code,TokenChars,TokenLen,TokenLine}) -> + ({A,Code,TokenChars,TokenLen,TokenLine,TokenCol,TokenLoc}) -> Vs = [{TokenChars,"TokenChars"}, {TokenLen,"TokenLen"}, - {TokenLine,"TokenLine"}, + {TokenLine or TokenLoc,"TokenLine"}, + {TokenCol or TokenLoc,"TokenCol"}, {TokenChars,"YYtcs"}, {TokenLen or TokenChars,"TokenLen"}], Vars = [if F -> S; true -> "_" end || {F,S} <- Vs], Name = list_to_atom(lists:concat([yyaction_,A])), - [Chars,Len,Line,_,_] = Vars, - Args = [V || V <- [Chars,Len,Line], V =/= "_"], + [Chars,Len,Line,Col,_,_] = Vars, + Args = [V || V <- [Chars,Len,Line,Col], V =/= "_"], ArgsChars = lists:join(", ", Args), - {A,Code,Vars,Name,Args,ArgsChars} + {A,Code,Vars,Name,Args,ArgsChars, TokenLoc} end, As). out_action(File, {A,empty_action}) -> - io:fwrite(File, "yyaction(~w, _, _, _) -> skip_token;~n", [A]); -out_action(File, {A,_Code,Vars,Name,_Args,ArgsChars}) -> - [_,_,Line,Tcs,Len] = Vars, - io:fwrite(File, "yyaction(~w, ~s, ~s, ~s) ->~n", [A,Len,Tcs,Line]), + io:fwrite(File, "yyaction(~w, _, _, _, _) -> skip_token;~n", [A]); +out_action(File, {A,_Code,Vars,Name,_Args,ArgsChars,_TokenLoc}) -> + [_,_,Line,Col,Tcs,Len] = Vars, + io:fwrite(File, "yyaction(~w, ~s, ~s, ~s, ~s) ->~n", [A,Len,Tcs,Line,Col]), if Tcs =/= "_" -> io:fwrite(File, " TokenChars = yypre(YYtcs, TokenLen),~n", []); @@ -1615,13 +1721,17 @@ out_action(File, {A,_Code,Vars,Name,_Args,ArgsChars}) -> out_action_code(_File, _XrlFile, _Deterministic, {_A,empty_action}) -> ok; -out_action_code(File, XrlFile, Deterministic, {_A,Code,_Vars,Name,Args,ArgsChars}) -> +out_action_code(File, XrlFile, Deterministic, {_A,Code,_Vars,Name,Args,ArgsChars, TokenLoc}) -> %% Should set the file to the .erl file, but instead assumes that %% ?LEEXINC is syntactically correct. io:fwrite(File, "\n-compile({inline,~w/~w}).\n", [Name, length(Args)]), L = erl_scan:line(hd(Code)), output_file_directive(File, XrlFile, Deterministic, L-2), io:fwrite(File, "~s(~s) ->~n", [Name, ArgsChars]), + if + TokenLoc -> io:fwrite(File," TokenLoc={TokenLine,TokenCol},~n",[]); + true -> ok + end, io:fwrite(File, " ~ts\n", [pp_tokens(Code, L, File)]). %% pp_tokens(Tokens, Line, File) -> [char()]. diff --git a/lib/parsetools/test/leex_SUITE.erl b/lib/parsetools/test/leex_SUITE.erl index 8d7e44629c..92e74034b5 100644 --- a/lib/parsetools/test/leex_SUITE.erl +++ b/lib/parsetools/test/leex_SUITE.erl @@ -45,7 +45,7 @@ pt/1, man/1, ex/1, ex2/1, not_yet/1, line_wrap/1, otp_10302/1, otp_11286/1, unicode/1, otp_13916/1, otp_14285/1, - otp_17023/1, compiler_warnings/1]). + otp_17023/1, compiler_warnings/1, column_support/1]). % Default timetrap timeout (set in init_per_testcase). -define(default_timeout, test_server:minutes(1)). @@ -66,7 +66,7 @@ all() -> groups() -> [{checks, [], [file, compile, syntax, deterministic]}, - {examples, [], [pt, man, ex, ex2, not_yet, unicode]}, + {examples, [], [pt, man, ex, ex2, not_yet, unicode, column_support]}, {tickets, [], [otp_10302, otp_11286, otp_13916, otp_14285, otp_17023, compiler_warnings]}, {bugs, [], [line_wrap]}]. @@ -118,6 +118,17 @@ file(Config) when is_list(Config) -> {'EXIT', {badarg, _}} = (catch leex:file(Filename, includefile)), + {'EXIT', {badarg, _}} = + (catch leex:file(Filename, {tab_size,0})), + {'EXIT', {badarg, _}} = + (catch leex:file(Filename, {tab_size,"4"})), + {'EXIT', {badarg, _}} = + (catch leex:file(Filename, {tab_size,3.5})), + {'EXIT', {badarg, _}} = + (catch leex:file(Filename, {error_location,{line,column}})), + {'EXIT', {badarg, _}} = + (catch leex:file(Filename, {error_location,col})), + Mini = <<"Definitions.\n" "D = [0-9]\n" "Rules.\n" @@ -417,17 +428,17 @@ pt(Config) when is_list(Config) -> "L = [a-z]\n" "Rules.\n" - "{L}+ : {token,{word,TokenLine,TokenChars}}.\n" + "{L}+ : {token,{word,TokenLoc,TokenChars}}.\n" "abc{D}+ : {skip_token,\"sture\" ++ string:substr(TokenChars, 4)}.\n" - "{D}+ : {token,{integer,TokenLine,list_to_integer(TokenChars)}}.\n" + "{D}+ : {token,{integer,TokenLoc,list_to_integer(TokenChars)}}.\n" "\\s : .\n" "\\r\\n : {end_token,{crlf,TokenLine}}.\n" "Erlang code.\n" "-export([t/0]).\n" "t() -> - {ok,[{word,1,\"sture\"},{integer,1,123}],1} = - string(\"abc123\"), ok. ">>, + {ok,[{word,{1,7},\"sture\"},{integer,{1,12},123}],{1,15}} = + string(\"abc123\"), ok. ">>, default, ok}], @@ -442,10 +453,10 @@ unicode(Config) when is_list(Config) -> "Definitions.\n" "RTLarrow = (â)\n" "Rules.\n" - "{RTLarrow} : {token,{\"â\",TokenLine}}.\n" + "{RTLarrow} : {token,{\"â\",TokenLoc}}.\n" "Erlang code.\n" "-export([t/0]).\n" - "t() -> {ok, [{\"â\", 1}], 1} = string(\"â\"), ok.">>, + "t() -> {ok, [{\"â\", {1,1}}], {1,4}} = string(\"â\"), ok.">>, default, ok}], @@ -460,34 +471,33 @@ man(Config) when is_list(Config) -> <<"Definitions.\n" "Rules.\n" "[a-z][0-9a-zA-Z_]* :\n" - " {token,{atom,TokenLine,list_to_atom(TokenChars)}}.\n" + " {token,{atom,TokenLoc,list_to_atom(TokenChars)}}.\n" "[A-Z_][0-9a-zA-Z_]* :\n" - " {token,{var,TokenLine,list_to_atom(TokenChars)}}.\n" + " {token,{var,TokenLoc,list_to_atom(TokenChars)}}.\n" "(\\+|-)?[0-9]+\\.[0-9]+((E|e)(\\+|-)?[0-9]+)? : \n" - " {token,{float,TokenLine,list_to_float(TokenChars)}}.\n" + " {token,{float,TokenLoc,list_to_float(TokenChars)}}.\n" "\\s : skip_token.\n" "Erlang code.\n" "-export([t/0]).\n" "t() ->\n" - " {ok,[{float,1,3.14},{atom,1,atom},{var,1,'V314'}],1} =\n" + " {ok,[{float,{1,1},3.14},{atom,{1,5},atom},{var,{1,10},'V314'}],{1,14}} =\n" " string(\"3.14atom V314\"),\n" " ok.\n">>, default, ok}, - - {man_2, + {man_2, <<"Definitions.\n" "D = [0-9]\n" "Rules.\n" "{D}+ :\n" - " {token,{integer,TokenLine,list_to_integer(TokenChars)}}.\n" + " {token,{integer,TokenLoc,list_to_integer(TokenChars)}}.\n" "{D}+\\.{D}+((E|e)(\\+|\\-)?{D}+)? :\n" - " {token,{float,TokenLine,list_to_float(TokenChars)}}.\n" + " {token,{float,TokenLoc,list_to_float(TokenChars)}}.\n" "\\s : skip_token.\n" "Erlang code.\n" "-export([t/0]).\n" "t() ->\n" - " {ok,[{float,1,3.14},{integer,1,314}],1} = \n" + " {ok,[{float,{1,1},3.14},{integer,{1,6},314}],{1,9}} = \n" " string(\"3.14 314\"),\n" " ok.\n">>, default, @@ -505,13 +515,13 @@ ex(Config) when is_list(Config) -> "D = [0-543-705-982]\n" "Rules.\n" "{D}+ :\n" - " {token,{integer,TokenLine,list_to_integer(TokenChars)}}.\n" + " {token,{integer,TokenLoc,list_to_integer(TokenChars)}}.\n" "[^235]+ :\n" - " {token,{list_to_atom(TokenChars),TokenLine}}.\n" + " {token,{list_to_atom(TokenChars),TokenLoc}}.\n" "Erlang code.\n" "-export([t/0]).\n" "t() ->\n" - " {ok,[{integer,1,12},{' c\\na',1},{integer,2,34},{b789a,2}],2} =\n" + " {ok,[{integer,{1,1},12},{' c\\na',{1,3}},{integer,{2,2},34},{b789a,{2,4}}],{2,9}} =\n" " string(\"12 c\\na34b789a\"),\n" " ok.\n">>, default, @@ -528,7 +538,7 @@ ex(Config) when is_list(Config) -> "Erlang code.\n" "-export([t/0]).\n" "t() ->\n" - " {ok,[chars,zyx],1} = string(\"abcdef zyx123\"),\n" + " {ok,[chars,zyx],{1,14}} = string(\"abcdef zyx123\"),\n" " ok.\n">>, default, ok}, @@ -541,7 +551,7 @@ ex(Config) when is_list(Config) -> "Erlang code.\n" "-export([t/0]).\n" "t() ->\n" - " {ok,[],1} = string(\"\"), ok.\n">>, % string("a") would loop... + " {ok,[],{1,1}} = string(\"\"), ok.\n">>, % string("a") would loop... default, ok}, @@ -574,12 +584,12 @@ ex(Config) when is_list(Config) -> "Erlang code.\n" "-export([t/0]).\n" "t() ->\n" - " {ok,[{white,\"\\b\\f\"}],1} = string(\"\\b\\f\"),\n" - " {ok,[{form,\"ff\\f\"}],1} = string(\"ff\\f\"),\n" - " {ok,[{string,\"\\\"foo\\\"\"}],1} = string(\"\\\"foo\\\"\"),\n" - " {ok,[{char,\"$.\"}],1} = string(\"$\\.\"),\n" - " {ok,[{list,\"[a,b,c]\"}],1} = string(\"[a,b,c]\"),\n" - " {ok,[{other,\"$^\\\\\"}],1} = string(\"$^\\\\\"),\n" + " {ok,[{white,\"\\b\\f\"}],{1,3}} = string(\"\\b\\f\"),\n" + " {ok,[{form,\"ff\\f\"}],{1,4}} = string(\"ff\\f\"),\n" + " {ok,[{string,\"\\\"foo\\\"\"}],{1,6}} = string(\"\\\"foo\\\"\"),\n" + " {ok,[{char,\"$.\"}],{1,3}} = string(\"$\\.\"),\n" + " {ok,[{list,\"[a,b,c]\"}],{1,8}} = string(\"[a,b,c]\"),\n" + " {ok,[{other,\"$^\\\\\"}],{1,4}} = string(\"$^\\\\\"),\n" " ok.\n">>, default, ok}, @@ -607,7 +617,7 @@ ex(Config) when is_list(Config) -> "Erlang code.\n" "-export([t/0]).\n" "t() ->\n" - " {ok,[{hex,[17,171,48,172]}],1} =\n" + " {ok,[{hex,[17,171,48,172]}],{1,7}} =\n" " string(\"\\x{11}\\xab0\\xac\"),\n" " ok.\n">>, default, @@ -637,47 +647,47 @@ WS = ([\\000-\\s]|%.*) Rules. {D}+\\.{D}+((E|e)(\\+|\\-)?{D}+)? : - {token,{float,TokenLine,list_to_float(TokenChars)}}. -{D}+#{H}+ : base(TokenLine, TokenChars). -{D}+ : {token,{integer,TokenLine,list_to_integer(TokenChars)}}. + {token,{float,TokenLoc,list_to_float(TokenChars)}}. +{D}+#{H}+ : base(TokenLoc, TokenChars). +{D}+ : {token,{integer,TokenLoc,list_to_integer(TokenChars)}}. {L}{A}* : Atom = list_to_atom(TokenChars), {token,case reserved_word(Atom) of - true -> {Atom,TokenLine}; - false -> {atom,TokenLine,Atom} + true -> {Atom,TokenLoc}; + false -> {atom,TokenLoc,Atom} end}. '(\\\\\\^.|\\\\.|[^'])*' : %% Strip quotes. S = lists:sublist(TokenChars, 2, TokenLen - 2), case catch list_to_atom(string_gen(S)) of {'EXIT',_} -> {error,\"illegal atom \" ++ TokenChars}; - Atom -> {token,{atom,TokenLine,Atom}} + Atom -> {token,{atom,TokenLoc,Atom}} end. -({U}|_){A}* : {token,{var,TokenLine,list_to_atom(TokenChars)}}. +({U}|_){A}* : {token,{var,TokenLoc,list_to_atom(TokenChars)}}. \"(\\\\\\^.|\\\\.|[^\"])*\" : %% Strip quotes. S = lists:sublist(TokenChars, 2, TokenLen - 2), - {token,{string,TokenLine,string_gen(S)}}. + {token,{string,TokenLoc,string_gen(S)}}. \\$(\\\\{O}{O}{O}|\\\\\\^.|\\\\.|.) : - {token,{char,TokenLine,cc_convert(TokenChars)}}. --> : {token,{'->',TokenLine}}. -:- : {token,{':-',TokenLine}}. -\\|\\| : {token,{'||',TokenLine}}. -<- : {token,{'<-',TokenLine}}. -\\+\\+ : {token,{'++',TokenLine}}. --- : {token,{'--',TokenLine}}. -=/= : {token,{'=/=',TokenLine}}. -== : {token,{'==',TokenLine}}. -=:= : {token,{'=:=',TokenLine}}. -/= : {token,{'/=',TokenLine}}. ->= : {token,{'>=',TokenLine}}. -=< : {token,{'=<',TokenLine}}. -<= : {token,{'<=',TokenLine}}. -<< : {token,{'<<',TokenLine}}. ->> : {token,{'>>',TokenLine}}. -:: : {token,{'::',TokenLine}}. + {token,{char,TokenLoc,cc_convert(TokenChars)}}. +-> : {token,{'->',TokenLoc}}. +:- : {token,{':-',TokenLoc}}. +\\|\\| : {token,{'||',TokenLoc}}. +<- : {token,{'<-',TokenLoc}}. +\\+\\+ : {token,{'++',TokenLoc}}. +-- : {token,{'--',TokenLoc}}. +=/= : {token,{'=/=',TokenLoc}}. +== : {token,{'==',TokenLoc}}. +=:= : {token,{'=:=',TokenLoc}}. +/= : {token,{'/=',TokenLoc}}. +>= : {token,{'>=',TokenLoc}}. +=< : {token,{'=<',TokenLoc}}. +<= : {token,{'<=',TokenLoc}}. +<< : {token,{'<<',TokenLoc}}. +>> : {token,{'>>',TokenLoc}}. +:: : {token,{'::',TokenLoc}}. []()[}{|!?/;:,.*+#<>=-] : - {token,{list_to_atom(TokenChars),TokenLine}}. -\\.{WS} : {end_token,{dot,TokenLine}}. + {token,{list_to_atom(TokenChars),TokenLoc}}. +\\.{WS} : {end_token,{dot,TokenLoc}}. {WS}+ : skip_token. Erlang code. @@ -775,7 +785,7 @@ escape_char(C) -> C. XrlFile = filename:join(Dir, "erlang_scan.xrl"), ok = file:write_file(XrlFile, Xrl), ErlFile = filename:join(Dir, "erlang_scan.erl"), - {ok, _} = leex:file(XrlFile, []), + {ok, _} = leex:file(XrlFile, [{error_location, column}]), {ok, _} = compile:file(ErlFile, [{outdir,Dir}]), code:purge(erlang_scan), AbsFile = filename:rootname(ErlFile, ".erl"), @@ -785,79 +795,79 @@ escape_char(C) -> C. erlang_scan:tokens(Cont, Chars, Location) end, F1 = fun(Cont, Chars, Location) -> - erlang_scan:token(Cont, Chars, Location) - end, + erlang_scan:token(Cont, Chars, Location) + end, fun() -> S = "ab cd. ", - {ok, Ts, 1} = scan_tokens_1(S, F, 1), - {ok, Ts, 1} = scan_token_1(S, F1, 1), - {ok, Ts, 1} = scan_tokens(S, F, 1), - {ok, Ts, 1} = erlang_scan:string(S, 1) + {ok, Ts, {1,8}} = scan_tokens_1(S, F, {1,1}), + {ok, Ts, {1,8}} = scan_token_1(S, F1, {1,1}), + {ok, Ts, {1,8}} = scan_tokens(S, F, {1,1}), + {ok, Ts, {1,8}} = erlang_scan:string(S, {1,1}) end(), fun() -> S = "'ab\n cd'. ", - {ok, Ts, 2} = scan_tokens_1(S, F, 1), - {ok, Ts, 2} = scan_token_1(S, F1, 1), - {ok, Ts, 2} = scan_tokens(S, F, 1), - {ok, Ts, 2} = erlang_scan:string(S, 1) + {ok, Ts, {2,7}} = scan_tokens_1(S, F, {1,1}), + {ok, Ts, {2,7}} = scan_token_1(S, F1, {1,1}), + {ok, Ts, {2,7}} = scan_tokens(S, F, {1,1}), + {ok, Ts, {2,7}} = erlang_scan:string(S, {1,1}) end(), fun() -> S = "99. ", - {ok, Ts, 1} = scan_tokens_1(S, F, 1), - {ok, Ts, 1} = scan_token_1(S, F1, 1), - {ok, Ts, 1} = scan_tokens(S, F, 1), - {ok, Ts, 1} = erlang_scan:string(S, 1) + {ok, Ts, {1,5}} = scan_tokens_1(S, F, {1,1}), + {ok, Ts, {1,5}} = scan_token_1(S, F1, {1,1}), + {ok, Ts, {1,5}} = scan_tokens(S, F, {1,1}), + {ok, Ts, {1,5}} = erlang_scan:string(S, {1,1}) end(), - {ok,[{integer,1,99},{dot,1}],1} = erlang_scan:string("99. "), + {ok,[{integer,{1,1},99},{dot,{1,3}}],{1,5}} = erlang_scan:string("99. "), fun() -> Atom = "'" ++ lists:duplicate(1000,$a) ++ "'", S = Atom ++ ". ", Reason = "illegal atom " ++ Atom, - Err = {error,{1,erlang_scan,{user,Reason}},1}, - {done,Err,[]} = scan_tokens_1(S, F, 1), - {done,Err,[]} = scan_token_1(S, F1, 1), - {done,Err,[]} = scan_tokens(S, F, 1), - Err = erlang_scan:string(S, 1) + Err = {error,{{1,1003},erlang_scan,{user,Reason}},{1,1003}}, + {done,Err,[]} = scan_tokens_1(S, F, {1,1}), + {done,Err,[]} = scan_token_1(S, F1, {1,1}), + {done,Err,[]} = scan_tokens(S, F, {1,1}), + Err = erlang_scan:string(S, {1,1}) end(), fun() -> S = "\x{aaa}. ", - Err = {error,{1,erlang_scan,{illegal,[2730]}},1}, - {done,Err,[]} = scan_tokens_1(S, F, 1), - {done,Err,[_]} = scan_token_1(S, F1, 1), % Note: Rest non-empty - {done,Err,[]} = scan_tokens(S, F, 1), - Err = erlang_scan:string(S, 1) + Err = {error,{{1,1},erlang_scan,{illegal,[2730]}},{1,1}}, + {done,Err,[]} = scan_tokens_1(S, F, {1,1}), + {done,Err,[_]} = scan_token_1(S, F1, {1,1}), % Note: Rest non-empty + {done,Err,[]} = scan_tokens(S, F, {1,1}), + Err = erlang_scan:string(S, {1,1}) end(), fun() -> S = "\x{aaa} + 1. 34", - Err = {error,{1,erlang_scan,{illegal,[2730]}},1}, - {done,Err,[]} = scan_tokens_1(S, F, 1), - {done,Err,[_]} = scan_token_1(S, F1, 1), % Note: Rest non-empty - {done,Err,"34"} = scan_tokens(S, F, 1), - Err = erlang_scan:string(S, 1) + Err = {error,{{1,1},erlang_scan,{illegal,[2730]}},{1,1}}, + {done,Err,[]} = scan_tokens_1(S, F, {1,1}), + {done,Err,[_]} = scan_token_1(S, F1, {1,1}), % Note: Rest non-empty + {done,Err,"34"} = scan_tokens(S, F, {1,1}), + Err = erlang_scan:string(S, {1,1}) end(), fun() -> S = "\x{aaa} \x{bbb}. 34", - Err = {error,{1,erlang_scan,{illegal,[2730]}},1}, - {done,Err,[]} = scan_tokens_1(S, F, 1), - {done,Err,[_]} = scan_token_1(S, F1, 1), % Note: Rest non-empty - {done,Err,"34"} = scan_tokens(S, F, 1), - Err = erlang_scan:string(S, 1) + Err = {error,{{1,1},erlang_scan,{illegal,[2730]}},{1,1}}, + {done,Err,[]} = scan_tokens_1(S, F, {1,1}), + {done,Err,[_]} = scan_token_1(S, F1, {1,1}), % Note: Rest non-empty + {done,Err,"34"} = scan_tokens(S, F, {1,1}), + Err = erlang_scan:string(S, {1,1}) end(), fun() -> S = "\x{aaa} 18#34. 34", - Err = {error,{1,erlang_scan,{illegal,[2730]}},1}, - {done,Err,[]} = scan_tokens_1(S, F, 1), - {done,Err,[_]} = scan_token_1(S, F1, 1), % Note: Rest non-empty - {done,Err,"34"} = scan_tokens(S, F, 1), - Err = erlang_scan:string(S, 1) + Err = {error,{{1,1},erlang_scan,{illegal,[2730]}},{1,1}}, + {done,Err,[]} = scan_tokens_1(S, F, {1,1}), + {done,Err,[_]} = scan_token_1(S, F1, {1,1}), % Note: Rest non-empty + {done,Err,"34"} = scan_tokens(S, F, {1,1}), + Err = erlang_scan:string(S, {1,1}) end(), fun() -> S = "\x{aaa}"++eof, - Err = {error,{1,erlang_scan,{illegal,[2730]}},1}, - {done,Err,eof} = scan_tokens_1(S, F, 1), - {done,Err,[_]} = scan_token_1(S, F1, 1), % Note: Rest non-empty - {done,Err,eof} = scan_tokens(S, F, 1), - Err = erlang_scan:string(S, 1) + Err = {error,{{1,1},erlang_scan,{illegal,[2730]}},{1,1}}, + {done,Err,eof} = scan_tokens_1(S, F, {1,1}), + {done,Err,[_]} = scan_token_1(S, F1, {1,1}), % Note: Rest non-empty + {done,Err,eof} = scan_tokens(S, F, {1,1}), + Err = erlang_scan:string(S, {1,1}) end(), ok. @@ -912,8 +922,8 @@ line_wrap(Config) when is_list(Config) -> <<" Definitions. Rules. -[a]+[\\n]*= : {token, {first, TokenLine}}. -[a]+ : {token, {second, TokenLine}}. +[a]+[\\n]*= : {token, {first, TokenLoc}}. +[a]+ : {token, {second, TokenLoc}}. [\\s\\r\\n\\t]+ : skip_token. Erlang code. ">>, @@ -928,20 +938,20 @@ Erlang code. code:load_abs(AbsFile, test_line_wrap), fun() -> S = "aaa\naaa", - {ok,[{second,1},{second,2}],2} = test_line_wrap:string(S) + {ok,[{second,{1,1}},{second,{2,1}}],2} = test_line_wrap:string(S) end(), fun() -> S = "aaa\naaa", - {ok,[{second,3},{second,4}],4} = test_line_wrap:string(S, 3) + {ok,[{second,{3,1}},{second,{4,1}}],4} = test_line_wrap:string(S, 3) end(), fun() -> - {done,{ok,{second,1},1},"\na"} = test_line_wrap:token([], "a\na"), + {done,{ok,{second,{1,1}},1},"\na"} = test_line_wrap:token([], "a\na"), {more,Cont1} = test_line_wrap:token([], "\na"), - {done,{ok,{second,2},2},eof} = test_line_wrap:token(Cont1, eof) + {done,{ok,{second,{2,1}},2},eof} = test_line_wrap:token(Cont1, eof) end(), fun() -> {more,Cont1} = test_line_wrap:tokens([], "a\na"), - {done,{ok,[{second,1},{second,2}],2},eof} = test_line_wrap:tokens(Cont1, eof) + {done,{ok,[{second,{1,1}},{second,{2,1}}],2},eof} = test_line_wrap:tokens(Cont1, eof) end(), ok. @@ -1044,7 +1054,7 @@ otp_10302(Config) when is_list(Config) -> "-export([t/0]).\n" "t() ->\n" " %% Häpp, 'Häpp',\"\\x{400}B\",\"örn_Ð\"\n" - " {ok, [R], 1} = string(\"tip\"),\n" + " {ok, [R], {1,4}} = string(\"tip\"),\n" " {tip,foo,'Häpp',[1024,66],[246,114,110,95,1024]} = R,\n" " Häpp = foo,\n" " {tip, Häpp, 'Häpp',\"\\x{400}B\",\"örn_Ð\"} = R,\n" @@ -1065,7 +1075,7 @@ otp_10302(Config) when is_list(Config) -> "-export([t/0]).\n" "t() ->\n" " %% Häpp, 'Häpp',\"\\x{400}B\",\"örn_Ð\"\n" - " {ok, [R], 1} = string(\"tip\"),\n" + " {ok, [R], {1,4}} = string(\"tip\"),\n" " {tip,foo,'Häpp',[1024,66],[195,182,114,110,95,208,128]} = R,\n" " Häpp = foo,\n" " {tip, Häpp, 'Häpp',\"\\x{400}B\",\"örn_Ð\"} = R,\n" @@ -1139,23 +1149,23 @@ otp_13916(Config) when is_list(Config) -> "Rules.\n" "%% mark line break(s) and empty lines by token 'break'\n" "%% in order to use as delimiters\n" - "{B}({S}*{B})+ : {token, {break, TokenLine}}.\n" - "{B} : {token, {break, TokenLine}}.\n" - "{S}+ : {token, {blank, TokenLine, TokenChars}}.\n" - "{W}+ : {token, {word, TokenLine, TokenChars}}.\n" + "{B}({S}*{B})+ : {token, {break, TokenLoc}}.\n" + "{B} : {token, {break, TokenLoc}}.\n" + "{S}+ : {token, {blank, TokenLoc, TokenChars}}.\n" + "{W}+ : {token, {word, TokenLoc, TokenChars}}.\n" "Erlang code.\n" "-export([t/0]).\n" "t() ->\n" - " {ok,[{break,1},{blank,4,\" \"},{word,4,\"breaks\"}],4} =\n" + " {ok,[{break,{1,1}},{blank,{4,1},\" \"},{word,{4,3},\"breaks\"}],{4,9}} =\n" " string(\"\\n\\n \\n breaks\"),\n" - " {ok,[{break,1},{word,4,\"works\"}],4} =\n" + "{ok,[{break,{1,1}},{word,{4,1},\"works\"}],{4,6}} =\n" " string(\"\\n\\n \\nworks\"),\n" - " {ok,[{break,1},{word,4,\"L4\"},{break,4},\n" - " {word,5,\"L5\"},{break,5},{word,7,\"L7\"}], 7} =\n" + " {ok,[{break,{1,1}},{word,{4,1},\"L4\"},{break,{4,3}},\n" + " {word,{5,1},\"L5\"},{break,{5,3}},{word,{7,1},\"L7\"}], {7,3}} =\n" " string(\"\\n\\n \\nL4\\nL5\\n\\nL7\"),\n" - " {ok,[{break,1},{blank,4,\" \"},{word,4,\"L4\"},\n" - " {break,4},{blank,5,\" \"},{word,5,\"L5\"},\n" - " {break,5},{blank,7,\" \"},{word,7,\"L7\"}], 7} =\n" + "{ok,[{break,{1,1}},{blank,{4,1},\" \"},{word,{4,2} ,\"L4\"},\n" + " {break,{4,4}},{blank,{5,1},\" \"},{word,{5,2},\"L5\"},\n" + " {break,{5,4}},{blank,{7,1},\" \"},{word,{7,2},\"L7\"}], {7,4}} =\n" " string(\"\\n\\n \\n L4\\n L5\\n\\n L7\"),\n" " ok.\n">>, default, @@ -1164,6 +1174,7 @@ otp_13916(Config) when is_list(Config) -> ok. otp_14285(Config) -> + %% x{400} takes 2 bytes to represent Ts = [{otp_14285_1, <<"%% encoding: latin-1\n" "Definitions.\n" @@ -1173,11 +1184,11 @@ otp_14285(Config) -> "U = [\\x{400}]\n" "Rules.\n" "{L}+ : {token,l}.\n" - "{U}+ : {token,'\\x{400}'}.\n" + "{U}+ : {token,{TokenLine,'\\x{400}'}}.\n" "Erlang code.\n" "-export([t/0]).\n" "t() ->\n" - " {ok,['\\x{400}'],1} = string(\"\\x{400}\"), ok.\n">>, + " {ok,[{1,'\\x{400}'}],{1,3}} = string(\"\\x{400}\"), ok.\n">>, default, ok}, {otp_14285_2, @@ -1193,7 +1204,7 @@ otp_14285(Config) -> "Erlang code.\n" "-export([t/0]).\n" "t() ->\n" - " {ok,['\x{400}'],1} = string(\"\x{400}\"), ok.\n">>, + " {ok,['\x{400}'],{1,3}} = string(\"\x{400}\"), ok.\n"/utf8>>, default, ok}], run(Config, Ts), @@ -1225,6 +1236,54 @@ otp_17023(Config) -> end, ok. +%% Additional tests added with column support +column_support(Config) -> + Ts = [{token_col_var, + <<"Definitions.\n" + "D = [0-9]\n" + "W = [\\s\\n]\n" + "Rules.\n" + "{W}+ :\n" + "skip_token.\n" + "{D}+ :\n" + "{token,{integer,{TokenLine,TokenCol},list_to_integer(TokenChars)}}.\n" + "{D}+\\.{D}+((E|e)(\\+|\\-)?{D}+)? :\n" + "{token,{float,{TokenLine,TokenCol},list_to_float(TokenChars)}}.\n" + "Erlang code.\n" + "-export([t/0]).\n" + "t() ->\n" + "{ok,[{float, {2,1}, 4.44},{integer, {3,3}, 5},{integer, {7,3}, 7}],{8,2}}" + "= string(\"\n4.44 \n 5 \n \n\n\n 7 \n \"), ok.\n">>, + default, + ok}, + {tab, + <<"Definitions.\n" + "Rules.\n" + "[a]+[\\n]*= : {token, {first, TokenLoc}}.\n" + "[a]+ : {token, {second, TokenLoc}}.\n" + "[\\s\\r\\n\\t]+ : skip_token.\n" + "Erlang code.\n" + "-export([t/0]).\n" + "t() ->\n" + "{ok,[{second,{1,27}},{second,{2,19}}],{2,25}} = string(\" \t \t\t a\\n \t \t aaa\t\"), ok.\n">>, + default, + ok}, + {tab_custom_size, + <<"Definitions.\n" + "Rules.\n" + "[a]+[\\n]*= : {token, {first, TokenLoc}}.\n" + "[a]+ : {token, {second, TokenLoc}}.\n" + "[\\s\\r\\n\\t]+ : skip_token.\n" + "Erlang code.\n" + "-export([t/0]).\n" + "t() ->\n" + "{ok,[{second,{1,15}},{second,{2,9}}],{2,16}} = string(\" \t \t\t a\\n \t \t aaa\t\"), ok.\n">>, + default, + [{tab_size,3}], + ok}], + run(Config, Ts), + ok. + %% OTP-17499. GH-4918. compiler_warnings(Config) -> Xrl = @@ -1256,18 +1315,23 @@ writable(Fname) -> ok = file:write_file_info(Fname, Info#file_info{mode = Mode}). run(Config, Tests) -> - F = fun({N,P,Pre,E}) -> - case catch run_test(Config, P, Pre) of - E -> - ok; - Bad -> - ct:fail("~nTest ~p failed. Expected~n ~p~n" - "but got~n ~p~n", [N, E, Bad]) - end + F = fun F({N,P,Pre,E}) -> + F({N,P,Pre,[],E}); + F({N,P,Pre,Opts,E}) -> + case catch run_test(Config,P,Pre,Opts) of + E -> + ok; + Bad -> + ct:fail("~nTest ~p failed. Expected~n ~p~n" + "but got~n ~p~n", [N, E, Bad]) + end end, lists:foreach(F, Tests). run_test(Config, Def, Pre) -> + run_test(Config, Def, Pre, []). + +run_test(Config, Def, Pre, LOpts0) -> %% io:format("testing ~s~n", [binary_to_list(Def)]), DefFile = 'leex_test.xrl', Filename = 'leex_test.erl', @@ -1276,14 +1340,14 @@ run_test(Config, Def, Pre) -> ErlFile = filename:join(DataDir, Filename), Opts = [return, warn_unused_vars,{outdir,DataDir}], ok = file:write_file(XrlFile, Def), - LOpts = [return, {report, false} | + LOpts = LOpts0 ++ [return, {report, false} | case Pre of default -> []; _ -> [{includefile,Pre}] end], - XOpts = [verbose, dfa_graph], % just to get some code coverage... + XOpts = [verbose, dfa_graph, {error_location, column}], % just to get some code coverage... LRet = leex:file(XrlFile, XOpts ++ LOpts), case LRet of {ok, _Outfile, _LWs} -> -- 2.35.3
Locations
Projects
Search
Status Monitor
Help
OpenBuildService.org
Documentation
API Documentation
Code of Conduct
Contact
Support
@OBShq
Terms
openSUSE Build Service is sponsored by
The Open Build Service is an
openSUSE project
.
Sign Up
Log In
Places
Places
All Projects
Status Monitor
