mewin/glslang
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #637 from KhronosGroup/token-paste

Browse Source 
PP: Implement token pasting for PP identifiers.
This commit is contained in:
John Kessenich
2016-12-19 14:31:57 -07:00
committed by GitHub
parent e6cbc5b19d d485e0b710
commit 0955b1cb35
10 changed files with 304 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
glslang/Include/revision.h
View File

@@ -2,5 +2,5 @@
// For the version, it uses the latest git tag followed by the number of commits.
// For the date, it uses the current date (when then script is run).
#define GLSLANG_REVISION "Overload400-PrecQual.1695"
#define GLSLANG_DATE "16-Dec-2016"
#define GLSLANG_REVISION "Overload400-PrecQual.1696"
#define GLSLANG_DATE "19-Dec-2016"

95
glslang/MachineIndependent/preprocessor/Pp.cpp
View File

@@ -933,36 +933,38 @@ int TPpContext::readCPPline(TPpToken* ppToken)
return token;
}
TPpContext::TokenStream* TPpContext::PrescanMacroArg(TokenStream* a, TPpToken* ppToken, bool newLineOkay)
// Macro-expand a macro argument 'arg' to create 'expandedArg'.
// Does not replace 'arg'.
// Returns nullptr if no expanded argument is created.
TPpContext::TokenStream* TPpContext::PrescanMacroArg(TokenStream* arg, TPpToken* ppToken, bool newLineOkay)
{
int token;
TokenStream *n;
RewindTokenStream(a);
RewindTokenStream(arg);
do {
token = ReadToken(a, ppToken);
token = ReadToken(arg, ppToken);
if (token == PpAtomIdentifier && LookUpSymbol(ppToken->atom))
break;
} while (token != EndOfInput);
if (token == EndOfInput)
return a;
return nullptr;
n = new TokenStream;
TokenStream* expandedArg = new TokenStream;
pushInput(new tMarkerInput(this));
pushTokenStreamInput(a);
pushTokenStreamInput(arg);
while ((token = scanToken(ppToken)) != tMarkerInput::marker) {
if (token == PpAtomIdentifier && MacroExpand(ppToken->atom, ppToken, false, newLineOkay) != 0)
continue;
RecordToken(n, token, ppToken);
RecordToken(expandedArg, token, ppToken);
}
popInput();
delete a;
return n;
return expandedArg;
}
//
// Return the next token for a macro expansion, handling macro args.
// Return the next token for a macro expansion, handling macro arguments,
// whose semantics are dependent on being adjacent to ##.
//
int TPpContext::tMacroInput::scan(TPpToken* ppToken)
{
@@ -971,6 +973,39 @@ int TPpContext::tMacroInput::scan(TPpToken* ppToken)
token = pp->ReadToken(mac->body, ppToken);
} while (token == ' '); // handle white space in macro
// Hash operators basically turn off a round of macro substitution
// (the round done on the argument before the round done on the RHS of the
// macro definition):
//
// "A parameter in the replacement list, unless preceded by a # or ##
// preprocessing token or followed by a ## preprocessing token (see below),
// is replaced by the corresponding argument after all macros contained
// therein have been expanded."
//
// "If, in the replacement list, a parameter is immediately preceded or
// followed by a ## preprocessing token, the parameter is replaced by the
// corresponding argument's preprocessing token sequence."
bool pasting = false;
if (postpaste) {
// don't expand next token
pasting = true;
postpaste = false;
}
if (prepaste) {
// already know we should be on a ##, verify
assert(token == PpAtomPaste);
prepaste = false;
postpaste = true;
}
// see if are preceding a ##
if (peekMacPasting()) {
prepaste = true;
pasting = true;
}
// TODO: preprocessor: properly handle whitespace (or lack of it) between tokens when expanding
if (token == PpAtomIdentifier) {
int i;
@@ -978,7 +1013,10 @@ int TPpContext::tMacroInput::scan(TPpToken* ppToken)
if (mac->args[i] == ppToken->atom)
break;
if (i >= 0) {
pp->pushTokenStreamInput(args[i]);
TokenStream* arg = expandedArgs[i];
if (arg == nullptr || pasting)
arg = args[i];
pp->pushTokenStreamInput(arg, prepaste);
return pp->scanToken(ppToken);
}
@@ -990,6 +1028,31 @@ int TPpContext::tMacroInput::scan(TPpToken* ppToken)
return token;
}
// See if the next non-white-space token in the macro is ##
bool TPpContext::tMacroInput::peekMacPasting()
{
// don't return early, have to restore this
size_t savePos = mac->body->current;
// skip white-space
int ltoken;
do {
ltoken = pp->lReadByte(mac->body);
} while (ltoken == ' ');
// check for ##
bool pasting = false;
if (ltoken == '#') {
ltoken = pp->lReadByte(mac->body);
if (ltoken == '#')
pasting = true;
}
mac->body->current = savePos;
return pasting;
}
// return a textual zero, for scanning a macro that was never defined
int TPpContext::tZeroInput::scan(TPpToken* ppToken)
{
@@ -1080,6 +1143,9 @@ int TPpContext::MacroExpand(int atom, TPpToken* ppToken, bool expandUndef, bool
in->args.resize(in->mac->argc);
for (int i = 0; i < in->mac->argc; i++)
in->args[i] = new TokenStream;
in->expandedArgs.resize(in->mac->argc);
for (int i = 0; i < in->mac->argc; i++)
in->expandedArgs[i] = nullptr;
int arg = 0;
bool tokenRecorded = false;
do {
@@ -1143,8 +1209,11 @@ int TPpContext::MacroExpand(int atom, TPpToken* ppToken, bool expandUndef, bool
}
parseContext.ppError(loc, "Too many args in macro", "macro expansion", GetAtomString(atom));
}
// We need both expanded and non-expanded forms of the argument, for whether or
// not token pasting is in play.
for (int i = 0; i < in->mac->argc; i++)
in->args[i] = PrescanMacroArg(in->args[i], ppToken, newLineOkay);
in->expandedArgs[i] = PrescanMacroArg(in->args[i], ppToken, newLineOkay);
}
pushInput(in);

24
glslang/MachineIndependent/preprocessor/PpContext.h
View File

@@ -129,6 +129,7 @@ public:
void setPreamble(const char* preamble, size_t length);
const char* tokenize(TPpToken* ppToken);
int tokenPaste(TPpToken&);
class tInput {
public:
@@ -138,6 +139,8 @@ public:
virtual int scan(TPpToken*) = 0;
virtual int getch() = 0;
virtual void ungetch() = 0;
virtual bool peekPasting() { return false; } // true when about to see ##
virtual bool endOfReplacementList() { return false; } // true when at the end of a macro replacement list (RHS of #define)
// Will be called when we start reading tokens from this instance
virtual void notifyActivated() {}
@@ -235,6 +238,8 @@ protected:
}
int getChar() { return inputStack.back()->getch(); }
void ungetChar() { inputStack.back()->ungetch(); }
bool peekPasting() { return !inputStack.empty() && inputStack.back()->peekPasting(); }
bool endOfReplacementList() { return inputStack.empty() || inputStack.back()->endOfReplacementList(); }
static const int maxMacroArgs = 64;
static const int maxIfNesting = 64;
@@ -245,18 +250,29 @@ protected:
class tMacroInput : public tInput {
public:
tMacroInput(TPpContext* pp) : tInput(pp) { }
tMacroInput(TPpContext* pp) : tInput(pp), prepaste(false), postpaste(false) { }
virtual ~tMacroInput()
{
for (size_t i = 0; i < args.size(); ++i)
delete args[i];
for (size_t i = 0; i < expandedArgs.size(); ++i)
delete expandedArgs[i];
}
virtual int scan(TPpToken*);
virtual int getch() { assert(0); return EndOfInput; }
virtual void ungetch() { assert(0); }
bool peekPasting() override { return prepaste; }
bool endOfReplacementList() override { return mac->body->current >= mac->body->data.size(); }
MacroSymbol *mac;
TVector<TokenStream*> args;
TVector<TokenStream*> expandedArgs;
protected:
bool peekMacPasting();
bool prepaste; // true if we are just before ##
bool postpaste; // true if we are right after ##
};
class tMarkerInput : public tInput {
@@ -329,17 +345,19 @@ protected:
void RecordToken(TokenStream* pTok, int token, TPpToken* ppToken);
void RewindTokenStream(TokenStream *pTok);
int ReadToken(TokenStream* pTok, TPpToken* ppToken);
void pushTokenStreamInput(TokenStream *ts);
void pushTokenStreamInput(TokenStream *ts, bool pasting = false);
void UngetToken(int token, TPpToken* ppToken);
class tTokenInput : public tInput {
public:
tTokenInput(TPpContext* pp, TokenStream* t) : tInput(pp), tokens(t) { }
tTokenInput(TPpContext* pp, TokenStream* t, bool prepasting) : tInput(pp), tokens(t), lastTokenPastes(prepasting) { }
virtual int scan(TPpToken *);
virtual int getch() { assert(0); return EndOfInput; }
virtual void ungetch() { assert(0); }
virtual bool peekPasting() override;
protected:
TokenStream *tokens;
bool lastTokenPastes; // true if the last token in the input is to be pasted, rather than consumed as a token
};
class tUngotTokenInput : public tInput {

46
glslang/MachineIndependent/preprocessor/PpScanner.cpp
View File

@@ -739,6 +739,11 @@ const char* TPpContext::tokenize(TPpToken* ppToken)
for(;;) {
token = scanToken(ppToken);
ppToken->token = token;
// Handle token-pasting logic
token = tokenPaste(*ppToken);
ppToken->token = token;
if (token == EndOfInput) {
missingEndifCheck();
return nullptr;
@@ -800,6 +805,47 @@ const char* TPpContext::tokenize(TPpToken* ppToken)
}
}
//
// Do all token-pasting related combining of two pasted tokens when getting a
// stream of tokens from a replacement list. Degenerates to no processing if a
// replacement list is not the source of the token stream.
//
int TPpContext::tokenPaste(TPpToken& ppToken)
{
// starting with ## is illegal, skip to next token
if (ppToken.token == PpAtomPaste) {
parseContext.ppError(ppToken.loc, "unexpected location", "##", "");
ppToken.token = scanToken(&ppToken);
}
// ## can be chained, process all in the chain at once
while (peekPasting()) {
TPpToken pastedPpToken;
// next token has to be ##
pastedPpToken.token = scanToken(&pastedPpToken);
assert(pastedPpToken.token == PpAtomPaste);
if (endOfReplacementList()) {
parseContext.ppError(ppToken.loc, "unexpected location; end of replacement list", "##", "");
break;
}
// get the token after the ##
scanToken(&pastedPpToken);
// combine the tokens
if (strlen(ppToken.name) + strlen(pastedPpToken.name) > MaxTokenLength)
parseContext.ppError(ppToken.loc, "combined tokens are too long", "##", "");
strncat(ppToken.name, pastedPpToken.name, MaxTokenLength - strlen(ppToken.name));
ppToken.atom = LookUpAddString(ppToken.name);
if (ppToken.token != PpAtomIdentifier)
parseContext.ppError(ppToken.loc, "only supported for preprocessing identifiers", "##", "");
}
return ppToken.token;
}
// Checks if we've seen balanced #if...#endif
void TPpContext::missingEndifCheck()
{

35
glslang/MachineIndependent/preprocessor/PpTokens.cpp
View File

@@ -160,7 +160,7 @@ void TPpContext::RecordToken(TokenStream *pTok, int token, TPpToken* ppToken)
}
/*
* Reset a token stream in preperation for reading.
* Reset a token stream in preparation for reading.
*/
void TPpContext::RewindTokenStream(TokenStream *pTok)
{
@@ -187,9 +187,7 @@ int TPpContext::ReadToken(TokenStream *pTok, TPpToken *ppToken)
if (lReadByte(pTok) == '#') {
parseContext.requireProfile(ppToken->loc, ~EEsProfile, "token pasting (##)");
parseContext.profileRequires(ppToken->loc, ~EEsProfile, 130, 0, "token pasting (##)");
parseContext.error(ppToken->loc, "token pasting not implemented (internal error)", "##", "");
//return PpAtomPaste;
return ReadToken(pTok, ppToken);
ltoken = PpAtomPaste;
} else
lUnreadByte(pTok);
}
@@ -279,9 +277,34 @@ int TPpContext::tTokenInput::scan(TPpToken* ppToken)
return pp->ReadToken(tokens, ppToken);
}
void TPpContext::pushTokenStreamInput(TokenStream* ts)
// We are pasting if the entire macro is preceding a pasting operator
// (lastTokenPastes) and we are also on the last token.
bool TPpContext::tTokenInput::peekPasting()
{
pushInput(new tTokenInput(this, ts));
if (! lastTokenPastes)
return false;
// Getting here means the last token will be pasted.
// Are we at the last non-whitespace token?
size_t savePos = tokens->current;
bool moreTokens = false;
do {
int byte = pp->lReadByte(tokens);
if (byte == EndOfInput)
break;
if (byte != ' ') {
moreTokens = true;
break;
}
} while (true);
tokens->current = savePos;
return !moreTokens;
}
void TPpContext::pushTokenStreamInput(TokenStream* ts, bool prepasting)
{
pushInput(new tTokenInput(this, ts, prepasting));
RewindTokenStream(ts);
}