mewin/glslang
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Fix several issues in the preprocessor:

Browse Source 
- macro expansion of hexidecimal numbers 
 - give errors instead of warnings/silence on extra tokens after #endif, #else, etc.
 - give errors on reserved macro name use, reuse of argument, and redefinition with different whitespace presence
 - detect and give error for all cases of #elif and #else after #else



git-svn-id: https://cvs.khronos.org/svn/repos/ogl/trunk/ecosystem/public/sdk/tools/glslang@23982 e7fa87d3-cd2b-0410-9028-fcbf551c1848
This commit is contained in:
John Kessenich 2013-11-10 23:07:36 +00:00
parent cfe3ba5a18
commit 4d6570a1b3
14 changed files with 403 additions and 339 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
Test/baseResults/cppComplexExpr.vert.out
View File

@ -1,6 +1,7 @@
ERROR: 0:46: 'xyxwx' : illegal vector field selection ERROR: 0:46: 'xyxwx' : illegal vector field selection
ERROR: 0:46: 'xyxwx' : illegal vector field selection ERROR: 0:46: 'xyxwx' : illegal vector field selection
ERROR: 2 compilation errors. No code generated. ERROR: 0:51: '' : missing #endif
ERROR: 3 compilation errors. No code generated.
ERROR: node is still EOpNull! ERROR: node is still EOpNull!
0:4 Sequence 0:4 Sequence

60
Test/baseResults/cppSimple.vert.out
View File

@ -1,3 +1,4 @@
Warning, version 400 is not yet complete; some version-specific features are present, but many are missing.
ERROR: 0:77: '#error' : good1 ERROR: 0:77: '#error' : good1
ERROR: 0:81: '#error' : good2 ERROR: 0:81: '#error' : good2
ERROR: 0:85: '#error' : good3 ERROR: 0:85: '#error' : good3
@ -6,20 +7,42 @@ ERROR: 0:93: '#error' : good5
ERROR: 0:97: '#error' : good6 ERROR: 0:97: '#error' : good6
ERROR: 0:101: 'preprocessor' : expected ')' ERROR: 0:101: 'preprocessor' : expected ')'
ERROR: 0:101: '#error' : bad1 ERROR: 0:101: '#error' : bad1
WARNING: 0:104: '#if' : unexpected tokens following #if directive - expected a newline ERROR: 0:104: '#if' : unexpected tokens following directive
ERROR: 0:105: '#error' : bad2 ERROR: 0:105: '#error' : bad2
ERROR: 0:109: 'preprocessor' : expected ')' ERROR: 0:109: 'preprocessor' : expected ')'
ERROR: 0:109: '#error' : bad3 ERROR: 0:109: '#error' : bad3
WARNING: 0:112: '#if' : unexpected tokens following #if directive - expected a newline ERROR: 0:112: '#if' : unexpected tokens following directive
ERROR: 0:113: '#error' : bad4 ERROR: 0:113: '#error' : bad4
ERROR: 0:117: 'preprocessor' : expected ')' ERROR: 0:117: 'preprocessor' : expected ')'
ERROR: 0:117: '#error' : bad5 ERROR: 0:117: '#error' : bad5
WARNING: 0:120: '#if' : unexpected tokens following #if directive - expected a newline ERROR: 0:120: '#if' : unexpected tokens following directive
ERROR: 0:121: '#error' : bad6 ERROR: 0:121: '#error' : bad6
ERROR: 0:133: '#' : preprocessor directive cannot be preceded by another token ERROR: 0:122: '#endif' : unexpected tokens following directive
INTERNAL ERROR: 0:133: Unknown PP token ERROR: 0:135: '""' : string literals not supported
ERROR: 0:133: '' : syntax error ERROR: 0:136: '""' : string literals not supported
ERROR: 17 compilation errors. No code generated. ERROR: 0:136: 'length' : no matching overloaded function found
ERROR: 0:136: '=' : cannot convert from 'const float' to 'int'
ERROR: 0:138: ''' : character literals not supported
ERROR: 0:138: ''' : character literals not supported
ERROR: 0:141: '#define' : reserved built-in name prefix: GL_
ERROR: 0:142: '#define' : reserved built-in name prefix: GL_
ERROR: 0:143: '#define' : names containing consecutive underscores are reserved
ERROR: 0:144: '#define' : names containing consecutive underscores are reserved
ERROR: 0:145: '#define' : names containing consecutive underscores are reserved
ERROR: 0:148: '#else' : unexpected tokens following directive
ERROR: 0:149: '#else' : #elif after #else
ERROR: 0:155: '#else' : unexpected tokens following directive
ERROR: 0:158: '#else' : #else after #else
ERROR: 0:160: '#endif' : unexpected tokens following directive
ERROR: 0:164: '#define' : duplicate macro parameter
ERROR: 0:173: '#define' : Macro redefined; different number of arguments: m4
ERROR: 0:178: '#define' : Macro redefined; different number of arguments: m5
ERROR: 0:182: '#define' : Macro redefined; different number of arguments: m6
ERROR: 0:185: '#define' : Macro redefined; different substitutions: m7
ERROR: 0:192: '#define' : Macro redefined; different substitutions: m8
ERROR: 0:196: '#define' : Macro redefined; different argument names: m9
ERROR: 0:206: '' : missing #endif
ERROR: 43 compilation errors. No code generated.
ERROR: node is still EOpNull! ERROR: node is still EOpNull!
0:5 Sequence 0:5 Sequence
@ -59,7 +82,10 @@ ERROR: node is still EOpNull!
0:65 Constant: 0:65 Constant:
0:65 0.050000 0:65 0.050000
0:69 move second child to first child (4-component vector of float) 0:69 move second child to first child (4-component vector of float)
0:69 'gl_Position' (gl_Position 4-component vector of float) 0:69 gl_Position: direct index for structure (gl_Position 4-component vector of float)
0:69 '__anon__1' (out block{gl_Position,gl_PointSize,gl_ClipDistance,gl_ClipVertex,gl_FrontColor,gl_BackColor,gl_FrontSecondaryColor,gl_BackSecondaryColor,gl_TexCoord,gl_FogFragCoord})
0:69 Constant:
0:69 0 (const uint)
0:69 Construct vec4 (4-component vector of float) 0:69 Construct vec4 (4-component vector of float)
0:69 'sum' (float) 0:69 'sum' (float)
0:124 Sequence 0:124 Sequence
@ -76,17 +102,31 @@ ERROR: node is still EOpNull!
0:126 move second child to first child (int) 0:126 move second child to first child (int)
0:126 'version' (int) 0:126 'version' (int)
0:126 Constant: 0:126 Constant:
0:126 110 (const int) 0:126 400 (const int)
0:130 Sequence 0:130 Sequence
0:130 move second child to first child (float) 0:130 move second child to first child (float)
0:130 'twoPi' (float) 0:130 'twoPi' (float)
0:130 Constant: 0:130 Constant:
0:130 6.280000 0:130 6.280000
0:199 Sequence
0:199 move second child to first child (int)
0:199 'n' (int)
0:199 Constant:
0:199 15 (const int)
0:202 Sequence
0:202 move second child to first child (double)
0:202 'f' (double)
0:202 Constant:
0:202 0.000800
0:? Linker Objects 0:? Linker Objects
0:? 'sum' (float) 0:? 'sum' (float)
0:? 'linenumber' (int) 0:? 'linenumber' (int)
0:? 'filenumber' (int) 0:? 'filenumber' (int)
0:? 'version' (int) 0:? 'version' (int)
0:? 'twoPi' (float) 0:? 'twoPi' (float)
0:? 'tod' (float) 0:? 'a' (int)
0:? 'n' (int)
0:? 'f' (double)
0:? 'gl_VertexID' (gl_VertexId int)
0:? 'gl_InstanceID' (gl_InstanceId int)

3
Test/cppComplexExpr.vert
View File

@ -46,3 +46,6 @@ float foo()
return ADD(gl_Position.xyxwx, 3.0) // ERROR, should be this line number return ADD(gl_Position.xyxwx, 3.0) // ERROR, should be this line number
return ADD(gl_Position.y, 3.0) return ADD(gl_Position.y, 3.0)
} }
#if 0
// ERROR, EOF

81
Test/cppSimple.vert
View File

@ -1,4 +1,4 @@
#version 110 #version 400
#define ON #define ON
@ -119,7 +119,7 @@ sum += 0.05;
#if ((AA == BB || (AA == CC)))) #if ((AA == BB || (AA == CC))))
#error bad6 #error bad6
#endif #endif extra tokens
int linenumber = __LINE__; int linenumber = __LINE__;
int filenumber = __FILE__; int filenumber = __FILE__;
@ -129,5 +129,78 @@ int version = __VERSION__;
#define TWOPI (2.0 * PI) #define TWOPI (2.0 * PI)
float twoPi = TWOPI; float twoPi = TWOPI;
#define PASTE(a,b) a ## b //#define PASTE(a,b) a ## b
float PASTE(tod, ay) = 17; //float PASTE(tod, ay) = 17;
"boo" // ERROR
int a = length("aoenatuh"); // ERROR
'int'; // ERROR
// ERROR: all the following are reserved
#define GL_
#define GL_Macro 1
#define __M
#define M__
#define ABC__DE abc
#if 4
#else extra
#elif
// ERROR elif after else
#endif
#if blah
#if 0
#else extra
#ifdef M
#else
#else
// ERROR else after else
#endif extra
#endif
#endif
#define m1(a,a) // ERROR
#define m2(a,b)
// okay
#define m3 (a)
#define m3 (a)
// ERROR
#define m4(b)
#define m4 (b)
// ERROR
#define m5 (b)
#define m5(b)
// ERROR
#define m6(a)
#define m6
// ERROR (whitespace)
#define m7 (a)
#define m7 ( a)
#define m80(a,b) is + exactly m3 the same
#define m80(a,b) is + exactly m3 the same
// ERROR
#define m8(a,b) almost + exactly m3 the same
#define m8(a,b) almost + exactly m3 thee same
// ERROR
#define m9(a,b,c) aoe
#define m9(a,d,c) aoe
#define n1 0xf
int n = n1;
#define f1 .08e-2Lf
double f = f1;
#if 1
#else
// ERROR, missing #endif

6
glslang/MachineIndependent/Scan.cpp
View File

@ -480,6 +480,8 @@ int TScanContext::tokenize(TPpContext* pp, TParserToken& token)
parserToken = &token; parserToken = &token;
TPpToken ppToken; TPpToken ppToken;
tokenText = pp->tokenize(&ppToken); tokenText = pp->tokenize(&ppToken);
if (tokenText == 0)
return 0;
loc = ppToken.loc; loc = ppToken.loc;
parserToken->sType.lex.loc = loc; parserToken->sType.lex.loc = loc;
@ -531,8 +533,8 @@ int TScanContext::tokenize(TPpContext* pp, TParserToken& token)
case CPP_OR_ASSIGN: return OR_ASSIGN; case CPP_OR_ASSIGN: return OR_ASSIGN;
case CPP_XOR_ASSIGN: return XOR_ASSIGN; case CPP_XOR_ASSIGN: return XOR_ASSIGN;
case CPP_INTCONSTANT: parserToken->sType.lex.i = ppToken.ival; return INTCONSTANT; case CPP_INTCONSTANT: parserToken->sType.lex.i = ppToken.ival; return INTCONSTANT;
case CPP_UINTCONSTANT: parserToken->sType.lex.i = ppToken.ival; return UINTCONSTANT; case CPP_UINTCONSTANT: parserToken->sType.lex.i = ppToken.ival; return UINTCONSTANT;
case CPP_FLOATCONSTANT: parserToken->sType.lex.d = ppToken.dval; return FLOATCONSTANT; case CPP_FLOATCONSTANT: parserToken->sType.lex.d = ppToken.dval; return FLOATCONSTANT;
case CPP_DOUBLECONSTANT: parserToken->sType.lex.d = ppToken.dval; return DOUBLECONSTANT; case CPP_DOUBLECONSTANT: parserToken->sType.lex.d = ppToken.dval; return DOUBLECONSTANT;
case CPP_IDENTIFIER: return tokenizeIdentifier(); case CPP_IDENTIFIER: return tokenizeIdentifier();

7
glslang/MachineIndependent/Scan.h
View File

@ -46,8 +46,8 @@ namespace glslang {
// //
class TInputScanner { class TInputScanner {
public: public:
TInputScanner(int n, const char* const s[], size_t L[], int b = 0) : TInputScanner(int n, const char* const s[], size_t L[], int b = 0, int f = 0) :
numSources(n), sources(s), lengths(L), currentSource(0), currentChar(0), stringBias(b) numSources(n), sources(s), lengths(L), currentSource(0), currentChar(0), stringBias(b), finale(f)
{ {
loc = new TSourceLoc[numSources]; loc = new TSourceLoc[numSources];
loc[currentSource].string = -stringBias; loc[currentSource].string = -stringBias;
@ -105,7 +105,7 @@ public:
void setLine(int newLine) { loc[currentSource].line = newLine; } void setLine(int newLine) { loc[currentSource].line = newLine; }
void setString(int newString) { loc[currentSource].string = newString; } void setString(int newString) { loc[currentSource].string = newString; }
const TSourceLoc& getSourceLoc() const { return loc[currentSource]; } const TSourceLoc& getSourceLoc() const { return loc[std::max(0, std::min(currentSource, numSources - finale - 1))]; }
void consumeWhiteSpace(bool& foundNonSpaceTab); void consumeWhiteSpace(bool& foundNonSpaceTab);
bool consumeComment(); bool consumeComment();
@ -147,6 +147,7 @@ protected:
TSourceLoc* loc; // an array TSourceLoc* loc; // an array
int stringBias; // the first string that is the user's string number 0 int stringBias; // the first string that is the user's string number 0
int finale; // number of internal strings after user's last string
}; };
} // end namespace glslang } // end namespace glslang

2
glslang/MachineIndependent/ShaderLang.cpp
View File

@ -499,7 +499,7 @@ bool CompileDeferred(
lengths[0] = strlen(strings[0]); lengths[0] = strlen(strings[0]);
strings[numStrings + 1] = "\n int;"; strings[numStrings + 1] = "\n int;";
lengths[numStrings + 1] = strlen(strings[numStrings + 1]); lengths[numStrings + 1] = strlen(strings[numStrings + 1]);
TInputScanner fullInput(numStrings + 2, strings, lengths, 1); TInputScanner fullInput(numStrings + 2, strings, lengths, 1, 1);
// Push a new symbol allocation scope that can for the shader's globals. // Push a new symbol allocation scope that can for the shader's globals.
symbolTable.push(); symbolTable.push();

236
glslang/MachineIndependent/preprocessor/Pp.cpp
View File

@ -133,13 +133,10 @@ int TPpContext::FinalCPP()
{ {
mem_FreePool(pool); mem_FreePool(pool);
if (ifdepth)
parseContext.error(parseContext.getCurrentLoc(), "missing #endif", "#if", "");
return 1; return 1;
} }
int TPpContext::CPPdefine(TPpToken * ppToken) int TPpContext::CPPdefine(TPpToken* ppToken)
{ {
int token, atom, args[maxMacroArgs], argc; int token, atom, args[maxMacroArgs], argc;
MacroSymbol mac; MacroSymbol mac;
@ -147,10 +144,21 @@ int TPpContext::CPPdefine(TPpToken * ppToken)
memset(&mac, 0, sizeof(mac)); memset(&mac, 0, sizeof(mac));
token = currentInput->scan(this, currentInput, ppToken); token = currentInput->scan(this, currentInput, ppToken);
if (token != CPP_IDENTIFIER) { if (token != CPP_IDENTIFIER) {
parseContext.error(ppToken->loc, "must be followed by macro atom", "#define", ""); parseContext.error(ppToken->loc, "must be followed by macro name", "#define", "");
return token; return token;
} }
atom = ppToken->atom; atom = ppToken->atom;
const char* definedName = GetAtomString(atom);
if (ppToken->loc.string >= 0) {
// We are in user code; check for reserved name use:
// "All macro names containing two consecutive underscores ( __ ) are reserved for future use as predefined
// macro names. All macro names prefixed with "GL_" ("GL" followed by a single underscore) are also
// reserved."
if (strncmp(definedName, "GL_", 3) == 0)
parseContext.error(ppToken->loc, "reserved built-in name prefix:", "#define", "GL_");
else if (strstr(definedName, "__") != 0)
parseContext.error(ppToken->loc, "names containing consecutive underscores are reserved", "#define", "");
}
token = currentInput->scan(this, currentInput, ppToken); token = currentInput->scan(this, currentInput, ppToken);
if (token == '(' && !ppToken->ival) { if (token == '(' && !ppToken->ival) {
// gather arguments // gather arguments
@ -164,8 +172,21 @@ int TPpContext::CPPdefine(TPpToken * ppToken)
return token; return token;
} }
if (argc < maxMacroArgs) // check for duplication
args[argc++] = ppToken->atom; bool duplicate = false;
for (int a = 0; a < argc; ++a) {
if (args[a] == ppToken->atom) {
parseContext.error(ppToken->loc, "duplicate macro parameter", "#define", "");
duplicate = true;
break;
}
}
if (! duplicate) {
if (argc < maxMacroArgs)
args[argc++] = ppToken->atom;
else
parseContext.error(ppToken->loc, "too many macro parameters", "#define", "");
}
token = currentInput->scan(this, currentInput, ppToken); token = currentInput->scan(this, currentInput, ppToken);
} while (token == ','); } while (token == ',');
if (token != ')') { if (token != ')') {
@ -178,41 +199,51 @@ int TPpContext::CPPdefine(TPpToken * ppToken)
memcpy(mac.args, args, argc * sizeof(int)); memcpy(mac.args, args, argc * sizeof(int));
token = currentInput->scan(this, currentInput, ppToken); token = currentInput->scan(this, currentInput, ppToken);
} }
mac.body = NewTokenStream(GetAtomString(atom), pool); TSourceLoc defineLoc = ppToken->loc; // because ppToken is going to go to the next line before we report errors
mac.body = NewTokenStream(pool);
while (token != '\n') { while (token != '\n') {
while (token == '\\') { if (token == '\\') {
token = currentInput->scan(this, currentInput, ppToken); token = currentInput->scan(this, currentInput, ppToken);
if (token == '\n') if (token == '\n')
token = currentInput->scan(this, currentInput, ppToken); token = currentInput->scan(this, currentInput, ppToken);
else
RecordToken(mac.body, '\\', ppToken);
} }
RecordToken(mac.body, token, ppToken); RecordToken(mac.body, token, ppToken);
int spaceCandidate = currentInput->getch(this, currentInput, ppToken);
if (spaceCandidate == ' ' || spaceCandidate == '\t')
RecordToken(mac.body, ' ', 0);
else
currentInput->ungetch(this, currentInput, spaceCandidate, ppToken);
token = currentInput->scan(this, currentInput, ppToken); token = currentInput->scan(this, currentInput, ppToken);
}; }
symb = LookUpSymbol(atom); symb = LookUpSymbol(atom);
if (symb) { if (symb) {
if (!symb->mac.undef) { if (! symb->mac.undef) {
// already defined -- need to make sure they are identical // Already defined -- need to make sure they are identical:
// "Two replacement lists are identical if and only if the preprocessing tokens in both have the same number,
// ordering, spelling, and white-space separation, where all white-space separations are considered identical."
if (symb->mac.argc != mac.argc) if (symb->mac.argc != mac.argc)
goto error; parseContext.error(defineLoc, "Macro redefined; different number of arguments:", "#define", GetAtomString(atom));
for (argc=0; argc < mac.argc; argc++) else {
if (symb->mac.args[argc] != mac.args[argc]) for (argc=0; argc < mac.argc; argc++) {
goto error; if (symb->mac.args[argc] != mac.args[argc])
RewindTokenStream(symb->mac.body); parseContext.error(defineLoc, "Macro redefined; different argument names:", "#define", GetAtomString(atom));
RewindTokenStream(mac.body);
do {
int old_lval, old_token;
old_token = ReadToken(symb->mac.body, ppToken);
old_lval = ppToken->ival;
token = ReadToken(mac.body, ppToken);
if (token != old_token || ppToken->ival != old_lval) {
error:
parseContext.error(ppToken->loc, "Macro Redefined", "#define", GetAtomString(atom));
break;
} }
} while (token > 0); RewindTokenStream(symb->mac.body);
RewindTokenStream(mac.body);
int newToken;
do {
int oldToken;
TPpToken oldPpToken;
TPpToken newPpToken;
oldToken = ReadToken(symb->mac.body, &oldPpToken);
newToken = ReadToken(mac.body, &newPpToken);
if (oldToken != newToken || oldPpToken != newPpToken) {
parseContext.error(defineLoc, "Macro redefined; different substitutions:", "#define", GetAtomString(atom));
break;
}
} while (newToken > 0);
}
} }
} else { } else {
symb = AddSymbol(atom); symb = AddSymbol(atom);
@ -220,7 +251,7 @@ error:
symb->mac = mac; symb->mac = mac;
return '\n'; return '\n';
} // CPPdefine }
int TPpContext::CPPundef(TPpToken * ppToken) int TPpContext::CPPundef(TPpToken * ppToken)
{ {
@ -246,23 +277,25 @@ int TPpContext::CPPundef(TPpToken * ppToken)
parseContext.error(ppToken->loc, "can only be followed by a single macro name", "#undef", ""); parseContext.error(ppToken->loc, "can only be followed by a single macro name", "#undef", "");
return token; return token;
} // CPPundef }
/* CPPelse -- skip forward to appropriate spot. This is actually used /* Skip forward to appropriate spot. This is used both
** to skip to a #endif after seeing an #else, AND to skip to a #else, ** to skip to a #endif after seeing an #else, AND to skip to a #else,
** #elif, or #endif after a #if/#ifdef/#ifndef/#elif test was false ** #elif, or #endif after a #if/#ifdef/#ifndef/#elif test was false.
*/ */
int TPpContext::CPPelse(int matchelse, TPpToken* ppToken)
int TPpContext::CPPelse(int matchelse, TPpToken * ppToken)
{ {
int atom; int atom;
int depth = 0; int depth = 0;
int token = currentInput->scan(this, currentInput, ppToken); int token = currentInput->scan(this, currentInput, ppToken);
while (token > 0) { while (token != EOF) {
if (token != '#') { if (token != '#') {
while (token != '\n') while (token != '\n' && token != EOF)
token = currentInput->scan(this, currentInput, ppToken); token = currentInput->scan(this, currentInput, ppToken);
if (token == EOF)
return EOF;
token = currentInput->scan(this, currentInput, ppToken); token = currentInput->scan(this, currentInput, ppToken);
continue; continue;
@ -277,6 +310,7 @@ int TPpContext::CPPelse(int matchelse, TPpToken * ppToken)
ifdepth++; ifdepth++;
elsetracker++; elsetracker++;
} else if (atom == endifAtom) { } else if (atom == endifAtom) {
token = extraTokenCheck(atom, ppToken, currentInput->scan(this, currentInput, ppToken));
elsedepth[elsetracker] = 0; elsedepth[elsetracker] = 0;
--elsetracker; --elsetracker;
if (depth == 0) { if (depth == 0) {
@ -289,13 +323,8 @@ int TPpContext::CPPelse(int matchelse, TPpToken * ppToken)
--ifdepth; --ifdepth;
} else if (matchelse && depth == 0) { } else if (matchelse && depth == 0) {
if (atom == elseAtom ) { if (atom == elseAtom ) {
token = extraTokenCheck(atom, ppToken, currentInput->scan(this, currentInput, ppToken));
// found the #else we are looking for // found the #else we are looking for
token = currentInput->scan(this, currentInput, ppToken);
if (token != '\n') {
parseContext.warn(ppToken->loc, "#else", "unexpected tokens following #else directive - expected a newline", "");
while (token != '\n')
token = currentInput->scan(this, currentInput, ppToken);
}
break; break;
} else if (atom == elifAtom) { } else if (atom == elifAtom) {
/* we decrement ifdepth here, because CPPif will increment /* we decrement ifdepth here, because CPPif will increment
@ -306,11 +335,48 @@ int TPpContext::CPPelse(int matchelse, TPpToken * ppToken)
--elsetracker; --elsetracker;
} }
return CPPif (ppToken); return CPPif(ppToken);
} }
} else if ((atom == elseAtom) && (!ChkCorrectElseNesting())) } else if (atom == elseAtom || atom == elifAtom) {
parseContext.error(ppToken->loc, "#else after #else", "#else", ""); if (! ChkCorrectElseNesting()) {
}; // end while if (atom == elseAtom)
parseContext.error(ppToken->loc, "#else after #else", "#else", "");
else
parseContext.error(ppToken->loc, "#elif after #else", "#else", "");
}
if (atom == elseAtom)
token = extraTokenCheck(atom, ppToken, currentInput->scan(this, currentInput, ppToken));
}
}
return token;
}
int TPpContext::extraTokenCheck(int atom, TPpToken* ppToken, int token)
{
if (token != '\n') {
static const char* message = "unexpected tokens following directive";
const char* label;
if (atom == elseAtom)
label = "#else";
else if (atom == elifAtom)
label = "#elif";
else if (atom == endifAtom)
label = "#endif";
else if (atom == ifAtom)
label = "#if";
else
label = "";
if (parseContext.messages & EShMsgRelaxedErrors)
parseContext.warn(ppToken->loc, message, label, "");
else
parseContext.error(ppToken->loc, message, label, "");
while (token != '\n')
token = currentInput->scan(this, currentInput, ppToken);
}
return token; return token;
} }
@ -491,7 +557,7 @@ int TPpContext::eval(int token, int prec, int *res, int *err, TPpToken * ppToken
return token; return token;
} // eval } // eval
int TPpContext::CPPif (TPpToken * ppToken) int TPpContext::CPPif(TPpToken* ppToken)
{ {
int token = currentInput->scan(this, currentInput, ppToken); int token = currentInput->scan(this, currentInput, ppToken);
int res = 0, err = 0; int res = 0, err = 0;
@ -503,17 +569,12 @@ int TPpContext::CPPif (TPpToken * ppToken)
return 0; return 0;
} }
token = eval(token, MIN_PREC, &res, &err, ppToken); token = eval(token, MIN_PREC, &res, &err, ppToken);
if (token != '\n') { token = extraTokenCheck(ifAtom, ppToken, token);
parseContext.warn(ppToken->loc, "unexpected tokens following #if directive - expected a newline", "#if", ""); if (!res && !err)
while (token != '\n')
token = currentInput->scan(this, currentInput, ppToken);
}
if (!res && !err) {
token = CPPelse(1, ppToken); token = CPPelse(1, ppToken);
}
return token; return token;
} // CPPif }
int TPpContext::CPPifdef(int defined, TPpToken * ppToken) int TPpContext::CPPifdef(int defined, TPpToken * ppToken)
{ {
@ -540,8 +601,9 @@ int TPpContext::CPPifdef(int defined, TPpToken * ppToken)
if (((s && !s->mac.undef) ? 1 : 0) != defined) if (((s && !s->mac.undef) ? 1 : 0) != defined)
token = CPPelse(1, ppToken); token = CPPelse(1, ppToken);
} }
return token; return token;
} // CPPifdef }
// Handle #line // Handle #line
int TPpContext::CPPline(TPpToken * ppToken) int TPpContext::CPPline(TPpToken * ppToken)
@ -746,15 +808,9 @@ int TPpContext::readCPPline(TPpToken * ppToken)
token = CPPdefine(ppToken); token = CPPdefine(ppToken);
} else if (ppToken->atom == elseAtom) { } else if (ppToken->atom == elseAtom) {
if (ChkCorrectElseNesting()) { if (ChkCorrectElseNesting()) {
if (! ifdepth) { if (! ifdepth)
parseContext.error(ppToken->loc, "mismatched statements", "#else", ""); parseContext.error(ppToken->loc, "mismatched statements", "#else", "");
} token = extraTokenCheck(elseAtom, ppToken, currentInput->scan(this, currentInput, ppToken));
token = currentInput->scan(this, currentInput, ppToken);
if (token != '\n') {
parseContext.warn(ppToken->loc, "unexpected tokens following #else directive - expected a newline", "#else", "");
while (token != '\n')
token = currentInput->scan(this, currentInput, ppToken);
}
token = CPPelse(0, ppToken); token = CPPelse(0, ppToken);
} else { } else {
parseContext.error(ppToken->loc, "#else after a #else", "#else", ""); parseContext.error(ppToken->loc, "#else after a #else", "#else", "");
@ -762,9 +818,8 @@ int TPpContext::readCPPline(TPpToken * ppToken)
return 0; return 0;
} }
} else if (ppToken->atom == elifAtom) { } else if (ppToken->atom == elifAtom) {
if (! ifdepth) { if (! ifdepth)
parseContext.error(ppToken->loc, "mismatched statements", "#elif", ""); parseContext.error(ppToken->loc, "mismatched statements", "#elif", "");
}
// this token is really a dont care, but we still need to eat the tokens // this token is really a dont care, but we still need to eat the tokens
token = currentInput->scan(this, currentInput, ppToken); token = currentInput->scan(this, currentInput, ppToken);
while (token != '\n') while (token != '\n')
@ -777,6 +832,7 @@ int TPpContext::readCPPline(TPpToken * ppToken)
parseContext.error(ppToken->loc, "mismatched statements", "#endif", ""); parseContext.error(ppToken->loc, "mismatched statements", "#endif", "");
else else
--ifdepth; --ifdepth;
token = extraTokenCheck(endifAtom, ppToken, currentInput->scan(this, currentInput, ppToken));
} else if (ppToken->atom == ifAtom) { } else if (ppToken->atom == ifAtom) {
token = CPPif (ppToken); token = CPPif (ppToken);
} else if (ppToken->atom == ifdefAtom) { } else if (ppToken->atom == ifdefAtom) {
@ -800,12 +856,12 @@ int TPpContext::readCPPline(TPpToken * ppToken)
parseContext.error(ppToken->loc, "Invalid Directive", "#", GetAtomString(ppToken->atom)); parseContext.error(ppToken->loc, "Invalid Directive", "#", GetAtomString(ppToken->atom));
} }
} }
while (token != '\n' && token != 0 && token != EOF) {
while (token != '\n' && token != 0 && token != EOF)
token = currentInput->scan(this, currentInput, ppToken); token = currentInput->scan(this, currentInput, ppToken);
}
return token; return token;
} // readCPPline }
void TPpContext::FreeMacro(MacroSymbol *s) { void TPpContext::FreeMacro(MacroSymbol *s) {
DeleteTokenStream(s->body); DeleteTokenStream(s->body);
@ -843,9 +899,10 @@ TPpContext::TokenStream* TPpContext::PrescanMacroArg(TokenStream *a, TPpToken *
token = ReadToken(a, ppToken); token = ReadToken(a, ppToken);
if (token == CPP_IDENTIFIER && LookUpSymbol(ppToken->atom)) if (token == CPP_IDENTIFIER && LookUpSymbol(ppToken->atom))
break; break;
} while (token > 0); } while (token != EOF);
if (token <= 0) return a; if (token == EOF)
n = NewTokenStream("macro arg", 0); return a;
n = NewTokenStream(0);
PushEofSrc(); PushEofSrc();
ReadFromTokenStream(a, 0, 0); ReadFromTokenStream(a, 0, 0);
while ((token = currentInput->scan(this, currentInput, ppToken)) > 0) { while ((token = currentInput->scan(this, currentInput, ppToken)) > 0) {
@ -855,14 +912,15 @@ TPpContext::TokenStream* TPpContext::PrescanMacroArg(TokenStream *a, TPpToken *
} }
PopEofSrc(); PopEofSrc();
DeleteTokenStream(a); DeleteTokenStream(a);
return n; return n;
} // PrescanMacroArg }
// //
// These are called through function pointers // These are called through function pointers
// //
/* macro_scan --- /*
** return the next token for a macro expansion, handling macro args ** return the next token for a macro expansion, handling macro args
*/ */
int TPpContext::macro_scan(TPpContext* pp, TPpContext::InputSrc* inInput, TPpToken* ppToken) int TPpContext::macro_scan(TPpContext* pp, TPpContext::InputSrc* inInput, TPpToken* ppToken)
@ -870,7 +928,11 @@ int TPpContext::macro_scan(TPpContext* pp, TPpContext::InputSrc* inInput, TPpTok
TPpContext::MacroInputSrc* in = (TPpContext::MacroInputSrc*)inInput; TPpContext::MacroInputSrc* in = (TPpContext::MacroInputSrc*)inInput;
int i; int i;
int token = pp->ReadToken(in->mac->body, ppToken); int token;
do {
token = pp->ReadToken(in->mac->body, ppToken);
} while (token == ' '); // handle white space in macro
// TODO: preprocessor: properly handle whitespace (or lack of it) between tokens when expanding
if (token == CPP_IDENTIFIER) { if (token == CPP_IDENTIFIER) {
for (i = in->mac->argc-1; i>=0; i--) for (i = in->mac->argc-1; i>=0; i--)
if (in->mac->args[i] == ppToken->atom) if (in->mac->args[i] == ppToken->atom)
@ -882,7 +944,7 @@ int TPpContext::macro_scan(TPpContext* pp, TPpContext::InputSrc* inInput, TPpTok
} }
} }
if (token > 0) if (token != EOF)
return token; return token;
in->mac->busy = 0; in->mac->busy = 0;
@ -895,7 +957,7 @@ int TPpContext::macro_scan(TPpContext* pp, TPpContext::InputSrc* inInput, TPpTok
free(in); free(in);
return pp->currentInput->scan(pp, pp->currentInput, ppToken); return pp->currentInput->scan(pp, pp->currentInput, ppToken);
} // macro_scan }
// return a zero, for scanning a macro that was never defined // return a zero, for scanning a macro that was never defined
int TPpContext::zero_scan(TPpContext* pp, InputSrc *inInput, TPpToken* ppToken) int TPpContext::zero_scan(TPpContext* pp, InputSrc *inInput, TPpToken* ppToken)
@ -912,7 +974,7 @@ int TPpContext::zero_scan(TPpContext* pp, InputSrc *inInput, TPpToken* ppToken)
return CPP_INTCONSTANT; return CPP_INTCONSTANT;
} }
/* MacroExpand /*
** Check an identifier (atom) to see if it is a macro that should be expanded. ** Check an identifier (atom) to see if it is a macro that should be expanded.
** If it is, push an InputSrc that will produce the appropriate expansion ** If it is, push an InputSrc that will produce the appropriate expansion
** and return 1. ** and return 1.
@ -983,7 +1045,7 @@ int TPpContext::MacroExpand(int atom, TPpToken* ppToken, int expandUndef)
} }
in->args = (TokenStream**)malloc(in->mac->argc * sizeof(TokenStream *)); in->args = (TokenStream**)malloc(in->mac->argc * sizeof(TokenStream *));
for (i = 0; i < in->mac->argc; i++) for (i = 0; i < in->mac->argc; i++)
in->args[i] = NewTokenStream("macro arg", 0); in->args[i] = NewTokenStream(0);
i = 0; i = 0;
j = 0; j = 0;
do { do {
@ -1034,15 +1096,7 @@ int TPpContext::MacroExpand(int atom, TPpToken* ppToken, int expandUndef)
in->args[i] = PrescanMacroArg(in->args[i], ppToken); in->args[i] = PrescanMacroArg(in->args[i], ppToken);
} }
} }
#if 0
printf(" <%s:%d>found macro %s\n", GetAtomString(atable, loc.file),
loc.line, GetAtomString(atable, atom));
for (i = 0; i<in->mac->argc; i++) {
printf("\targ %s = '", GetAtomString(atable, in->mac->args[i]));
DumpTokenStream(stdout, in->args[i]);
printf("'\n");
}
#endif
/*retain the input source*/ /*retain the input source*/
in->base.prev = currentInput; in->base.prev = currentInput;
sym->mac.busy = 1; sym->mac.busy = 1;
@ -1050,7 +1104,7 @@ int TPpContext::MacroExpand(int atom, TPpToken* ppToken, int expandUndef)
currentInput = &in->base; currentInput = &in->base;
return 1; return 1;
} // MacroExpand }
int TPpContext::ChkCorrectElseNesting() int TPpContext::ChkCorrectElseNesting()
{ {

10
glslang/MachineIndependent/preprocessor/PpAtom.cpp
View File

@ -150,8 +150,12 @@ const char *TPpContext::GetAtomString(int atom)
return "<null atom>"; return "<null atom>";
if (atom < 0) if (atom < 0)
return "<EOF>"; return "<EOF>";
if ((size_t)atom < stringMap.size()) if ((size_t)atom < stringMap.size()) {
return stringMap[atom]->c_str(); if (stringMap[atom] == 0)
return "<invalid atom>";
else
return stringMap[atom]->c_str();
}
return "<invalid atom>"; return "<invalid atom>";
} }
@ -163,7 +167,7 @@ int TPpContext::AddAtomFixed(const char *s, int atom)
{ {
TAtomMap::const_iterator it = atomMap.insert(std::pair<TString, int>(s, atom)).first; TAtomMap::const_iterator it = atomMap.insert(std::pair<TString, int>(s, atom)).first;
if (stringMap.size() < (size_t)atom + 1) if (stringMap.size() < (size_t)atom + 1)
stringMap.resize(atom + 100); stringMap.resize(atom + 100, 0);
stringMap[atom] = &it->first; stringMap[atom] = &it->first;
return atom; return atom;

6
glslang/MachineIndependent/preprocessor/PpContext.cpp
View File

@ -106,9 +106,9 @@ void TPpContext::setInput(TInputScanner& input, bool versionWillBeError)
StringInputSrc *in = (StringInputSrc *)malloc(sizeof(StringInputSrc)); StringInputSrc *in = (StringInputSrc *)malloc(sizeof(StringInputSrc));
memset(in, 0, sizeof(StringInputSrc)); memset(in, 0, sizeof(StringInputSrc));
in->input = &input; in->input = &input;
in->base.scan = byte_scan; in->base.scan = sourceScan;
in->base.getch = (int (*)(TPpContext*, InputSrc *, TPpToken *))str_getch; in->base.getch = (int (*)(TPpContext*, InputSrc *, TPpToken *))sourceGetCh;
in->base.ungetch = (void (*)(TPpContext*, InputSrc *, int, TPpToken *))str_ungetch; in->base.ungetch = (void (*)(TPpContext*, InputSrc *, int, TPpToken *))sourceUngetCh;
in->base.prev = currentInput; in->base.prev = currentInput;
currentInput = &in->base; currentInput = &in->base;
errorOnVersion = versionWillBeError; errorOnVersion = versionWillBeError;

45
glslang/MachineIndependent/preprocessor/PpContext.h
View File

@ -84,7 +84,21 @@ namespace glslang {
class TPpToken { class TPpToken {
public: public:
TPpToken() { loc.line = 0; loc.string = 0; name[0] = 0; } TPpToken() : token(0), ival(0), dval(0.0), atom(0)
{
loc.line = 0;
loc.string = 0;
name[0] = 0;
}
bool operator==(const TPpToken& right)
{
return token == right.token && atom == right.atom &&
ival == right.ival && dval == right.dval &&
strcmp(name, right.name) == 0;
}
bool operator!=(const TPpToken& right) { return ! operator==(right); }
static const int maxTokenLength = 1024; static const int maxTokenLength = 1024;
TSourceLoc loc; TSourceLoc loc;
@ -110,6 +124,7 @@ public:
const char* tokenize(TPpToken* ppToken); const char* tokenize(TPpToken* ppToken);
// TODO: preprocessor simplification: this should be a base class, not a set of function pointers
struct InputSrc { struct InputSrc {
struct InputSrc *prev; struct InputSrc *prev;
int (*scan)(TPpContext*, struct InputSrc *, TPpToken *); int (*scan)(TPpContext*, struct InputSrc *, TPpToken *);
@ -127,7 +142,6 @@ public:
struct TokenStream { struct TokenStream {
TokenStream *next; TokenStream *next;
char *name;
TokenBlock *head; TokenBlock *head;
TokenBlock *current; TokenBlock *current;
}; };
@ -231,6 +245,7 @@ protected:
int CPPdefine(TPpToken * ppToken); int CPPdefine(TPpToken * ppToken);
int CPPundef(TPpToken * ppToken); int CPPundef(TPpToken * ppToken);
int CPPelse(int matchelse, TPpToken * ppToken); int CPPelse(int matchelse, TPpToken * ppToken);
int extraTokenCheck(int atom, TPpToken* ppToken, int token);
int eval(int token, int prec, int *res, int *err, TPpToken * ppToken); int eval(int token, int prec, int *res, int *err, TPpToken * ppToken);
int CPPif (TPpToken * ppToken); int CPPif (TPpToken * ppToken);
int CPPifdef(int defined, TPpToken * ppToken); int CPPifdef(int defined, TPpToken * ppToken);
@ -259,28 +274,26 @@ protected:
// //
// From PpTokens.cpp // From PpTokens.cpp
// //
char* idstr(const char *fstr, MemoryPool *pool);
TPpContext::TokenBlock* lNewBlock(TokenStream *fTok, MemoryPool *pool); TPpContext::TokenBlock* lNewBlock(TokenStream *fTok, MemoryPool *pool);
void lAddByte(TokenStream *fTok, unsigned char fVal); void lAddByte(TokenStream *fTok, unsigned char fVal);
int lReadByte(TokenStream *pTok); int lReadByte(TokenStream *pTok);
TokenStream *NewTokenStream(const char *name, MemoryPool *pool); TokenStream *NewTokenStream(MemoryPool *pool);
void DeleteTokenStream(TokenStream *pTok); void DeleteTokenStream(TokenStream *pTok);
void RecordToken(TokenStream *pTok, int token, TPpToken * ppToken); void RecordToken(TokenStream* pTok, int token, TPpToken* ppToken);
void RewindTokenStream(TokenStream *pTok); void RewindTokenStream(TokenStream *pTok);
int ReadToken(TokenStream *pTok, TPpToken * ppToken); int ReadToken(TokenStream* pTok, TPpToken* ppToken);
int ReadFromTokenStream(TokenStream *ts, int name, int (*final)(TPpContext *)); int ReadFromTokenStream(TokenStream *ts, int name, int (*final)(TPpContext *));
void UngetToken(int token, TPpToken * ppToken); void UngetToken(int token, TPpToken* ppToken);
void DumpTokenStream(FILE *fp, TokenStream *s, TPpToken * ppToken);
struct TokenInputSrc { struct TokenInputSrc {
InputSrc base; InputSrc base;
TokenStream *tokens; TokenStream *tokens;
int (*final)(TPpContext *); int (*final)(TPpContext *);
}; };
static int scan_token(TPpContext*, TokenInputSrc *in, TPpToken * ppToken); static int scan_token(TPpContext*, TokenInputSrc *in, TPpToken * ppToken);
struct UngotToken { struct UngotToken {
InputSrc base; InputSrc base;
int token; int token;
TPpToken lval; TPpToken lval;
}; };
static int reget_token(TPpContext *, UngotToken *t, TPpToken * ppToken); static int reget_token(TPpContext *, UngotToken *t, TPpToken * ppToken);
@ -292,12 +305,12 @@ protected:
TInputScanner* input; TInputScanner* input;
}; };
int InitScanner(TPpContext *cpp); int InitScanner(TPpContext *cpp);
static int str_getch(TPpContext*, StringInputSrc *in); static int sourceGetCh(TPpContext*, StringInputSrc *in);
static void str_ungetch(TPpContext*, StringInputSrc *in, int ch, TPpToken *type); static void sourceUngetCh(TPpContext*, StringInputSrc *in, int ch, TPpToken *type);
int ScanFromString(char *s); int ScanFromString(char *s);
int check_EOF(int token); bool check_EOF(int token);
int lFloatConst(char *str, int len, int ch, TPpToken * ppToken); int lFloatConst(char *str, int len, int ch, TPpToken * ppToken);
static int byte_scan(TPpContext*, InputSrc *in, TPpToken * ppToken); static int sourceScan(TPpContext*, InputSrc *in, TPpToken * ppToken);
// //
// From PpAtom.cpp // From PpAtom.cpp

48
glslang/MachineIndependent/preprocessor/PpScanner.cpp
View File

@ -105,7 +105,7 @@ int TPpContext::InitScanner(TPpContext *cpp)
return 1; return 1;
} }
int TPpContext::str_getch(TPpContext* pp, StringInputSrc *in) int TPpContext::sourceGetCh(TPpContext* pp, StringInputSrc *in)
{ {
int ch = in->input->get(); int ch = in->input->get();
@ -115,7 +115,7 @@ int TPpContext::str_getch(TPpContext* pp, StringInputSrc *in)
return ch; return ch;
} }
void TPpContext::str_ungetch(TPpContext* pp, StringInputSrc *in, int ch, TPpToken *type) void TPpContext::sourceUngetCh(TPpContext* pp, StringInputSrc *in, int ch, TPpToken *type)
{ {
in->input->unget(); in->input->unget();
} }
@ -130,7 +130,7 @@ void TPpContext::str_ungetch(TPpContext* pp, StringInputSrc *in, int ch, TPpToke
* letter 'e', or a precision ending (e.g., F or LF). * letter 'e', or a precision ending (e.g., F or LF).
*/ */
int TPpContext::lFloatConst(char *str, int len, int ch, TPpToken * ppToken) int TPpContext::lFloatConst(char* str, int len, int ch, TPpToken* ppToken)
{ {
bool HasDecimalOrExponent = false; bool HasDecimalOrExponent = false;
int declen, exp, ExpSign; int declen, exp, ExpSign;
@ -241,13 +241,12 @@ int TPpContext::lFloatConst(char *str, int len, int ch, TPpToken * ppToken)
return CPP_DOUBLECONSTANT; return CPP_DOUBLECONSTANT;
else else
return CPP_FLOATCONSTANT; return CPP_FLOATCONSTANT;
} // lFloatConst }
/////////////////////////////////////////////////////////////////////////////////////////////// //
///////////////////////////////////////// Normal Scanner ////////////////////////////////////// // Scanner used to tokenize source stream.
/////////////////////////////////////////////////////////////////////////////////////////////// //
int TPpContext::sourceScan(TPpContext* pp, InputSrc*, TPpToken* ppToken)
int TPpContext::byte_scan(TPpContext* pp, InputSrc *in, TPpToken * ppToken)
{ {
char tokenText[TPpToken::maxTokenLength + 1]; char tokenText[TPpToken::maxTokenLength + 1];
int AlreadyComplained = 0; int AlreadyComplained = 0;
@ -268,8 +267,6 @@ int TPpContext::byte_scan(TPpContext* pp, InputSrc *in, TPpToken * ppToken)
switch (ch) { switch (ch) {
default: default:
return ch; // Single character token return ch; // Single character token
case EOF:
return EOF;
case 'A': case 'B': case 'C': case 'D': case 'E': case 'A': case 'B': case 'C': case 'D': case 'E':
case 'F': case 'G': case 'H': case 'I': case 'J': case 'F': case 'G': case 'H': case 'I': case 'J':
case 'K': case 'L': case 'M': case 'N': case 'O': case 'K': case 'L': case 'M': case 'N': case 'O':
@ -704,14 +701,17 @@ int TPpContext::byte_scan(TPpContext* pp, InputSrc *in, TPpToken * ppToken)
} }
} }
} }
} // byte_scan }
//
// Return string pointer to next token.
// Return 0 when no more tokens.
//
const char* TPpContext::tokenize(TPpToken* ppToken) const char* TPpContext::tokenize(TPpToken* ppToken)
{ {
int token = '\n'; int token = '\n';
for(;;) { for(;;) {
const char* tokenString = 0; const char* tokenString = 0;
token = currentInput->scan(this, currentInput, ppToken); token = currentInput->scan(this, currentInput, ppToken);
ppToken->token = token; ppToken->token = token;
@ -742,7 +742,13 @@ const char* TPpContext::tokenize(TPpToken* ppToken)
else if (token == CPP_INTCONSTANT || token == CPP_UINTCONSTANT || else if (token == CPP_INTCONSTANT || token == CPP_UINTCONSTANT ||
token == CPP_FLOATCONSTANT || token == CPP_DOUBLECONSTANT) token == CPP_FLOATCONSTANT || token == CPP_DOUBLECONSTANT)
tokenString = ppToken->name; tokenString = ppToken->name;
else else if (token == CPP_STRCONSTANT) {
parseContext.error(ppToken->loc, "string literals not supported", "\"\"", "");
tokenString = 0;
} else if (token == '\'') {
parseContext.error(ppToken->loc, "character literals not supported", "\'", "");
tokenString = 0;
} else
tokenString = GetAtomString(token); tokenString = GetAtomString(token);
if (tokenString) { if (tokenString) {
@ -752,19 +758,19 @@ const char* TPpContext::tokenize(TPpToken* ppToken)
return tokenString; return tokenString;
} }
} }
}
return 0;
} // PpTokenize
//Checks if the token just read is EOF or not. //Checks if the token just read is EOF or not.
int TPpContext::check_EOF(int token) bool TPpContext::check_EOF(int token)
{ {
if (token == EOF) { if (token == EOF) {
if (ifdepth > 0) if (ifdepth > 0)
parseContext.error(parseContext.getCurrentLoc(), "missing #endif", "#if", ""); parseContext.error(parseContext.getCurrentLoc(), "missing #endif", "", "");
return 1;
return true;
} }
return 0;
return false;
} }
} // end namespace glslang } // end namespace glslang

234
glslang/MachineIndependent/preprocessor/PpTokens.cpp
View File

@ -75,9 +75,11 @@ NVIDIA SOFTWARE, HOWEVER CAUSED AND WHETHER UNDER THEORY OF CONTRACT,
TORT (INCLUDING NEGLIGENCE), STRICT LIABILITY OR OTHERWISE, EVEN IF TORT (INCLUDING NEGLIGENCE), STRICT LIABILITY OR OTHERWISE, EVEN IF
NVIDIA HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. NVIDIA HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
\****************************************************************************/ \****************************************************************************/
// //
// tokens.c // For recording and playing back the stream of tokens in a macro definition.
// //
#ifdef _WIN32 #ifdef _WIN32
#define _CRT_SECURE_NO_WARNINGS #define _CRT_SECURE_NO_WARNINGS
#define snprintf sprintf_s #define snprintf sprintf_s
@ -94,43 +96,6 @@ NVIDIA HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
namespace glslang { namespace glslang {
///////////////////////////////////////////////////////////////////////////////////////////////
//////////////////////// Preprocessor and Token Recorder and Playback: ////////////////////////
///////////////////////////////////////////////////////////////////////////////////////////////
/*
* idstr()
* Copy a string to a malloc'ed block and convert it into something suitable
* for an ID
*
*/
char* TPpContext::idstr(const char *fstr, MemoryPool *pool)
{
size_t len;
char *str, *t;
const char *f;
len = strlen(fstr);
if (!pool)
str = (char *) malloc(len + 1);
else
str = (char *) mem_Alloc(pool, len + 1);
for (f=fstr, t=str; *f; f++) {
if (isalnum(*f)) *t++ = *f;
else if (*f == '.' || *f == '/') *t++ = '_';
}
*t = 0;
return str;
} // idstr
/*
* lNewBlock()
*
*/
TPpContext::TokenBlock* TPpContext::lNewBlock(TokenStream *fTok, MemoryPool *pool) TPpContext::TokenBlock* TPpContext::lNewBlock(TokenStream *fTok, MemoryPool *pool)
{ {
TokenBlock *lBlock; TokenBlock *lBlock;
@ -152,12 +117,7 @@ TPpContext::TokenBlock* TPpContext::lNewBlock(TokenStream *fTok, MemoryPool *poo
fTok->current = lBlock; fTok->current = lBlock;
return lBlock; return lBlock;
} // lNewBlock }
/*
* lAddByte()
*
*/
void TPpContext::lAddByte(TokenStream *fTok, unsigned char fVal) void TPpContext::lAddByte(TokenStream *fTok, unsigned char fVal)
{ {
@ -166,14 +126,11 @@ void TPpContext::lAddByte(TokenStream *fTok, unsigned char fVal)
if (lBlock->count >= lBlock->max) if (lBlock->count >= lBlock->max)
lBlock = lNewBlock(fTok, 0); lBlock = lNewBlock(fTok, 0);
lBlock->data[lBlock->count++] = fVal; lBlock->data[lBlock->count++] = fVal;
} // lAddByte }
/* /*
* lReadByte() - Get the next byte from a stream. * Get the next byte from a stream.
*
*/ */
int TPpContext::lReadByte(TokenStream *pTok) int TPpContext::lReadByte(TokenStream *pTok)
{ {
TokenBlock *lBlock; TokenBlock *lBlock;
@ -191,16 +148,12 @@ int TPpContext::lReadByte(TokenStream *pTok)
lval = lBlock->data[lBlock->current++]; lval = lBlock->data[lBlock->current++];
} }
return lval; return lval;
} // lReadByte }
/////////////////////////////////////// Global Functions:////////////////////////////////////// //
// Make a token stream (used for reprocessing macros).
/* //
* NewTokenStream() TPpContext::TokenStream* TPpContext::NewTokenStream(MemoryPool *pool)
*
*/
TPpContext::TokenStream* TPpContext::NewTokenStream(const char *name, MemoryPool *pool)
{ {
TokenStream *pTok; TokenStream *pTok;
@ -209,17 +162,11 @@ TPpContext::TokenStream* TPpContext::NewTokenStream(const char *name, MemoryPool
else else
pTok = (TokenStream*)mem_Alloc(pool, sizeof(TokenStream)); pTok = (TokenStream*)mem_Alloc(pool, sizeof(TokenStream));
pTok->next = NULL; pTok->next = NULL;
pTok->name = idstr(name, pool);
pTok->head = NULL; pTok->head = NULL;
pTok->current = NULL; pTok->current = NULL;
lNewBlock(pTok, pool); lNewBlock(pTok, pool);
return pTok; return pTok;
} // NewTokenStream }
/*
* DeleteTokenStream()
*
*/
void TPpContext::DeleteTokenStream(TokenStream *pTok) void TPpContext::DeleteTokenStream(TokenStream *pTok)
{ {
@ -232,18 +179,14 @@ void TPpContext::DeleteTokenStream(TokenStream *pTok)
free(pBlock); free(pBlock);
pBlock = nBlock; pBlock = nBlock;
} }
if (pTok->name)
free(pTok->name);
free(pTok); free(pTok);
} }
} // DeleteTokenStream }
/* /*
* RecordToken() - Add a token to the end of a list for later playback or printout. * Add a token to the end of a list for later playback.
*
*/ */
void TPpContext::RecordToken(TokenStream *pTok, int token, TPpToken* ppToken)
void TPpContext::RecordToken(TokenStream *pTok, int token, TPpToken * ppToken)
{ {
const char *s; const char *s;
char *str = NULL; char *str = NULL;
@ -254,7 +197,6 @@ void TPpContext::RecordToken(TokenStream *pTok, int token, TPpToken * ppToken)
lAddByte(pTok, (unsigned char)(token & 0x7f)); lAddByte(pTok, (unsigned char)(token & 0x7f));
switch (token) { switch (token) {
case CPP_IDENTIFIER: case CPP_IDENTIFIER:
case CPP_TYPEIDENTIFIER:
case CPP_STRCONSTANT: case CPP_STRCONSTANT:
s = GetAtomString(ppToken->atom); s = GetAtomString(ppToken->atom);
while (*s) while (*s)
@ -274,33 +216,28 @@ void TPpContext::RecordToken(TokenStream *pTok, int token, TPpToken * ppToken)
break; break;
case '(': case '(':
lAddByte(pTok, (unsigned char)(ppToken->ival ? 1 : 0)); lAddByte(pTok, (unsigned char)(ppToken->ival ? 1 : 0));
break;
default: default:
break; break;
} }
} // RecordToken }
/* /*
* RewindTokenStream() - Reset a token stream in preperation for reading. * Reset a token stream in preperation for reading.
*
*/ */
void TPpContext::RewindTokenStream(TokenStream *pTok) void TPpContext::RewindTokenStream(TokenStream *pTok)
{ {
if (pTok->head) { if (pTok->head) {
pTok->current = pTok->head; pTok->current = pTok->head;
pTok->current->current = 0; pTok->current->current = 0;
} }
} // RewindTokenStream }
/* /*
* ReadToken() - Read the next token from a stream. * Read the next token from a token stream (not the source stream, but stream used to hold a tokenized macro).
*
*/ */
int TPpContext::ReadToken(TokenStream *pTok, TPpToken *ppToken) int TPpContext::ReadToken(TokenStream *pTok, TPpToken *ppToken)
{ {
//TODO: preprocessor simplification: why is this different than byte_scan
char tokenText[TPpToken::maxTokenLength + 1]; char tokenText[TPpToken::maxTokenLength + 1];
int ltoken, len; int ltoken, len;
char ch; char ch;
@ -311,89 +248,56 @@ int TPpContext::ReadToken(TokenStream *pTok, TPpToken *ppToken)
if (ltoken > 127) if (ltoken > 127)
ltoken += 128; ltoken += 128;
switch (ltoken) { switch (ltoken) {
case CPP_IDENTIFIER: case '(':
case CPP_TYPEIDENTIFIER: ppToken->ival = lReadByte(pTok);
len = 0;
ch = lReadByte(pTok);
while ((ch >= 'a' && ch <= 'z') ||
(ch >= 'A' && ch <= 'Z') ||
(ch >= '0' && ch <= '9') ||
ch == '_')
{
if (len < TPpToken::maxTokenLength) {
tokenText[len] = ch;
len++;
ch = lReadByte(pTok);
} else {
parseContext.error(ppToken->loc,"name too long", "", "");
break;
}
}
tokenText[len] = '\0';
assert(ch == '\0');
ppToken->atom = LookUpAddString(tokenText);
return CPP_IDENTIFIER;
break; break;
case CPP_STRCONSTANT: case CPP_STRCONSTANT:
len = 0; case CPP_IDENTIFIER:
while ((ch = lReadByte(pTok)) != 0) {
if (len < TPpToken::maxTokenLength)
tokenText[len++] = ch;
else
break;
}
tokenText[len] = 0;
ppToken->atom = LookUpAddString(tokenText);
break;
case CPP_FLOATCONSTANT: case CPP_FLOATCONSTANT:
case CPP_DOUBLECONSTANT: case CPP_DOUBLECONSTANT:
len = 0;
ch = lReadByte(pTok);
while ((ch >= '0' && ch <= '9') || ch=='e' || ch=='E' || ch=='.' || ch=='+' || ch=='-' || ch=='l' || ch=='L' || ch=='f'|| ch=='F')
{
if (len < TPpToken::maxTokenLength) {
tokenText[len] = ch;
len++;
ch = lReadByte(pTok);
} else {
parseContext.error(ppToken->loc,"float literal too long", "", "");
break;
}
}
tokenText[len] = '\0';
assert(ch == '\0');
strcpy(ppToken->name, tokenText);
ppToken->dval = atof(ppToken->name);
break;
case CPP_INTCONSTANT: case CPP_INTCONSTANT:
case CPP_UINTCONSTANT: case CPP_UINTCONSTANT:
len = 0; len = 0;
ch = lReadByte(pTok); ch = lReadByte(pTok);
while ((ch >= '0' && ch <= '9') || ch == 'u' || ch == 'U') while (ch != 0) {
{
if (len < TPpToken::maxTokenLength) { if (len < TPpToken::maxTokenLength) {
tokenText[len] = ch; tokenText[len] = ch;
len++; len++;
ch = lReadByte(pTok); ch = lReadByte(pTok);
} else { } else {
parseContext.error(ppToken->loc,"integer literal too long", "", ""); parseContext.error(ppToken->loc, "token too long", "", "");
break; break;
} }
} }
tokenText[len] = '\0'; tokenText[len] = 0;
assert(ch == '\0');
strcpy(ppToken->name,tokenText); switch (ltoken) {
ppToken->ival = atoi(ppToken->name); case CPP_IDENTIFIER:
break; case CPP_STRCONSTANT:
case '(': ppToken->atom = LookUpAddString(tokenText);
ppToken->ival = lReadByte(pTok); break;
break; case CPP_FLOATCONSTANT:
case CPP_DOUBLECONSTANT:
strcpy(ppToken->name, tokenText);
ppToken->dval = atof(ppToken->name);
break;
case CPP_INTCONSTANT:
case CPP_UINTCONSTANT:
strcpy(ppToken->name, tokenText);
if (len > 0 && tokenText[0] == '0') {
if (len > 1 && tokenText[1] == 'x' || tokenText[1] == 'X')
ppToken->ival = strtol(ppToken->name, 0, 16);
else
ppToken->ival = strtol(ppToken->name, 0, 8);
} else
ppToken->ival = atoi(ppToken->name);
break;
}
} }
return ltoken; return ltoken;
} }
return EOF; return EOF;
} // ReadToken }
int TPpContext::scan_token(TPpContext* pp, TokenInputSrc *in, TPpToken * ppToken) int TPpContext::scan_token(TPpContext* pp, TokenInputSrc *in, TPpToken * ppToken)
{ {
@ -434,9 +338,7 @@ int TPpContext::reget_token(TPpContext* pp, UngotToken *t, TPpToken * ppToken)
return token; return token;
} }
typedef int (*scanFnPtr_t); void TPpContext::UngetToken(int token, TPpToken* ppToken)
void TPpContext::UngetToken(int token, TPpToken * ppToken)
{ {
UngotToken *t = (UngotToken *) malloc(sizeof(UngotToken)); UngotToken *t = (UngotToken *) malloc(sizeof(UngotToken));
memset(t, 0, sizeof(UngotToken)); memset(t, 0, sizeof(UngotToken));
@ -447,38 +349,4 @@ void TPpContext::UngetToken(int token, TPpToken * ppToken)
currentInput = &t->base; currentInput = &t->base;
} }
void TPpContext::DumpTokenStream(FILE *fp, TokenStream *s, TPpToken * ppToken)
{
int token;
if (fp == 0) fp = stdout;
RewindTokenStream(s);
while ((token = ReadToken(s, ppToken)) > 0) {
switch (token) {
case CPP_IDENTIFIER:
case CPP_TYPEIDENTIFIER:
printf("%s ", GetAtomString(ppToken->atom));
break;
case CPP_STRCONSTANT:
printf("\"%s\"", GetAtomString(ppToken->atom));
break;
case CPP_FLOATCONSTANT:
case CPP_DOUBLECONSTANT:
printf("%g9.6 ", ppToken->dval);
break;
case CPP_INTCONSTANT:
case CPP_UINTCONSTANT:
printf("%d ", ppToken->ival);
break;
default:
if (token >= 127)
printf("%s ", GetAtomString(token));
else
printf("%c", token);
break;
}
}
}
} // end namespace glslang } // end namespace glslang

1
glslang/MachineIndependent/preprocessor/PpTokens.h
View File

@ -99,7 +99,6 @@ NVIDIA HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#define CPP_OR_OP 276 #define CPP_OR_OP 276
#define CPP_INC_OP 277 #define CPP_INC_OP 277
#define CPP_STRCONSTANT 278 #define CPP_STRCONSTANT 278
#define CPP_TYPEIDENTIFIER 279
#define CPP_RIGHT_ASSIGN 280 #define CPP_RIGHT_ASSIGN 280
#define CPP_LEFT_ASSIGN 281 #define CPP_LEFT_ASSIGN 281
#define CPP_AND_ASSIGN 282 #define CPP_AND_ASSIGN 282