mewin/glslang
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Fix several issues in the preprocessor:

Browse Source 
- macro expansion of hexidecimal numbers 
 - give errors instead of warnings/silence on extra tokens after #endif, #else, etc.
 - give errors on reserved macro name use, reuse of argument, and redefinition with different whitespace presence
 - detect and give error for all cases of #elif and #else after #else



git-svn-id: https://cvs.khronos.org/svn/repos/ogl/trunk/ecosystem/public/sdk/tools/glslang@23982 e7fa87d3-cd2b-0410-9028-fcbf551c1848
This commit is contained in:
John Kessenich
2013-11-10 23:07:36 +00:00
parent cfe3ba5a18
commit 4d6570a1b3
14 changed files with 403 additions and 339 deletions
Download Patch File Download Diff File Expand all files Collapse all files

236
glslang/MachineIndependent/preprocessor/Pp.cpp
View File

@@ -133,13 +133,10 @@ int TPpContext::FinalCPP()
{
mem_FreePool(pool);
if (ifdepth)
parseContext.error(parseContext.getCurrentLoc(), "missing #endif", "#if", "");
return 1;
}
int TPpContext::CPPdefine(TPpToken * ppToken)
int TPpContext::CPPdefine(TPpToken* ppToken)
{
int token, atom, args[maxMacroArgs], argc;
MacroSymbol mac;
@@ -147,10 +144,21 @@ int TPpContext::CPPdefine(TPpToken * ppToken)
memset(&mac, 0, sizeof(mac));
token = currentInput->scan(this, currentInput, ppToken);
if (token != CPP_IDENTIFIER) {
parseContext.error(ppToken->loc, "must be followed by macro atom", "#define", "");
parseContext.error(ppToken->loc, "must be followed by macro name", "#define", "");
return token;
}
atom = ppToken->atom;
const char* definedName = GetAtomString(atom);
if (ppToken->loc.string >= 0) {
// We are in user code; check for reserved name use:
// "All macro names containing two consecutive underscores ( __ ) are reserved for future use as predefined
// macro names. All macro names prefixed with "GL_" ("GL" followed by a single underscore) are also
// reserved."
if (strncmp(definedName, "GL_", 3) == 0)
parseContext.error(ppToken->loc, "reserved built-in name prefix:", "#define", "GL_");
else if (strstr(definedName, "__") != 0)
parseContext.error(ppToken->loc, "names containing consecutive underscores are reserved", "#define", "");
}
token = currentInput->scan(this, currentInput, ppToken);
if (token == '(' && !ppToken->ival) {
// gather arguments
@@ -164,8 +172,21 @@ int TPpContext::CPPdefine(TPpToken * ppToken)
return token;
}
if (argc < maxMacroArgs)
args[argc++] = ppToken->atom;
// check for duplication
bool duplicate = false;
for (int a = 0; a < argc; ++a) {
if (args[a] == ppToken->atom) {
parseContext.error(ppToken->loc, "duplicate macro parameter", "#define", "");
duplicate = true;
break;
}
}
if (! duplicate) {
if (argc < maxMacroArgs)
args[argc++] = ppToken->atom;
else
parseContext.error(ppToken->loc, "too many macro parameters", "#define", "");
}
token = currentInput->scan(this, currentInput, ppToken);
} while (token == ',');
if (token != ')') {
@@ -178,41 +199,51 @@ int TPpContext::CPPdefine(TPpToken * ppToken)
memcpy(mac.args, args, argc * sizeof(int));
token = currentInput->scan(this, currentInput, ppToken);
}
mac.body = NewTokenStream(GetAtomString(atom), pool);
TSourceLoc defineLoc = ppToken->loc; // because ppToken is going to go to the next line before we report errors
mac.body = NewTokenStream(pool);
while (token != '\n') {
while (token == '\\') {
if (token == '\\') {
token = currentInput->scan(this, currentInput, ppToken);
if (token == '\n')
token = currentInput->scan(this, currentInput, ppToken);
else
RecordToken(mac.body, '\\', ppToken);
}
RecordToken(mac.body, token, ppToken);
int spaceCandidate = currentInput->getch(this, currentInput, ppToken);
if (spaceCandidate == ' ' || spaceCandidate == '\t')
RecordToken(mac.body, ' ', 0);
else
currentInput->ungetch(this, currentInput, spaceCandidate, ppToken);
token = currentInput->scan(this, currentInput, ppToken);
};
}
symb = LookUpSymbol(atom);
if (symb) {
if (!symb->mac.undef) {
// already defined -- need to make sure they are identical
if (! symb->mac.undef) {
// Already defined -- need to make sure they are identical:
// "Two replacement lists are identical if and only if the preprocessing tokens in both have the same number,
// ordering, spelling, and white-space separation, where all white-space separations are considered identical."
if (symb->mac.argc != mac.argc)
goto error;
for (argc=0; argc < mac.argc; argc++)
if (symb->mac.args[argc] != mac.args[argc])
goto error;
RewindTokenStream(symb->mac.body);
RewindTokenStream(mac.body);
do {
int old_lval, old_token;
old_token = ReadToken(symb->mac.body, ppToken);
old_lval = ppToken->ival;
token = ReadToken(mac.body, ppToken);
if (token != old_token || ppToken->ival != old_lval) {
error:
parseContext.error(ppToken->loc, "Macro Redefined", "#define", GetAtomString(atom));
break;
parseContext.error(defineLoc, "Macro redefined; different number of arguments:", "#define", GetAtomString(atom));
else {
for (argc=0; argc < mac.argc; argc++) {
if (symb->mac.args[argc] != mac.args[argc])
parseContext.error(defineLoc, "Macro redefined; different argument names:", "#define", GetAtomString(atom));
}
} while (token > 0);
RewindTokenStream(symb->mac.body);
RewindTokenStream(mac.body);
int newToken;
do {
int oldToken;
TPpToken oldPpToken;
TPpToken newPpToken;
oldToken = ReadToken(symb->mac.body, &oldPpToken);
newToken = ReadToken(mac.body, &newPpToken);
if (oldToken != newToken || oldPpToken != newPpToken) {
parseContext.error(defineLoc, "Macro redefined; different substitutions:", "#define", GetAtomString(atom));
break;
}
} while (newToken > 0);
}
}
} else {
symb = AddSymbol(atom);
@@ -220,7 +251,7 @@ error:
symb->mac = mac;
return '\n';
} // CPPdefine
}
int TPpContext::CPPundef(TPpToken * ppToken)
{
@@ -246,23 +277,25 @@ int TPpContext::CPPundef(TPpToken * ppToken)
parseContext.error(ppToken->loc, "can only be followed by a single macro name", "#undef", "");
return token;
} // CPPundef
}
/* CPPelse -- skip forward to appropriate spot. This is actually used
/* Skip forward to appropriate spot. This is used both
** to skip to a #endif after seeing an #else, AND to skip to a #else,
** #elif, or #endif after a #if/#ifdef/#ifndef/#elif test was false
** #elif, or #endif after a #if/#ifdef/#ifndef/#elif test was false.
*/
int TPpContext::CPPelse(int matchelse, TPpToken * ppToken)
int TPpContext::CPPelse(int matchelse, TPpToken* ppToken)
{
int atom;
int depth = 0;
int token = currentInput->scan(this, currentInput, ppToken);
while (token > 0) {
while (token != EOF) {
if (token != '#') {
while (token != '\n')
while (token != '\n' && token != EOF)
token = currentInput->scan(this, currentInput, ppToken);
if (token == EOF)
return EOF;
token = currentInput->scan(this, currentInput, ppToken);
continue;
@@ -277,6 +310,7 @@ int TPpContext::CPPelse(int matchelse, TPpToken * ppToken)
ifdepth++;
elsetracker++;
} else if (atom == endifAtom) {
token = extraTokenCheck(atom, ppToken, currentInput->scan(this, currentInput, ppToken));
elsedepth[elsetracker] = 0;
--elsetracker;
if (depth == 0) {
@@ -289,13 +323,8 @@ int TPpContext::CPPelse(int matchelse, TPpToken * ppToken)
--ifdepth;
} else if (matchelse && depth == 0) {
if (atom == elseAtom ) {
token = extraTokenCheck(atom, ppToken, currentInput->scan(this, currentInput, ppToken));
// found the #else we are looking for
token = currentInput->scan(this, currentInput, ppToken);
if (token != '\n') {
parseContext.warn(ppToken->loc, "#else", "unexpected tokens following #else directive - expected a newline", "");
while (token != '\n')
token = currentInput->scan(this, currentInput, ppToken);
}
break;
} else if (atom == elifAtom) {
/* we decrement ifdepth here, because CPPif will increment
@@ -306,11 +335,48 @@ int TPpContext::CPPelse(int matchelse, TPpToken * ppToken)
--elsetracker;
}
return CPPif (ppToken);
return CPPif(ppToken);
}
} else if ((atom == elseAtom) && (!ChkCorrectElseNesting()))
parseContext.error(ppToken->loc, "#else after #else", "#else", "");
}; // end while
} else if (atom == elseAtom || atom == elifAtom) {
if (! ChkCorrectElseNesting()) {
if (atom == elseAtom)
parseContext.error(ppToken->loc, "#else after #else", "#else", "");
else
parseContext.error(ppToken->loc, "#elif after #else", "#else", "");
}
if (atom == elseAtom)
token = extraTokenCheck(atom, ppToken, currentInput->scan(this, currentInput, ppToken));
}
}
return token;
}
int TPpContext::extraTokenCheck(int atom, TPpToken* ppToken, int token)
{
if (token != '\n') {
static const char* message = "unexpected tokens following directive";
const char* label;
if (atom == elseAtom)
label = "#else";
else if (atom == elifAtom)
label = "#elif";
else if (atom == endifAtom)
label = "#endif";
else if (atom == ifAtom)
label = "#if";
else
label = "";
if (parseContext.messages & EShMsgRelaxedErrors)
parseContext.warn(ppToken->loc, message, label, "");
else
parseContext.error(ppToken->loc, message, label, "");
while (token != '\n')
token = currentInput->scan(this, currentInput, ppToken);
}
return token;
}
@@ -491,7 +557,7 @@ int TPpContext::eval(int token, int prec, int *res, int *err, TPpToken * ppToken
return token;
} // eval
int TPpContext::CPPif (TPpToken * ppToken)
int TPpContext::CPPif(TPpToken* ppToken)
{
int token = currentInput->scan(this, currentInput, ppToken);
int res = 0, err = 0;
@@ -503,17 +569,12 @@ int TPpContext::CPPif (TPpToken * ppToken)
return 0;
}
token = eval(token, MIN_PREC, &res, &err, ppToken);
if (token != '\n') {
parseContext.warn(ppToken->loc, "unexpected tokens following #if directive - expected a newline", "#if", "");
while (token != '\n')
token = currentInput->scan(this, currentInput, ppToken);
}
if (!res && !err) {
token = extraTokenCheck(ifAtom, ppToken, token);
if (!res && !err)
token = CPPelse(1, ppToken);
}
return token;
} // CPPif
}
int TPpContext::CPPifdef(int defined, TPpToken * ppToken)
{
@@ -540,8 +601,9 @@ int TPpContext::CPPifdef(int defined, TPpToken * ppToken)
if (((s && !s->mac.undef) ? 1 : 0) != defined)
token = CPPelse(1, ppToken);
}
return token;
} // CPPifdef
}
// Handle #line
int TPpContext::CPPline(TPpToken * ppToken)
@@ -746,15 +808,9 @@ int TPpContext::readCPPline(TPpToken * ppToken)
token = CPPdefine(ppToken);
} else if (ppToken->atom == elseAtom) {
if (ChkCorrectElseNesting()) {
if (! ifdepth) {
if (! ifdepth)
parseContext.error(ppToken->loc, "mismatched statements", "#else", "");
}
token = currentInput->scan(this, currentInput, ppToken);
if (token != '\n') {
parseContext.warn(ppToken->loc, "unexpected tokens following #else directive - expected a newline", "#else", "");
while (token != '\n')
token = currentInput->scan(this, currentInput, ppToken);
}
token = extraTokenCheck(elseAtom, ppToken, currentInput->scan(this, currentInput, ppToken));
token = CPPelse(0, ppToken);
} else {
parseContext.error(ppToken->loc, "#else after a #else", "#else", "");
@@ -762,9 +818,8 @@ int TPpContext::readCPPline(TPpToken * ppToken)
return 0;
}
} else if (ppToken->atom == elifAtom) {
if (! ifdepth) {
if (! ifdepth)
parseContext.error(ppToken->loc, "mismatched statements", "#elif", "");
}
// this token is really a dont care, but we still need to eat the tokens
token = currentInput->scan(this, currentInput, ppToken);
while (token != '\n')
@@ -777,6 +832,7 @@ int TPpContext::readCPPline(TPpToken * ppToken)
parseContext.error(ppToken->loc, "mismatched statements", "#endif", "");
else
--ifdepth;
token = extraTokenCheck(endifAtom, ppToken, currentInput->scan(this, currentInput, ppToken));
} else if (ppToken->atom == ifAtom) {
token = CPPif (ppToken);
} else if (ppToken->atom == ifdefAtom) {
@@ -800,12 +856,12 @@ int TPpContext::readCPPline(TPpToken * ppToken)
parseContext.error(ppToken->loc, "Invalid Directive", "#", GetAtomString(ppToken->atom));
}
}
while (token != '\n' && token != 0 && token != EOF) {
while (token != '\n' && token != 0 && token != EOF)
token = currentInput->scan(this, currentInput, ppToken);
}
return token;
} // readCPPline
}
void TPpContext::FreeMacro(MacroSymbol *s) {
DeleteTokenStream(s->body);
@@ -843,9 +899,10 @@ TPpContext::TokenStream* TPpContext::PrescanMacroArg(TokenStream *a, TPpToken *
token = ReadToken(a, ppToken);
if (token == CPP_IDENTIFIER && LookUpSymbol(ppToken->atom))
break;
} while (token > 0);
if (token <= 0) return a;
n = NewTokenStream("macro arg", 0);
} while (token != EOF);
if (token == EOF)
return a;
n = NewTokenStream(0);
PushEofSrc();
ReadFromTokenStream(a, 0, 0);
while ((token = currentInput->scan(this, currentInput, ppToken)) > 0) {
@@ -855,14 +912,15 @@ TPpContext::TokenStream* TPpContext::PrescanMacroArg(TokenStream *a, TPpToken *
}
PopEofSrc();
DeleteTokenStream(a);
return n;
} // PrescanMacroArg
}
//
// These are called through function pointers
//
/* macro_scan ---
/*
** return the next token for a macro expansion, handling macro args
*/
int TPpContext::macro_scan(TPpContext* pp, TPpContext::InputSrc* inInput, TPpToken* ppToken)
@@ -870,7 +928,11 @@ int TPpContext::macro_scan(TPpContext* pp, TPpContext::InputSrc* inInput, TPpTok
TPpContext::MacroInputSrc* in = (TPpContext::MacroInputSrc*)inInput;
int i;
int token = pp->ReadToken(in->mac->body, ppToken);
int token;
do {
token = pp->ReadToken(in->mac->body, ppToken);
} while (token == ' '); // handle white space in macro
// TODO: preprocessor: properly handle whitespace (or lack of it) between tokens when expanding
if (token == CPP_IDENTIFIER) {
for (i = in->mac->argc-1; i>=0; i--)
if (in->mac->args[i] == ppToken->atom)
@@ -882,7 +944,7 @@ int TPpContext::macro_scan(TPpContext* pp, TPpContext::InputSrc* inInput, TPpTok
}
}
if (token > 0)
if (token != EOF)
return token;
in->mac->busy = 0;
@@ -895,7 +957,7 @@ int TPpContext::macro_scan(TPpContext* pp, TPpContext::InputSrc* inInput, TPpTok
free(in);
return pp->currentInput->scan(pp, pp->currentInput, ppToken);
} // macro_scan
}
// return a zero, for scanning a macro that was never defined
int TPpContext::zero_scan(TPpContext* pp, InputSrc *inInput, TPpToken* ppToken)
@@ -912,7 +974,7 @@ int TPpContext::zero_scan(TPpContext* pp, InputSrc *inInput, TPpToken* ppToken)
return CPP_INTCONSTANT;
}
/* MacroExpand
/*
** Check an identifier (atom) to see if it is a macro that should be expanded.
** If it is, push an InputSrc that will produce the appropriate expansion
** and return 1.
@@ -983,7 +1045,7 @@ int TPpContext::MacroExpand(int atom, TPpToken* ppToken, int expandUndef)
}
in->args = (TokenStream**)malloc(in->mac->argc * sizeof(TokenStream *));
for (i = 0; i < in->mac->argc; i++)
in->args[i] = NewTokenStream("macro arg", 0);
in->args[i] = NewTokenStream(0);
i = 0;
j = 0;
do {
@@ -1034,15 +1096,7 @@ int TPpContext::MacroExpand(int atom, TPpToken* ppToken, int expandUndef)
in->args[i] = PrescanMacroArg(in->args[i], ppToken);
}
}
#if 0
printf(" <%s:%d>found macro %s\n", GetAtomString(atable, loc.file),
loc.line, GetAtomString(atable, atom));
for (i = 0; i<in->mac->argc; i++) {
printf("\targ %s = '", GetAtomString(atable, in->mac->args[i]));
DumpTokenStream(stdout, in->args[i]);
printf("'\n");
}
#endif
/*retain the input source*/
in->base.prev = currentInput;
sym->mac.busy = 1;
@@ -1050,7 +1104,7 @@ int TPpContext::MacroExpand(int atom, TPpToken* ppToken, int expandUndef)
currentInput = &in->base;
return 1;
} // MacroExpand
}
int TPpContext::ChkCorrectElseNesting()
{

10
glslang/MachineIndependent/preprocessor/PpAtom.cpp
View File

@@ -150,8 +150,12 @@ const char *TPpContext::GetAtomString(int atom)
return "<null atom>";
if (atom < 0)
return "<EOF>";
if ((size_t)atom < stringMap.size())
return stringMap[atom]->c_str();
if ((size_t)atom < stringMap.size()) {
if (stringMap[atom] == 0)
return "<invalid atom>";
else
return stringMap[atom]->c_str();
}
return "<invalid atom>";
}
@@ -163,7 +167,7 @@ int TPpContext::AddAtomFixed(const char *s, int atom)
{
TAtomMap::const_iterator it = atomMap.insert(std::pair<TString, int>(s, atom)).first;
if (stringMap.size() < (size_t)atom + 1)
stringMap.resize(atom + 100);
stringMap.resize(atom + 100, 0);
stringMap[atom] = &it->first;
return atom;

6
glslang/MachineIndependent/preprocessor/PpContext.cpp
View File

@@ -106,9 +106,9 @@ void TPpContext::setInput(TInputScanner& input, bool versionWillBeError)
StringInputSrc *in = (StringInputSrc *)malloc(sizeof(StringInputSrc));
memset(in, 0, sizeof(StringInputSrc));
in->input = &input;
in->base.scan = byte_scan;
in->base.getch = (int (*)(TPpContext*, InputSrc *, TPpToken *))str_getch;
in->base.ungetch = (void (*)(TPpContext*, InputSrc *, int, TPpToken *))str_ungetch;
in->base.scan = sourceScan;
in->base.getch = (int (*)(TPpContext*, InputSrc *, TPpToken *))sourceGetCh;
in->base.ungetch = (void (*)(TPpContext*, InputSrc *, int, TPpToken *))sourceUngetCh;
in->base.prev = currentInput;
currentInput = &in->base;
errorOnVersion = versionWillBeError;

45
glslang/MachineIndependent/preprocessor/PpContext.h
View File

@@ -84,7 +84,21 @@ namespace glslang {
class TPpToken {
public:
TPpToken() { loc.line = 0; loc.string = 0; name[0] = 0; }
TPpToken() : token(0), ival(0), dval(0.0), atom(0)
{
loc.line = 0;
loc.string = 0;
name[0] = 0;
}
bool operator==(const TPpToken& right)
{
return token == right.token && atom == right.atom &&
ival == right.ival && dval == right.dval &&
strcmp(name, right.name) == 0;
}
bool operator!=(const TPpToken& right) { return ! operator==(right); }
static const int maxTokenLength = 1024;
TSourceLoc loc;
@@ -110,6 +124,7 @@ public:
const char* tokenize(TPpToken* ppToken);
// TODO: preprocessor simplification: this should be a base class, not a set of function pointers
struct InputSrc {
struct InputSrc *prev;
int (*scan)(TPpContext*, struct InputSrc *, TPpToken *);
@@ -127,7 +142,6 @@ public:
struct TokenStream {
TokenStream *next;
char *name;
TokenBlock *head;
TokenBlock *current;
};
@@ -231,6 +245,7 @@ protected:
int CPPdefine(TPpToken * ppToken);
int CPPundef(TPpToken * ppToken);
int CPPelse(int matchelse, TPpToken * ppToken);
int extraTokenCheck(int atom, TPpToken* ppToken, int token);
int eval(int token, int prec, int *res, int *err, TPpToken * ppToken);
int CPPif (TPpToken * ppToken);
int CPPifdef(int defined, TPpToken * ppToken);
@@ -259,28 +274,26 @@ protected:
//
// From PpTokens.cpp
//
char* idstr(const char *fstr, MemoryPool *pool);
TPpContext::TokenBlock* lNewBlock(TokenStream *fTok, MemoryPool *pool);
void lAddByte(TokenStream *fTok, unsigned char fVal);
int lReadByte(TokenStream *pTok);
TokenStream *NewTokenStream(const char *name, MemoryPool *pool);
TokenStream *NewTokenStream(MemoryPool *pool);
void DeleteTokenStream(TokenStream *pTok);
void RecordToken(TokenStream *pTok, int token, TPpToken * ppToken);
void RecordToken(TokenStream* pTok, int token, TPpToken* ppToken);
void RewindTokenStream(TokenStream *pTok);
int ReadToken(TokenStream *pTok, TPpToken * ppToken);
int ReadToken(TokenStream* pTok, TPpToken* ppToken);
int ReadFromTokenStream(TokenStream *ts, int name, int (*final)(TPpContext *));
void UngetToken(int token, TPpToken * ppToken);
void DumpTokenStream(FILE *fp, TokenStream *s, TPpToken * ppToken);
void UngetToken(int token, TPpToken* ppToken);
struct TokenInputSrc {
InputSrc base;
TokenStream *tokens;
int (*final)(TPpContext *);
TokenStream *tokens;
int (*final)(TPpContext *);
};
static int scan_token(TPpContext*, TokenInputSrc *in, TPpToken * ppToken);
struct UngotToken {
InputSrc base;
int token;
TPpToken lval;
int token;
TPpToken lval;
};
static int reget_token(TPpContext *, UngotToken *t, TPpToken * ppToken);
@@ -292,12 +305,12 @@ protected:
TInputScanner* input;
};
int InitScanner(TPpContext *cpp);
static int str_getch(TPpContext*, StringInputSrc *in);
static void str_ungetch(TPpContext*, StringInputSrc *in, int ch, TPpToken *type);
static int sourceGetCh(TPpContext*, StringInputSrc *in);
static void sourceUngetCh(TPpContext*, StringInputSrc *in, int ch, TPpToken *type);
int ScanFromString(char *s);
int check_EOF(int token);
bool check_EOF(int token);
int lFloatConst(char *str, int len, int ch, TPpToken * ppToken);
static int byte_scan(TPpContext*, InputSrc *in, TPpToken * ppToken);
static int sourceScan(TPpContext*, InputSrc *in, TPpToken * ppToken);
//
// From PpAtom.cpp

48
glslang/MachineIndependent/preprocessor/PpScanner.cpp
View File

@@ -105,7 +105,7 @@ int TPpContext::InitScanner(TPpContext *cpp)
return 1;
}
int TPpContext::str_getch(TPpContext* pp, StringInputSrc *in)
int TPpContext::sourceGetCh(TPpContext* pp, StringInputSrc *in)
{
int ch = in->input->get();
@@ -115,7 +115,7 @@ int TPpContext::str_getch(TPpContext* pp, StringInputSrc *in)
return ch;
}
void TPpContext::str_ungetch(TPpContext* pp, StringInputSrc *in, int ch, TPpToken *type)
void TPpContext::sourceUngetCh(TPpContext* pp, StringInputSrc *in, int ch, TPpToken *type)
{
in->input->unget();
}
@@ -130,7 +130,7 @@ void TPpContext::str_ungetch(TPpContext* pp, StringInputSrc *in, int ch, TPpToke
* letter 'e', or a precision ending (e.g., F or LF).
*/
int TPpContext::lFloatConst(char *str, int len, int ch, TPpToken * ppToken)
int TPpContext::lFloatConst(char* str, int len, int ch, TPpToken* ppToken)
{
bool HasDecimalOrExponent = false;
int declen, exp, ExpSign;
@@ -241,13 +241,12 @@ int TPpContext::lFloatConst(char *str, int len, int ch, TPpToken * ppToken)
return CPP_DOUBLECONSTANT;
else
return CPP_FLOATCONSTANT;
} // lFloatConst
}
///////////////////////////////////////////////////////////////////////////////////////////////
///////////////////////////////////////// Normal Scanner //////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////////////////////
int TPpContext::byte_scan(TPpContext* pp, InputSrc *in, TPpToken * ppToken)
//
// Scanner used to tokenize source stream.
//
int TPpContext::sourceScan(TPpContext* pp, InputSrc*, TPpToken* ppToken)
{
char tokenText[TPpToken::maxTokenLength + 1];
int AlreadyComplained = 0;
@@ -268,8 +267,6 @@ int TPpContext::byte_scan(TPpContext* pp, InputSrc *in, TPpToken * ppToken)
switch (ch) {
default:
return ch; // Single character token
case EOF:
return EOF;
case 'A': case 'B': case 'C': case 'D': case 'E':
case 'F': case 'G': case 'H': case 'I': case 'J':
case 'K': case 'L': case 'M': case 'N': case 'O':
@@ -704,14 +701,17 @@ int TPpContext::byte_scan(TPpContext* pp, InputSrc *in, TPpToken * ppToken)
}
}
}
} // byte_scan
}
//
// Return string pointer to next token.
// Return 0 when no more tokens.
//
const char* TPpContext::tokenize(TPpToken* ppToken)
{
int token = '\n';
for(;;) {
const char* tokenString = 0;
token = currentInput->scan(this, currentInput, ppToken);
ppToken->token = token;
@@ -742,7 +742,13 @@ const char* TPpContext::tokenize(TPpToken* ppToken)
else if (token == CPP_INTCONSTANT || token == CPP_UINTCONSTANT ||
token == CPP_FLOATCONSTANT || token == CPP_DOUBLECONSTANT)
tokenString = ppToken->name;
else
else if (token == CPP_STRCONSTANT) {
parseContext.error(ppToken->loc, "string literals not supported", "\"\"", "");
tokenString = 0;
} else if (token == '\'') {
parseContext.error(ppToken->loc, "character literals not supported", "\'", "");
tokenString = 0;
} else
tokenString = GetAtomString(token);
if (tokenString) {
@@ -752,19 +758,19 @@ const char* TPpContext::tokenize(TPpToken* ppToken)
return tokenString;
}
}
return 0;
} // PpTokenize
}
//Checks if the token just read is EOF or not.
int TPpContext::check_EOF(int token)
bool TPpContext::check_EOF(int token)
{
if (token == EOF) {
if (ifdepth > 0)
parseContext.error(parseContext.getCurrentLoc(), "missing #endif", "#if", "");
return 1;
parseContext.error(parseContext.getCurrentLoc(), "missing #endif", "", "");
return true;
}
return 0;
return false;
}
} // end namespace glslang

234
glslang/MachineIndependent/preprocessor/PpTokens.cpp
View File

@@ -75,9 +75,11 @@ NVIDIA SOFTWARE, HOWEVER CAUSED AND WHETHER UNDER THEORY OF CONTRACT,
TORT (INCLUDING NEGLIGENCE), STRICT LIABILITY OR OTHERWISE, EVEN IF
NVIDIA HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
\****************************************************************************/
//
// tokens.c
// For recording and playing back the stream of tokens in a macro definition.
//
#ifdef _WIN32
#define _CRT_SECURE_NO_WARNINGS
#define snprintf sprintf_s
@@ -94,43 +96,6 @@ NVIDIA HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
namespace glslang {
///////////////////////////////////////////////////////////////////////////////////////////////
//////////////////////// Preprocessor and Token Recorder and Playback: ////////////////////////
///////////////////////////////////////////////////////////////////////////////////////////////
/*
* idstr()
* Copy a string to a malloc'ed block and convert it into something suitable
* for an ID
*
*/
char* TPpContext::idstr(const char *fstr, MemoryPool *pool)
{
size_t len;
char *str, *t;
const char *f;
len = strlen(fstr);
if (!pool)
str = (char *) malloc(len + 1);
else
str = (char *) mem_Alloc(pool, len + 1);
for (f=fstr, t=str; *f; f++) {
if (isalnum(*f)) *t++ = *f;
else if (*f == '.' || *f == '/') *t++ = '_';
}
*t = 0;
return str;
} // idstr
/*
* lNewBlock()
*
*/
TPpContext::TokenBlock* TPpContext::lNewBlock(TokenStream *fTok, MemoryPool *pool)
{
TokenBlock *lBlock;
@@ -152,12 +117,7 @@ TPpContext::TokenBlock* TPpContext::lNewBlock(TokenStream *fTok, MemoryPool *poo
fTok->current = lBlock;
return lBlock;
} // lNewBlock
/*
* lAddByte()
*
*/
}
void TPpContext::lAddByte(TokenStream *fTok, unsigned char fVal)
{
@@ -166,14 +126,11 @@ void TPpContext::lAddByte(TokenStream *fTok, unsigned char fVal)
if (lBlock->count >= lBlock->max)
lBlock = lNewBlock(fTok, 0);
lBlock->data[lBlock->count++] = fVal;
} // lAddByte
}
/*
* lReadByte() - Get the next byte from a stream.
*
* Get the next byte from a stream.
*/
int TPpContext::lReadByte(TokenStream *pTok)
{
TokenBlock *lBlock;
@@ -191,16 +148,12 @@ int TPpContext::lReadByte(TokenStream *pTok)
lval = lBlock->data[lBlock->current++];
}
return lval;
} // lReadByte
}
/////////////////////////////////////// Global Functions://////////////////////////////////////
/*
* NewTokenStream()
*
*/
TPpContext::TokenStream* TPpContext::NewTokenStream(const char *name, MemoryPool *pool)
//
// Make a token stream (used for reprocessing macros).
//
TPpContext::TokenStream* TPpContext::NewTokenStream(MemoryPool *pool)
{
TokenStream *pTok;
@@ -209,17 +162,11 @@ TPpContext::TokenStream* TPpContext::NewTokenStream(const char *name, MemoryPool
else
pTok = (TokenStream*)mem_Alloc(pool, sizeof(TokenStream));
pTok->next = NULL;
pTok->name = idstr(name, pool);
pTok->head = NULL;
pTok->current = NULL;
lNewBlock(pTok, pool);
return pTok;
} // NewTokenStream
/*
* DeleteTokenStream()
*
*/
}
void TPpContext::DeleteTokenStream(TokenStream *pTok)
{
@@ -232,18 +179,14 @@ void TPpContext::DeleteTokenStream(TokenStream *pTok)
free(pBlock);
pBlock = nBlock;
}
if (pTok->name)
free(pTok->name);
free(pTok);
}
} // DeleteTokenStream
}
/*
* RecordToken() - Add a token to the end of a list for later playback or printout.
*
* Add a token to the end of a list for later playback.
*/
void TPpContext::RecordToken(TokenStream *pTok, int token, TPpToken * ppToken)
void TPpContext::RecordToken(TokenStream *pTok, int token, TPpToken* ppToken)
{
const char *s;
char *str = NULL;
@@ -254,7 +197,6 @@ void TPpContext::RecordToken(TokenStream *pTok, int token, TPpToken * ppToken)
lAddByte(pTok, (unsigned char)(token & 0x7f));
switch (token) {
case CPP_IDENTIFIER:
case CPP_TYPEIDENTIFIER:
case CPP_STRCONSTANT:
s = GetAtomString(ppToken->atom);
while (*s)
@@ -274,33 +216,28 @@ void TPpContext::RecordToken(TokenStream *pTok, int token, TPpToken * ppToken)
break;
case '(':
lAddByte(pTok, (unsigned char)(ppToken->ival ? 1 : 0));
break;
default:
break;
}
} // RecordToken
}
/*
* RewindTokenStream() - Reset a token stream in preperation for reading.
*
* Reset a token stream in preperation for reading.
*/
void TPpContext::RewindTokenStream(TokenStream *pTok)
{
if (pTok->head) {
pTok->current = pTok->head;
pTok->current->current = 0;
}
} // RewindTokenStream
}
/*
* ReadToken() - Read the next token from a stream.
*
* Read the next token from a token stream (not the source stream, but stream used to hold a tokenized macro).
*/
int TPpContext::ReadToken(TokenStream *pTok, TPpToken *ppToken)
{
//TODO: preprocessor simplification: why is this different than byte_scan
char tokenText[TPpToken::maxTokenLength + 1];
int ltoken, len;
char ch;
@@ -311,89 +248,56 @@ int TPpContext::ReadToken(TokenStream *pTok, TPpToken *ppToken)
if (ltoken > 127)
ltoken += 128;
switch (ltoken) {
case CPP_IDENTIFIER:
case CPP_TYPEIDENTIFIER:
len = 0;
ch = lReadByte(pTok);
while ((ch >= 'a' && ch <= 'z') ||
(ch >= 'A' && ch <= 'Z') ||
(ch >= '0' && ch <= '9') ||
ch == '_')
{
if (len < TPpToken::maxTokenLength) {
tokenText[len] = ch;
len++;
ch = lReadByte(pTok);
} else {
parseContext.error(ppToken->loc,"name too long", "", "");
break;
}
}
tokenText[len] = '\0';
assert(ch == '\0');
ppToken->atom = LookUpAddString(tokenText);
return CPP_IDENTIFIER;
case '(':
ppToken->ival = lReadByte(pTok);
break;
case CPP_STRCONSTANT:
len = 0;
while ((ch = lReadByte(pTok)) != 0) {
if (len < TPpToken::maxTokenLength)
tokenText[len++] = ch;
else
break;
}
tokenText[len] = 0;
ppToken->atom = LookUpAddString(tokenText);
break;
case CPP_IDENTIFIER:
case CPP_FLOATCONSTANT:
case CPP_DOUBLECONSTANT:
len = 0;
ch = lReadByte(pTok);
while ((ch >= '0' && ch <= '9') || ch=='e' || ch=='E' || ch=='.' || ch=='+' || ch=='-' || ch=='l' || ch=='L' || ch=='f'|| ch=='F')
{
if (len < TPpToken::maxTokenLength) {
tokenText[len] = ch;
len++;
ch = lReadByte(pTok);
} else {
parseContext.error(ppToken->loc,"float literal too long", "", "");
break;
}
}
tokenText[len] = '\0';
assert(ch == '\0');
strcpy(ppToken->name, tokenText);
ppToken->dval = atof(ppToken->name);
break;
case CPP_INTCONSTANT:
case CPP_UINTCONSTANT:
len = 0;
ch = lReadByte(pTok);
while ((ch >= '0' && ch <= '9') || ch == 'u' || ch == 'U')
{
while (ch != 0) {
if (len < TPpToken::maxTokenLength) {
tokenText[len] = ch;
len++;
ch = lReadByte(pTok);
} else {
parseContext.error(ppToken->loc,"integer literal too long", "", "");
parseContext.error(ppToken->loc, "token too long", "", "");
break;
}
}
tokenText[len] = '\0';
assert(ch == '\0');
strcpy(ppToken->name,tokenText);
ppToken->ival = atoi(ppToken->name);
break;
case '(':
ppToken->ival = lReadByte(pTok);
break;
tokenText[len] = 0;
switch (ltoken) {
case CPP_IDENTIFIER:
case CPP_STRCONSTANT:
ppToken->atom = LookUpAddString(tokenText);
break;
case CPP_FLOATCONSTANT:
case CPP_DOUBLECONSTANT:
strcpy(ppToken->name, tokenText);
ppToken->dval = atof(ppToken->name);
break;
case CPP_INTCONSTANT:
case CPP_UINTCONSTANT:
strcpy(ppToken->name, tokenText);
if (len > 0 && tokenText[0] == '0') {
if (len > 1 && tokenText[1] == 'x' || tokenText[1] == 'X')
ppToken->ival = strtol(ppToken->name, 0, 16);
else
ppToken->ival = strtol(ppToken->name, 0, 8);
} else
ppToken->ival = atoi(ppToken->name);
break;
}
}
return ltoken;
}
return EOF;
} // ReadToken
}
int TPpContext::scan_token(TPpContext* pp, TokenInputSrc *in, TPpToken * ppToken)
{
@@ -434,9 +338,7 @@ int TPpContext::reget_token(TPpContext* pp, UngotToken *t, TPpToken * ppToken)
return token;
}
typedef int (*scanFnPtr_t);
void TPpContext::UngetToken(int token, TPpToken * ppToken)
void TPpContext::UngetToken(int token, TPpToken* ppToken)
{
UngotToken *t = (UngotToken *) malloc(sizeof(UngotToken));
memset(t, 0, sizeof(UngotToken));
@@ -447,38 +349,4 @@ void TPpContext::UngetToken(int token, TPpToken * ppToken)
currentInput = &t->base;
}
void TPpContext::DumpTokenStream(FILE *fp, TokenStream *s, TPpToken * ppToken)
{
int token;
if (fp == 0) fp = stdout;
RewindTokenStream(s);
while ((token = ReadToken(s, ppToken)) > 0) {
switch (token) {
case CPP_IDENTIFIER:
case CPP_TYPEIDENTIFIER:
printf("%s ", GetAtomString(ppToken->atom));
break;
case CPP_STRCONSTANT:
printf("\"%s\"", GetAtomString(ppToken->atom));
break;
case CPP_FLOATCONSTANT:
case CPP_DOUBLECONSTANT:
printf("%g9.6 ", ppToken->dval);
break;
case CPP_INTCONSTANT:
case CPP_UINTCONSTANT:
printf("%d ", ppToken->ival);
break;
default:
if (token >= 127)
printf("%s ", GetAtomString(token));
else
printf("%c", token);
break;
}
}
}
} // end namespace glslang

1
glslang/MachineIndependent/preprocessor/PpTokens.h
View File

@@ -99,7 +99,6 @@ NVIDIA HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#define CPP_OR_OP 276
#define CPP_INC_OP 277
#define CPP_STRCONSTANT 278
#define CPP_TYPEIDENTIFIER 279
#define CPP_RIGHT_ASSIGN 280
#define CPP_LEFT_ASSIGN 281
#define CPP_AND_ASSIGN 282