mewin/glslang
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Fix memory corruption problem in the preprocessor, removing custom hash-tables/etc. and replacing with std containers.

Browse Source 
git-svn-id: https://cvs.khronos.org/svn/repos/ogl/trunk/ecosystem/public/sdk/tools/glslang@23623 e7fa87d3-cd2b-0410-9028-fcbf551c1848
This commit is contained in:
John Kessenich 2013-10-20 18:18:36 +00:00
parent ab3080353a
commit 1f4104fbb1
7 changed files with 172 additions and 957 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

145
glslang/MachineIndependent/preprocessor/Pp.cpp
View File

@ -98,67 +98,59 @@ namespace glslang {
int TPpContext::InitCPP()
{
TPpContext::AtomTable* atable = &atomTable;
// Add various atoms needed by the CPP line scanner:
bindAtom = LookUpAddString(atable, "bind");
constAtom = LookUpAddString(atable, "const");
defaultAtom = LookUpAddString(atable, "default");
defineAtom = LookUpAddString(atable, "define");
definedAtom = LookUpAddString(atable, "defined");
elifAtom = LookUpAddString(atable, "elif");
elseAtom = LookUpAddString(atable, "else");
endifAtom = LookUpAddString(atable, "endif");
ifAtom = LookUpAddString(atable, "if");
ifdefAtom = LookUpAddString(atable, "ifdef");
ifndefAtom = LookUpAddString(atable, "ifndef");
includeAtom = LookUpAddString(atable, "include");
lineAtom = LookUpAddString(atable, "line");
pragmaAtom = LookUpAddString(atable, "pragma");
texunitAtom = LookUpAddString(atable, "texunit");
undefAtom = LookUpAddString(atable, "undef");
errorAtom = LookUpAddString(atable, "error");
__LINE__Atom = LookUpAddString(atable, "__LINE__");
__FILE__Atom = LookUpAddString(atable, "__FILE__");
__VERSION__Atom = LookUpAddString(atable, "__VERSION__");
versionAtom = LookUpAddString(atable, "version");
coreAtom = LookUpAddString(atable, "core");
compatibilityAtom = LookUpAddString(atable, "compatibility");
esAtom = LookUpAddString(atable, "es");
extensionAtom = LookUpAddString(atable, "extension");
macros = NewScopeInPool(mem_CreatePool(0, 0));
return 1;
} // InitCPP
int TPpContext::FreeCPP()
{
if (macros) {
mem_FreePool(macros->pool);
macros = 0;
}
bindAtom = LookUpAddString("bind");
constAtom = LookUpAddString("const");
defaultAtom = LookUpAddString("default");
defineAtom = LookUpAddString("define");
definedAtom = LookUpAddString("defined");
elifAtom = LookUpAddString("elif");
elseAtom = LookUpAddString("else");
endifAtom = LookUpAddString("endif");
ifAtom = LookUpAddString("if");
ifdefAtom = LookUpAddString("ifdef");
ifndefAtom = LookUpAddString("ifndef");
includeAtom = LookUpAddString("include");
lineAtom = LookUpAddString("line");
pragmaAtom = LookUpAddString("pragma");
texunitAtom = LookUpAddString("texunit");
undefAtom = LookUpAddString("undef");
errorAtom = LookUpAddString("error");
__LINE__Atom = LookUpAddString("__LINE__");
__FILE__Atom = LookUpAddString("__FILE__");
__VERSION__Atom = LookUpAddString("__VERSION__");
versionAtom = LookUpAddString("version");
coreAtom = LookUpAddString("core");
compatibilityAtom = LookUpAddString("compatibility");
esAtom = LookUpAddString("es");
extensionAtom = LookUpAddString("extension");
pool = mem_CreatePool(0, 0);
return 1;
}
int TPpContext::FinalCPP()
{
mem_FreePool(pool);
if (ifdepth)
parseContext.error(parseContext.currentLoc, "missing #endif", "#if", "");
return 1;
}
int TPpContext::CPPdefine(TPpToken * yylvalpp)
{
int token, name, args[maxMacroArgs], argc;
int token, atom, args[maxMacroArgs], argc;
MacroSymbol mac;
Symbol *symb;
memset(&mac, 0, sizeof(mac));
token = currentInput->scan(this, currentInput, yylvalpp);
if (token != CPP_IDENTIFIER) {
parseContext.error(yylvalpp->loc, "must be followed by macro name", "#define", "");
parseContext.error(yylvalpp->loc, "must be followed by macro atom", "#define", "");
return token;
}
name = yylvalpp->atom;
atom = yylvalpp->atom;
token = currentInput->scan(this, currentInput, yylvalpp);
if (token == '(' && !yylvalpp->ival) {
// gather arguments
@ -182,11 +174,11 @@ int TPpContext::CPPdefine(TPpToken * yylvalpp)
return token;
}
mac.argc = argc;
mac.args = (int*)mem_Alloc(macros->pool, argc * sizeof(int));
mac.args = (int*)mem_Alloc(pool, argc * sizeof(int));
memcpy(mac.args, args, argc * sizeof(int));
token = currentInput->scan(this, currentInput, yylvalpp);
}
mac.body = NewTokenStream(GetAtomString(&atomTable, name), macros->pool);
mac.body = NewTokenStream(GetAtomString(atom), pool);
while (token != '\n') {
while (token == '\\') {
token = currentInput->scan(this, currentInput, yylvalpp);
@ -199,34 +191,33 @@ int TPpContext::CPPdefine(TPpToken * yylvalpp)
token = currentInput->scan(this, currentInput, yylvalpp);
};
symb = LookUpSymbol(macros, name);
symb = LookUpSymbol(atom);
if (symb) {
if (!symb->details.mac.undef) {
if (!symb->mac.undef) {
// already defined -- need to make sure they are identical
if (symb->details.mac.argc != mac.argc)
if (symb->mac.argc != mac.argc)
goto error;
for (argc=0; argc < mac.argc; argc++)
if (symb->details.mac.args[argc] != mac.args[argc])
if (symb->mac.args[argc] != mac.args[argc])
goto error;
RewindTokenStream(symb->details.mac.body);
RewindTokenStream(symb->mac.body);
RewindTokenStream(mac.body);
do {
int old_lval, old_token;
old_token = ReadToken(symb->details.mac.body, yylvalpp);
old_token = ReadToken(symb->mac.body, yylvalpp);
old_lval = yylvalpp->ival;
token = ReadToken(mac.body, yylvalpp);
if (token != old_token || yylvalpp->ival != old_lval) {
error:
parseContext.error(yylvalpp->loc, "Macro Redefined", "#define", GetStringOfAtom(&atomTable, name));
parseContext.error(yylvalpp->loc, "Macro Redefined", "#define", GetAtomString(atom));
break;
}
} while (token > 0);
}
//FreeMacro(&symb->details.mac);
} else {
symb = AddSymbol(&yylvalpp->loc, macros, name, MACRO_S);
symb = AddSymbol(atom);
}
symb->details.mac = mac;
symb->mac = mac;
return '\n';
} // CPPdefine
@ -246,9 +237,9 @@ int TPpContext::CPPundef(TPpToken * yylvalpp)
return token;
}
symb = LookUpSymbol(macros, yylvalpp->atom);
symb = LookUpSymbol(yylvalpp->atom);
if (symb) {
symb->details.mac.undef = 1;
symb->mac.undef = 1;
}
token = currentInput->scan(this, currentInput, yylvalpp);
if (token != '\n')
@ -411,8 +402,8 @@ int TPpContext::eval(int token, int prec, int *res, int *err, TPpToken * yylvalp
return token;
}
*res = (s = LookUpSymbol(macros, yylvalpp->atom))
? !s->details.mac.undef : 0;
*res = (s = LookUpSymbol(yylvalpp->atom))
? !s->mac.undef : 0;
token = currentInput->scan(this, currentInput, yylvalpp);
if (needclose) {
if (token != ')') {
@ -539,14 +530,14 @@ int TPpContext::CPPifdef(int defined, TPpToken * yylvalpp)
else
parseContext.error(yylvalpp->loc, "must be followed by macro name", "#ifndef", "");
} else {
Symbol *s = LookUpSymbol(macros, name);
Symbol *s = LookUpSymbol(name);
token = currentInput->scan(this, currentInput, yylvalpp);
if (token != '\n') {
parseContext.error(yylvalpp->loc, "unexpected tokens following #ifdef directive - expected a newline", "#ifdef", "");
while (token != '\n')
token = currentInput->scan(this, currentInput, yylvalpp);
}
if (((s && !s->details.mac.undef) ? 1 : 0) != defined)
if (((s && !s->mac.undef) ? 1 : 0) != defined)
token = CPPelse(1, yylvalpp);
}
return token;
@ -592,9 +583,9 @@ int TPpContext::CPPerror(TPpToken * yylvalpp)
token == CPP_FLOATCONSTANT || token == CPP_DOUBLECONSTANT) {
message.append(yylvalpp->name);
} else if (token == CPP_IDENTIFIER || token == CPP_STRCONSTANT) {
message.append(GetStringOfAtom(&atomTable, yylvalpp->atom));
message.append(GetAtomString(yylvalpp->atom));
} else {
message.append(GetStringOfAtom(&atomTable, token));
message.append(GetAtomString(token));
}
message.append(" ");
token = currentInput->scan(this, currentInput, yylvalpp);
@ -630,7 +621,7 @@ int TPpContext::CPPpragma(TPpToken * yylvalpp)
}
switch (token) {
case CPP_IDENTIFIER:
SrcStr = GetAtomString(&atomTable, yylvalpp->atom);
SrcStr = GetAtomString(yylvalpp->atom);
allTokens[tokenCount] = (char*)malloc(strlen(SrcStr) + 1);
strcpy(allTokens[tokenCount++], SrcStr);
break;
@ -720,7 +711,7 @@ int TPpContext::CPPextension(TPpToken * yylvalpp)
if (token != CPP_IDENTIFIER)
parseContext.error(yylvalpp->loc, "extension name expected", "#extension", "");
strcpy(extensionName, GetAtomString(&atomTable, yylvalpp->atom));
strcpy(extensionName, GetAtomString(yylvalpp->atom));
token = currentInput->scan(this, currentInput, yylvalpp);
if (token != ':') {
@ -734,7 +725,7 @@ int TPpContext::CPPextension(TPpToken * yylvalpp)
return token;
}
parseContext.updateExtensionBehavior(extensionName, GetAtomString(&atomTable, yylvalpp->atom));
parseContext.updateExtensionBehavior(extensionName, GetAtomString(yylvalpp->atom));
token = currentInput->scan(this, currentInput, yylvalpp);
if (token == '\n')
@ -807,7 +798,7 @@ int TPpContext::readCPPline(TPpToken * yylvalpp)
} else if (yylvalpp->atom == extensionAtom) {
token = CPPextension(yylvalpp);
} else {
parseContext.error(yylvalpp->loc, "Invalid Directive", "#", GetStringOfAtom(&atomTable, yylvalpp->atom));
parseContext.error(yylvalpp->loc, "Invalid Directive", "#", GetAtomString(yylvalpp->atom));
}
}
while (token != '\n' && token != 0 && token != EOF) {
@ -853,7 +844,7 @@ TPpContext::TokenStream* TPpContext::PrescanMacroArg(TokenStream *a, TPpToken *
RewindTokenStream(a);
do {
token = ReadToken(a, yylvalpp);
if (token == CPP_IDENTIFIER && LookUpSymbol(macros, yylvalpp->atom))
if (token == CPP_IDENTIFIER && LookUpSymbol(yylvalpp->atom))
break;
} while (token > 0);
if (token <= 0) return a;
@ -934,7 +925,7 @@ int TPpContext::zero_scan(TPpContext* pp, InputSrc *inInput, TPpToken* yylvalpp)
*/
int TPpContext::MacroExpand(int atom, TPpToken* yylvalpp, int expandUndef)
{
Symbol *sym = LookUpSymbol(macros, atom);
Symbol *sym = LookUpSymbol(atom);
MacroInputSrc *in;
int i, j, token;
int depth = 0;
@ -964,11 +955,11 @@ int TPpContext::MacroExpand(int atom, TPpToken* yylvalpp, int expandUndef)
}
// no recursive expansions
if (sym && sym->details.mac.busy)
if (sym && sym->mac.busy)
return 0;
// not expanding of undefined symbols
if ((! sym || sym->details.mac.undef) && ! expandUndef)
if ((! sym || sym->mac.undef) && ! expandUndef)
return 0;
in = (MacroInputSrc*)malloc(sizeof(*in));
@ -976,7 +967,7 @@ int TPpContext::MacroExpand(int atom, TPpToken* yylvalpp, int expandUndef)
in->base.line = currentInput->line;
in->base.name = currentInput->name;
if ((! sym || sym->details.mac.undef) && expandUndef) {
if ((! sym || sym->mac.undef) && expandUndef) {
// push input
in->base.scan = zero_scan;
in->base.prev = currentInput;
@ -986,8 +977,8 @@ int TPpContext::MacroExpand(int atom, TPpToken* yylvalpp, int expandUndef)
}
in->base.scan = macro_scan;
in->mac = &sym->details.mac;
if (sym->details.mac.args) {
in->mac = &sym->mac;
if (sym->mac.args) {
token = currentInput->scan(this, currentInput, yylvalpp);
if (token != '(') {
UngetToken(token, yylvalpp);
@ -1005,7 +996,7 @@ int TPpContext::MacroExpand(int atom, TPpToken* yylvalpp, int expandUndef)
while (1) {
token = currentInput->scan(this, currentInput, yylvalpp);
if (token <= 0) {
parseContext.error(yylvalpp->loc, "EOF in macro", "preprocessor", GetStringOfAtom(&atomTable, atom));
parseContext.error(yylvalpp->loc, "EOF in macro", "preprocessor", GetAtomString(atom));
return 1;
}
@ -1026,7 +1017,7 @@ int TPpContext::MacroExpand(int atom, TPpToken* yylvalpp, int expandUndef)
} while (i < in->mac->argc);
if (i < in->mac->argc)
parseContext.error(yylvalpp->loc, "Too few args in Macro", "preprocessor", GetStringOfAtom(&atomTable, atom));
parseContext.error(yylvalpp->loc, "Too few args in Macro", "preprocessor", GetAtomString(atom));
else if (token != ')') {
depth=0;
while (token >= 0 && (depth > 0 || token != ')')) {
@ -1038,11 +1029,11 @@ int TPpContext::MacroExpand(int atom, TPpToken* yylvalpp, int expandUndef)
}
if (token <= 0) {
parseContext.error(yylvalpp->loc, "EOF in macro", "preprocessor", GetStringOfAtom(&atomTable, atom));
parseContext.error(yylvalpp->loc, "EOF in macro", "preprocessor", GetAtomString(atom));
return 1;
}
parseContext.error(yylvalpp->loc, "Too many args in Macro", "preprocessor", GetStringOfAtom(&atomTable, atom));
parseContext.error(yylvalpp->loc, "Too many args in Macro", "preprocessor", GetAtomString(atom));
}
for (i = 0; i<in->mac->argc; i++) {
in->args[i] = PrescanMacroArg(in->args[i], yylvalpp);
@ -1059,8 +1050,8 @@ int TPpContext::MacroExpand(int atom, TPpToken* yylvalpp, int expandUndef)
#endif
/*retain the input source*/
in->base.prev = currentInput;
sym->details.mac.busy = 1;
RewindTokenStream(sym->details.mac.body);
sym->mac.busy = 1;
RewindTokenStream(sym->mac.body);
currentInput = &in->base;
return 1;

580
glslang/MachineIndependent/preprocessor/PpAtom.cpp
View File

@ -98,10 +98,6 @@ namespace {
using namespace glslang;
///////////////////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////// String table: //////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////////////////////
const struct {
int val;
const char *str;
@ -129,579 +125,71 @@ const struct {
{ CPP_INC_OP, "++" },
};
///////////////////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////// String table: //////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////////////////////
#define INIT_STRING_TABLE_SIZE 16384
/*
* InitStringTable() - Initialize the string table.
*
*/
int InitStringTable(TPpContext::StringTable *stable)
{
stable->strings = (char *) malloc(INIT_STRING_TABLE_SIZE);
if (!stable->strings)
return 0;
// Zero-th offset means "empty" so don't use it.
stable->nextFree = 1;
stable->size = INIT_STRING_TABLE_SIZE;
return 1;
} // InitStringTable
/*
* FreeStringTable() - Free the string table.
*
*/
void FreeStringTable(TPpContext::StringTable *stable)
{
if (stable->strings)
free(stable->strings);
stable->strings = NULL;
stable->nextFree = 0;
stable->size = 0;
} // FreeStringTable
/*
* HashString() - Hash a string with the base hash function.
*
*/
int HashString(const char *s)
{
int hval = 0;
while (*s) {
hval = (hval*13507 + *s*197) ^ (hval >> 2);
s++;
}
return hval & 0x7fffffff;
} // HashString
/*
* HashString2() - Hash a string with the incrimenting hash function.
*
*/
int HashString2(const char *s)
{
int hval = 0;
while (*s) {
hval = (hval*729 + *s*37) ^ (hval >> 1);
s++;
}
return hval;
} // HashString2
/*
* AddString() - Add a string to a string table. Return it's offset.
*
*/
int AddString(TPpContext::StringTable *stable, const char *s)
{
int len, loc;
char *str;
len = (int) strlen(s);
if (stable->nextFree + len + 1 >= stable->size) {
assert(stable->size < 1000000);
str = (char *) malloc(stable->size*2);
memcpy(str, stable->strings, stable->size);
free(stable->strings);
stable->strings = str;
}
loc = stable->nextFree;
strcpy(&stable->strings[loc], s);
stable->nextFree += len + 1;
return loc;
} // AddString
///////////////////////////////////////////////////////////////////////////////////////////////
/////////////////////////////////////////// Hash table: ///////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////////////////////
#define INIT_HASH_TABLE_SIZE 2047
/*
* InitHashTable() - Initialize the hash table.
*
*/
int InitHashTable(TPpContext::HashTable *htable, int fsize)
{
int ii;
htable->entry = (TPpContext::HashEntry *) malloc(sizeof(TPpContext::HashEntry)*fsize);
if (! htable->entry)
return 0;
htable->size = fsize;
for (ii = 0; ii < fsize; ii++) {
htable->entry[ii].index = 0;
htable->entry[ii].value = 0;
}
htable->entries = 0;
for (ii = 0; ii <= TPpContext::hashTableMaxCollisions; ii++)
htable->counts[ii] = 0;
return 1;
} // InitHashTable
/*
* FreeHashTable() - Free the hash table.
*
*/
void FreeHashTable(TPpContext::HashTable *htable)
{
if (htable->entry)
free(htable->entry);
htable->entry = NULL;
htable->size = 0;
htable->entries = 0;
} // FreeHashTable
/*
* Empty() - See if a hash table entry is empty.
*
*/
int Empty(TPpContext::HashTable *htable, int hashloc)
{
assert(hashloc >= 0 && hashloc < htable->size);
if (htable->entry[hashloc].index == 0) {
return 1;
} else {
return 0;
}
} // Empty
/*
* Match() - See if a hash table entry is matches a string.
*
*/
int Match(TPpContext::HashTable *htable, TPpContext::StringTable *stable, const char *s, int hashloc)
{
int strloc;
strloc = htable->entry[hashloc].index;
if (!strcmp(s, &stable->strings[strloc])) {
return 1;
} else {
return 0;
}
} // Match
///////////////////////////////////////////////////////////////////////////////////////////////
/////////////////////////////////////////// Atom table: ///////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////////////////////
#define INIT_ATOM_TABLE_SIZE 1024
/*
* GrowAtomTable() - Grow the atom table to at least "size" if it's smaller.
*
*/
int GrowAtomTable(TPpContext::AtomTable *atable, int size)
{
int *newmap, *newrev;
if (atable->size < size) {
if (atable->amap) {
newmap = (int*)realloc(atable->amap, sizeof(int)*size);
newrev = (int*)realloc(atable->arev, sizeof(int)*size);
} else {
newmap = (int*)malloc(sizeof(int)*size);
newrev = (int*)malloc(sizeof(int)*size);
atable->size = 0;
}
if (!newmap || !newrev) {
/* failed to grow -- error */
if (newmap)
atable->amap = newmap;
if (newrev)
atable->amap = newrev;
return -1;
}
memset(&newmap[atable->size], 0, (size - atable->size) * sizeof(int));
memset(&newrev[atable->size], 0, (size - atable->size) * sizeof(int));
atable->amap = newmap;
atable->arev = newrev;
atable->size = size;
}
return 0;
} // GrowAtomTable
/*
* lReverse() - Reverse the bottom 20 bits of a 32 bit int.
*
*/
int lReverse(int fval)
{
unsigned int in = fval;
int result = 0, cnt = 0;
while(in) {
result <<= 1;
result |= in&1;
in >>= 1;
cnt++;
}
// Don't use all 31 bits. One million atoms is plenty and sometimes the
// upper bits are used for other things.
if (cnt < 20)
result <<= 20 - cnt;
return result;
} // lReverse
/*
* AllocateAtom() - Allocate a new atom. Associated with the "undefined" value of -1.
*
*/
int AllocateAtom(TPpContext::AtomTable *atable)
{
if (atable->nextFree >= atable->size)
GrowAtomTable(atable, atable->nextFree*2);
atable->amap[atable->nextFree] = -1;
atable->arev[atable->nextFree] = lReverse(atable->nextFree);
atable->nextFree++;
return atable->nextFree - 1;
} // AllocateAtom
/*
* SetAtomValue() - Allocate a new atom associated with "hashindex".
*
*/
void SetAtomValue(TPpContext::AtomTable *atable, int atomnumber, int hashindex)
{
atable->amap[atomnumber] = atable->htable.entry[hashindex].index;
atable->htable.entry[hashindex].value = atomnumber;
} // SetAtomValue
/*
* FindHashLoc() - Find the hash location for this string. Return -1 it hash table is full.
*
*/
int FindHashLoc(TPpContext::AtomTable *atable, const char *s)
{
int hashloc, hashdelta, count;
int FoundEmptySlot = 0;
#ifdef DUMP_TABLE
int collision[TPpContext::hashTableMaxCollisions + 1];
#endif
hashloc = HashString(s) % atable->htable.size;
if (!Empty(&atable->htable, hashloc)) {
if (Match(&atable->htable, &atable->stable, s, hashloc))
return hashloc;
#ifdef DUMP_TABLE
collision[0] = hashloc;
#endif
hashdelta = HashString2(s);
count = 0;
while (count < TPpContext::hashTableMaxCollisions) {
hashloc = ((hashloc + hashdelta) & 0x7fffffff) % atable->htable.size;
if (!Empty(&atable->htable, hashloc)) {
if (Match(&atable->htable, &atable->stable, s, hashloc)) {
return hashloc;
}
} else {
FoundEmptySlot = 1;
break;
}
count++;
#ifdef DUMP_TABLE
collision[count] = hashloc;
#endif
}
if (! FoundEmptySlot) {
#ifdef DUMP_TABLE
{
int ii;
char str[200];
printf(str, "*** Hash failed with more than %d collisions. Must increase hash table size. ***",
hashTableMaxCollisions);
printf(str, "*** New string \"%s\", hash=%04x, delta=%04x", s, collision[0], hashdelta);
for (ii = 0; ii <= hashTableMaxCollisions; ii++) {
printf(str, "*** Collides on try %d at hash entry %04x with \"%s\"",
ii + 1, collision[ii], GetAtomString(atable, atable->htable.entry[collision[ii]].value));
}
}
#endif
return -1;
} else {
atable->htable.counts[count]++;
}
}
return hashloc;
} // FindHashLoc
} // end anonymous namespace
namespace glslang {
/*
* IncreaseHashTableSize()
*
*/
int TPpContext::IncreaseHashTableSize(AtomTable *atable)
//
// Map a new or existing string to an atom, inventing a new atom if necessary.
//
int TPpContext::LookUpAddString(const char *s)
{
int ii, strloc, oldhashloc, value, size;
AtomTable oldtable;
char *s;
// Save the old atom table and create a new one:
oldtable = *atable;
size = oldtable.htable.size*2 + 1;
if (! InitAtomTable(atable, size))
return 0;
// Add all the existing values to the new atom table preserving their atom values:
for (ii = atable->nextFree; ii < oldtable.nextFree; ii++) {
strloc = oldtable.amap[ii];
s = &oldtable.stable.strings[strloc];
oldhashloc = FindHashLoc(&oldtable, s);
assert(oldhashloc >= 0);
value = oldtable.htable.entry[oldhashloc].value;
AddAtomFixed(atable, s, value);
TAtomMap::const_iterator it = atomMap.find(s);
if (it == atomMap.end())
return AddAtomFixed(s, nextAtom++);
else
return it->second;
}
FreeAtomTable(&oldtable);
return 1;
} // IncreaseHashTableSize
/*
* LookUpAddStringHash() - Lookup a string in the hash table. If it's not there, add it and
* initialize the atom value in the hash table to 0. Return the hash table index.
*/
int TPpContext::LookUpAddStringHash(AtomTable *atable, const char *s)
//
// Map an already created atom to its string.
//
const char *TPpContext::GetAtomString(int atom)
{
int hashloc, strloc;
while(1) {
hashloc = FindHashLoc(atable, s);
if (hashloc >= 0)
break;
IncreaseHashTableSize(atable);
}
if (Empty(&atable->htable, hashloc)) {
atable->htable.entries++;
strloc = AddString(&atable->stable, s);
atable->htable.entry[hashloc].index = strloc;
atable->htable.entry[hashloc].value = 0;
}
return hashloc;
} // LookUpAddStringHash
/*
* LookUpAddString() - Lookup a string in the hash table. If it's not there, add it and
* initialize the atom value in the hash table to the next atom number.
* Return the atom value of string.
*/
int TPpContext::LookUpAddString(AtomTable *atable, const char *s)
{
int hashindex, atom;
hashindex = LookUpAddStringHash(atable, s);
atom = atable->htable.entry[hashindex].value;
if (atom == 0) {
atom = AllocateAtom(atable);
SetAtomValue(atable, atom, hashindex);
}
return atom;
} // LookUpAddString
/*
* GetAtomString()
*
*/
const char *TPpContext::GetAtomString(AtomTable *atable, int atom)
{
int soffset;
if (atom > 0 && atom < atable->nextFree) {
soffset = atable->amap[atom];
if (soffset > 0 && soffset < atable->stable.nextFree) {
return &atable->stable.strings[soffset];
} else {
return "<internal error: bad soffset>";
}
} else {
if (atom == 0) {
if (atom == 0)
return "<null atom>";
} else {
if (atom == EOF) {
if (atom < 0)
return "<EOF>";
} else {
if ((size_t)atom < stringMap.size())
return stringMap[atom]->c_str();
return "<invalid atom>";
}
}
}
} // GetAtomString
/*
* GetReversedAtom()
*
*/
int TPpContext::GetReversedAtom(TPpContext::AtomTable *atable, int atom)
//
// Add forced mapping of string to atom.
//
int TPpContext::AddAtomFixed(const char *s, int atom)
{
if (atom > 0 && atom < atable->nextFree)
return atable->arev[atom];
else
return 0;
} // GetReversedAtom
TAtomMap::const_iterator it = atomMap.insert(std::pair<TString, int>(s, atom)).first;
if (stringMap.size() < (size_t)atom + 1)
stringMap.resize(atom + 100);
stringMap[atom] = &it->first;
/*
* AddAtom() - Add a string to the atom, hash and string tables if it isn't already there.
* Return it's atom index.
*/
int TPpContext::AddAtom(TPpContext::AtomTable *atable, const char *s)
{
int atom = LookUpAddString(atable, s);
return atom;
} // AddAtom
/*
* AddAtomFixed() - Add an atom to the hash and string tables if it isn't already there.
* Assign it the atom value of "atom".
*/
int TPpContext::AddAtomFixed(AtomTable *atable, const char *s, int atom)
{
int hashindex, lsize;
hashindex = LookUpAddStringHash(atable, s);
if (atable->nextFree >= atable->size || atom >= atable->size) {
lsize = atable->size*2;
if (lsize <= atom)
lsize = atom + 1;
GrowAtomTable(atable, lsize);
}
atable->amap[atom] = atable->htable.entry[hashindex].index;
atable->htable.entry[hashindex].value = atom;
//if (atom >= atable->nextFree)
// atable->nextFree = atom + 1;
while (atom >= atable->nextFree) {
atable->arev[atable->nextFree] = lReverse(atable->nextFree);
atable->nextFree++;
}
return atom;
} // AddAtomFixed
/*
* InitAtomTable() - Initialize the atom table.
*
*/
int TPpContext::InitAtomTable(AtomTable *atable, int htsize)
//
// Initialize the atom table.
//
void TPpContext::InitAtomTable()
{
int ii;
htsize = htsize <= 0 ? INIT_HASH_TABLE_SIZE : htsize;
if (! InitStringTable(&atable->stable))
return 0;
if (! InitHashTable(&atable->htable, htsize))
return 0;
atable->nextFree = 0;
atable->amap = NULL;
atable->size = 0;
atable->arev = 0;
GrowAtomTable(atable, INIT_ATOM_TABLE_SIZE);
if (!atable->amap)
return 0;
// Initialize lower part of atom table to "<undefined>" atom:
AddAtomFixed(atable, "<undefined>", 0);
for (ii = 0; ii < CPP_FIRST_USER_TOKEN_SY; ii++)
atable->amap[ii] = atable->amap[0];
// Add single character tokens to the atom table:
{
const char *s = "~!%^&*()-+=|,.<>/?;:[]{}#";
char t[2];
t[1] = '\0';
while (*s) {
t[0] = *s;
AddAtomFixed(atable, t, s[0]);
AddAtomFixed(t, s[0]);
s++;
}
}
// Add multiple character scanner tokens :
for (int ii = 0; ii < sizeof(tokens)/sizeof(tokens[0]); ii++)
AddAtomFixed(tokens[ii].str, tokens[ii].val);
for (ii = 0; ii < sizeof(tokens)/sizeof(tokens[0]); ii++)
AddAtomFixed(atable, tokens[ii].str, tokens[ii].val);
AddAtom(atable, "<*** end fixed atoms ***>");
return 1;
} // InitAtomTable
///////////////////////////////////////////////////////////////////////////////////////////////
////////////////////////////////// Debug Printing Functions: //////////////////////////////////
///////////////////////////////////////////////////////////////////////////////////////////////
/*
* PrintAtomTable()
*
*/
void TPpContext::PrintAtomTable(AtomTable *atable)
{
int ii;
char str[200];
for (ii = 0; ii < atable->nextFree; ii++) {
printf(str, "%d: \"%s\"", ii, &atable->stable.strings[atable->amap[ii]]);
nextAtom = CPP_FIRST_USER_TOKEN_SY;
}
printf(str, "Hash table: size=%d, entries=%d, collisions=",
atable->htable.size, atable->htable.entries);
for (ii = 0; ii < hashTableMaxCollisions; ii++) {
printf(str, " %d", atable->htable.counts[ii]);
}
} // PrintAtomTable
/*
* GetStringOfAtom()
*
*/
char* TPpContext::GetStringOfAtom(AtomTable *atable, int atom)
{
char* chr_str;
chr_str=&atable->stable.strings[atable->amap[atom]];
return chr_str;
} // GetStringOfAtom
/*
* FreeAtomTable() - Free the atom table and associated memory
*
*/
void TPpContext::FreeAtomTable(AtomTable *atable)
{
FreeStringTable(&atable->stable);
FreeHashTable(&atable->htable);
if (atable->amap)
free(atable->amap);
if (atable->arev)
free(atable->arev);
atable->amap = NULL;
atable->arev = NULL;
atable->nextFree = 0;
atable->size = 0;
} // FreeAtomTable
} // end namespace glslang

8
glslang/MachineIndependent/preprocessor/PpContext.cpp
View File

@ -84,10 +84,9 @@ NVIDIA HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
namespace glslang {
TPpContext::TPpContext(TParseContext& pc) :
preamble(0), strings(0), notAVersionToken(false), parseContext(pc),
ScopeList(0), CurrentScope(0), GlobalScope(0)
preamble(0), strings(0), notAVersionToken(false), parseContext(pc)
{
InitAtomTable(&atomTable, 0);
InitAtomTable();
InitScanner(this);
ifdepth = 0;
@ -98,9 +97,8 @@ TPpContext::TPpContext(TParseContext& pc) :
TPpContext::~TPpContext()
{
FinalCPP();
delete [] preamble;
FreeAtomTable(&atomTable);
FreeScanner();
}
void TPpContext::setPreamble(const char* p, size_t l)

104
glslang/MachineIndependent/preprocessor/PpContext.h
View File

@ -80,6 +80,8 @@ NVIDIA HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "../ParseHelper.h"
#include <hash_map>
namespace glslang {
class TPpToken {
@ -141,42 +143,8 @@ public:
};
//
// From PpAtom.cpp
// From Pp.cpp
//
struct StringTable {
char *strings;
int nextFree;
int size;
};
struct HashEntry {
int index; // String table offset of string representation
int value; // Atom (symbol) value
};
static const int hashTableMaxCollisions = 3;
struct HashTable {
HashEntry *entry;
int size;
int entries;
int counts[hashTableMaxCollisions + 1];
};
struct AtomTable {
StringTable stable; // String table.
HashTable htable; // Hashes string to atom number and token value. Multiple strings can
// have the same token value but each unique string is a unique atom.
int *amap; // Maps atom value to offset in string table. Atoms all map to unique
// strings except for some undefined values in the lower, fixed part
// of the atom table that map to "<undefined>". The lowest 256 atoms
// correspond to single character ASCII values except for alphanumeric
// characters and '_', which can be other tokens. Next come the
// language tokens with their atom values equal to the token value.
// Then come predefined atoms, followed by user specified identifiers.
int *arev; // Reversed atom for symbol table use.
int nextFree;
int size;
};
struct MacroSymbol {
int argc;
@ -186,19 +154,9 @@ public:
unsigned undef:1;
};
typedef enum symbolkind {
MACRO_S
} symbolkind;
struct Symbol {
Symbol *left, *right;
Symbol *next;
int name; // Name atom
TSourceLoc loc;
symbolkind kind;
union {
int atom;
MacroSymbol mac;
} details;
};
struct SymbolList {
@ -206,18 +164,9 @@ public:
Symbol *symb;
};
struct Scope {
Scope *next, *prev; // doubly-linked list of all scopes
Scope *parent;
Scope *funScope; // Points to base scope of enclosing function
MemoryPool *pool; // pool used for allocation in this scope
Symbol *symbols;
int level; // 0 = super globals, 1 = globals, etc.
// Only used at global scope (level 1):
SymbolList *programs; // List of programs for this compilation.
};
MemoryPool *pool;
typedef std::hash_map<int, Symbol*> TSymbol;
TSymbol symbols; // this has light use... just defined macros
protected:
char* preamble; // string to parse, all before line 1 of string 0, it is 0 if no preamble
@ -277,11 +226,9 @@ protected:
int compatibilityAtom;
int esAtom;
int extensionAtom;
Scope *macros;
TSourceLoc ifloc; /* outermost #if */
int InitCPP();
int FreeCPP();
int FinalCPP();
int CPPdefine(TPpToken * yylvalpp);
int CPPundef(TPpToken * yylvalpp);
@ -307,18 +254,9 @@ protected:
//
// from PpSymbols.cpp
//
Scope *ScopeList;
Scope *CurrentScope;
Scope *GlobalScope;
Scope *NewScopeInPool(MemoryPool *pool);
void PushScope(Scope *fScope);
Scope *PopScope(void);
Symbol *NewSymbol(TSourceLoc *loc, Scope *fScope, int name, symbolkind kind);
void lAddToTree(Symbol **fSymbols, Symbol *fSymb, AtomTable *atable);
Symbol *AddSymbol(TSourceLoc *loc, Scope *fScope, int atom, symbolkind kind);
Symbol *LookUpLocalSymbol(Scope *fScope, int atom);
Symbol *LookUpSymbol(Scope *fScope, int atom);
Symbol *NewSymbol(int name);
Symbol *AddSymbol(int atom);
Symbol *LookUpSymbol(int atom);
//
// From PpTokens.cpp
@ -356,7 +294,6 @@ protected:
char *p;
};
int InitScanner(TPpContext *cpp);
int FreeScanner(void);
static int str_getch(TPpContext*, StringInputSrc *in);
static void str_ungetch(TPpContext*, StringInputSrc *in, int ch, TPpToken *type);
int ScanFromString(char *s);
@ -367,18 +304,15 @@ protected:
//
// From PpAtom.cpp
//
AtomTable atomTable;
int InitAtomTable(AtomTable *atable, int htsize);
void FreeAtomTable(AtomTable *atable);
int AddAtom(AtomTable *atable, const char *s);
int AddAtomFixed(AtomTable *atable, const char *s, int atom);
void PrintAtomTable(AtomTable *atable);
int IncreaseHashTableSize(TPpContext::AtomTable *atable);
int LookUpAddStringHash(AtomTable *atable, const char *s);
int LookUpAddString(AtomTable *atable, const char *s);
const char *GetAtomString(AtomTable *atable, int atom);
int GetReversedAtom(AtomTable *atable, int atom);
char* GetStringOfAtom(AtomTable *atable, int atom);
typedef std::hash_map<const TString, int> TAtomMap;
typedef TVector<const TString*> TStringMap;
TAtomMap atomMap;
TStringMap stringMap;
int nextAtom;
void InitAtomTable();
int AddAtomFixed(const char *s, int atom);
int LookUpAddString(const char *s);
const char *GetAtomString(int atom);
//
// From PpMemory.cpp

15
glslang/MachineIndependent/preprocessor/PpScanner.cpp
View File

@ -122,11 +122,6 @@ int TPpContext::InitScanner(TPpContext *cpp)
return 1;
} // InitScanner
int TPpContext::FreeScanner(void)
{
return (FreeCPP());
}
/*
* str_getch()
* takes care of reading from multiple strings.
@ -386,7 +381,7 @@ int TPpContext::byte_scan(TPpContext* pp, InputSrc *in, TPpToken * yylvalpp)
tokenText[len] = '\0';
pp->currentInput->ungetch(pp, pp->currentInput, ch, yylvalpp);
yylvalpp->atom = pp->LookUpAddString(&pp->atomTable, tokenText);
yylvalpp->atom = pp->LookUpAddString(tokenText);
return CPP_IDENTIFIER;
case '0':
@ -769,7 +764,7 @@ int TPpContext::byte_scan(TPpContext* pp, InputSrc *in, TPpToken * yylvalpp)
};
tokenText[len] = '\0';
if (ch == '"') {
yylvalpp->atom = pp->LookUpAddString(&pp->atomTable, tokenText);
yylvalpp->atom = pp->LookUpAddString(tokenText);
return CPP_STRCONSTANT;
} else {
pp->parseContext.error(yylvalpp->loc, "end of line in string", "string", "");
@ -785,7 +780,7 @@ const char* TPpContext::tokenize(TPpToken* yylvalpp)
for(;;) {
char* tokenString = 0;
const char* tokenString = 0;
token = currentInput->scan(this, currentInput, yylvalpp);
yylvalpp->ppToken = token;
if (check_EOF(token))
@ -813,12 +808,12 @@ const char* TPpContext::tokenize(TPpToken* yylvalpp)
continue;
if (token == CPP_IDENTIFIER)
tokenString = GetStringOfAtom(&atomTable, yylvalpp->atom);
tokenString = GetAtomString(yylvalpp->atom);
else if (token == CPP_INTCONSTANT || token == CPP_UINTCONSTANT ||
token == CPP_FLOATCONSTANT || token == CPP_DOUBLECONSTANT)
tokenString = yylvalpp->name;
else
tokenString = GetStringOfAtom(&atomTable, token);
tokenString = GetAtomString(token);
if (tokenString) {
if (tokenString[0] != 0)

229
glslang/MachineIndependent/preprocessor/PpSymbols.cpp
View File

@ -90,237 +90,46 @@ NVIDIA HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
/////////////////////////////////// Symbol Table Variables: ///////////////////////////////////
///////////////////////////////////////////////////////////////////////////////////////////////
namespace {
using namespace glslang;
void unlinkScope(void *_scope, void* scopeList)
{
TPpContext::Scope *scope = (TPpContext::Scope*)_scope;
if (scope->next)
scope->next->prev = scope->prev;
if (scope->prev)
scope->prev->next = scope->next;
else
*(TPpContext::Scope**)scopeList = scope->next;
}
} // end anonymous namespace
namespace glslang {
/*
* NewScope()
* Allocate a new symbol node;
*
*/
TPpContext::Scope* TPpContext::NewScopeInPool(MemoryPool *pool)
{
Scope *lScope;
lScope = (Scope*)mem_Alloc(pool, sizeof(Scope));
lScope->pool = pool;
lScope->parent = NULL;
lScope->funScope = NULL;
lScope->symbols = NULL;
lScope->level = 0;
lScope->programs = NULL;
if ((lScope->next = ScopeList))
ScopeList->prev = lScope;
lScope->prev = 0;
ScopeList = lScope;
mem_AddCleanup(pool, unlinkScope, lScope, &ScopeList);
return lScope;
} // NewScope
/*
* PushScope()
*
*/
void TPpContext::PushScope(Scope *fScope)
{
Scope *lScope;
if (CurrentScope) {
fScope->level = CurrentScope->level + 1;
if (fScope->level == 1) {
if (!GlobalScope) {
/* HACK - CTD -- if GlobalScope==NULL and level==1, we're
* defining a function in the superglobal scope. Things
* will break if we leave the level as 1, so we arbitrarily
* set it to 2 */
fScope->level = 2;
}
}
if (fScope->level >= 2) {
lScope = fScope;
while (lScope->level > 2)
lScope = lScope->next;
fScope->funScope = lScope;
}
} else {
fScope->level = 0;
}
fScope->parent = CurrentScope;
CurrentScope = fScope;
} // PushScope
/*
* PopScope()
*
*/
TPpContext::Scope* TPpContext::PopScope(void)
{
Scope *lScope;
lScope = CurrentScope;
if (CurrentScope)
CurrentScope = CurrentScope->parent;
return lScope;
} // PopScope
/*
* NewSymbol() - Allocate a new symbol node;
*
*/
TPpContext::Symbol* TPpContext::NewSymbol(TSourceLoc *loc, Scope *fScope, int name, symbolkind kind)
TPpContext::Symbol* TPpContext::NewSymbol(int atom)
{
Symbol *lSymb;
char *pch;
int ii;
lSymb = (Symbol *) mem_Alloc(fScope->pool, sizeof(Symbol));
lSymb->left = NULL;
lSymb->right = NULL;
lSymb->next = NULL;
lSymb->name = name;
lSymb->loc = *loc;
lSymb->kind = kind;
lSymb = (Symbol *) mem_Alloc(pool, sizeof(Symbol));
lSymb->atom = atom;
// Clear union area:
pch = (char *) &lSymb->details;
for (ii = 0; ii < sizeof(lSymb->details); ii++)
// Clear macro
pch = (char *) &lSymb->mac;
for (ii = 0; ii < sizeof(lSymb->mac); ii++)
*pch++ = 0;
return lSymb;
} // NewSymbol
/*
* lAddToTree() - Using a binary tree is not a good idea for basic atom values because they
* are generated in order. We'll fix this later (by reversing the bit pattern).
*/
void TPpContext::lAddToTree(Symbol **fSymbols, Symbol *fSymb, AtomTable *atable)
{
TPpContext::Symbol *lSymb;
int lrev, frev;
lSymb = *fSymbols;
if (lSymb) {
frev = GetReversedAtom(atable, fSymb->name);
while (lSymb) {
lrev = GetReversedAtom(atable, lSymb->name);
if (lrev == frev) {
printf("GetAtomString(atable, fSymb->name)");
break;
} else {
if (lrev > frev) {
if (lSymb->left) {
lSymb = lSymb->left;
} else {
lSymb->left = fSymb;
break;
}
} else {
if (lSymb->right) {
lSymb = lSymb->right;
} else {
lSymb->right = fSymb;
break;
}
}
}
}
} else {
*fSymbols = fSymb;
}
} // lAddToTree
/*
* AddSymbol() - Add a variable, type, or function name to a scope.
*
*/
TPpContext::Symbol* TPpContext::AddSymbol(TSourceLoc *loc, Scope *fScope, int atom, symbolkind kind)
TPpContext::Symbol* TPpContext::AddSymbol(int atom)
{
Symbol *lSymb;
if (!fScope)
fScope = CurrentScope;
lSymb = NewSymbol(loc, fScope, atom, kind);
lAddToTree(&fScope->symbols, lSymb, &atomTable);
lSymb = NewSymbol(atom);
symbols[lSymb->atom] = lSymb;
return lSymb;
} // AddSymbol
}
/*********************************************************************************************/
/************************************ Symbol Semantic Functions ******************************/
/*********************************************************************************************/
/*
* LookUpLocalSymbol()
*
*/
TPpContext::Symbol* TPpContext::LookUpLocalSymbol(Scope *fScope, int atom)
TPpContext::Symbol* TPpContext::LookUpSymbol(int atom)
{
Symbol *lSymb;
int rname, ratom;
ratom = GetReversedAtom(&atomTable, atom);
if (!fScope)
fScope = CurrentScope;
lSymb = fScope->symbols;
while (lSymb) {
rname = GetReversedAtom(&atomTable, lSymb->name);
if (rname == ratom) {
return lSymb;
} else {
if (rname > ratom) {
lSymb = lSymb->left;
} else {
lSymb = lSymb->right;
TSymbol::iterator it = symbols.find(atom);
if (it == symbols.end())
return 0;
else
return it->second;
}
}
}
return NULL;
} // LookUpLocalSymbol
/*
* LookUpSymbol()
*
*/
TPpContext::Symbol* TPpContext::LookUpSymbol(Scope *fScope, int atom)
{
Symbol *lSymb;
if (!fScope)
fScope = CurrentScope;
while (fScope) {
lSymb = LookUpLocalSymbol(fScope, atom);
if (lSymb)
return lSymb;
fScope = fScope->parent;
}
return NULL;
} // LookUpSymbol
} // end namespace glslang

12
glslang/MachineIndependent/preprocessor/PpTokens.cpp
View File

@ -256,7 +256,7 @@ void TPpContext::RecordToken(TokenStream *pTok, int token, TPpToken * yylvalpp)
case CPP_IDENTIFIER:
case CPP_TYPEIDENTIFIER:
case CPP_STRCONSTANT:
s = GetAtomString(&atomTable, yylvalpp->atom);
s = GetAtomString(yylvalpp->atom);
while (*s)
lAddByte(pTok, (unsigned char) *s++);
lAddByte(pTok, 0);
@ -331,7 +331,7 @@ int TPpContext::ReadToken(TokenStream *pTok, TPpToken *yylvalpp)
}
tokenText[len] = '\0';
assert(ch == '\0');
yylvalpp->atom = LookUpAddString(&atomTable, tokenText);
yylvalpp->atom = LookUpAddString(tokenText);
return CPP_IDENTIFIER;
break;
case CPP_STRCONSTANT:
@ -344,7 +344,7 @@ int TPpContext::ReadToken(TokenStream *pTok, TPpToken *yylvalpp)
}
tokenText[len] = 0;
yylvalpp->atom = LookUpAddString(&atomTable, tokenText);
yylvalpp->atom = LookUpAddString(tokenText);
break;
case CPP_FLOATCONSTANT:
case CPP_DOUBLECONSTANT:
@ -465,10 +465,10 @@ void TPpContext::DumpTokenStream(FILE *fp, TokenStream *s, TPpToken * yylvalpp)
switch (token) {
case CPP_IDENTIFIER:
case CPP_TYPEIDENTIFIER:
printf("%s ", GetAtomString(&atomTable, yylvalpp->atom));
printf("%s ", GetAtomString(yylvalpp->atom));
break;
case CPP_STRCONSTANT:
printf("\"%s\"", GetAtomString(&atomTable, yylvalpp->atom));
printf("\"%s\"", GetAtomString(yylvalpp->atom));
break;
case CPP_FLOATCONSTANT:
case CPP_DOUBLECONSTANT:
@ -480,7 +480,7 @@ void TPpContext::DumpTokenStream(FILE *fp, TokenStream *s, TPpToken * yylvalpp)
break;
default:
if (token >= 127)
printf("%s ", GetAtomString(&atomTable, token));
printf("%s ", GetAtomString(token));
else
printf("%c", token);
break;