mewin/glslang
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

SPV compression: Final check-in enabling this on MSVC 2012. All compression submissions from Steve (spvremapper@lunarg.com).

Browse Source 
git-svn-id: https://cvs.khronos.org/svn/repos/ogl/trunk/ecosystem/public/sdk/tools/glslang@31236 e7fa87d3-cd2b-0410-9028-fcbf551c1848
This commit is contained in:
John Kessenich 2015-05-20 16:04:17 +00:00
parent 3c4a276282
commit 01685c3ff8
4 changed files with 374 additions and 435 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
README-spirv-remap.txt
View File

@ -48,11 +48,11 @@ for Linux. Command line arguments can be provided in any order.
Perform ID remapping on all shaders in "*.spv", writing new files with
the same basenames to /tmp/out_dir.
spirv-remap --map all --input *.spv --output /tmp/out_dir
spirv-remap -v --map all --input *.spv --output /tmp/out_dir
2. Perform all possible size reductions
spirv-remap-linux-64 --do-everything --input *.spv --output /tmp/out_dir
spirv-remap-linux-64 -v --do-everything --input *.spv --output /tmp/out_dir
Note that --do-everything is a synonym for:

131
SPIRV/SPVRemapper.cpp
View File

@ -58,7 +58,7 @@ namespace spv {
}
// hash opcode, with special handling for OpExtInst
std::uint32_t spirvbin_t::asOpCodeHash(int word)
std::uint32_t spirvbin_t::asOpCodeHash(unsigned word)
{
const spv::Op opCode = asOpCode(word);
@ -196,7 +196,7 @@ namespace spv {
}
}
const auto inst_fn_nop = [](spv::Op, int) { return false; };
const auto inst_fn_nop = [](spv::Op, unsigned) { return false; };
const auto op_fn_nop = [](spv::Id&) { };
// g++ doesn't like these defined in the class proper in an anonymous namespace.
@ -242,7 +242,7 @@ namespace spv {
// Parse a literal string from the SPIR binary and return it as an std::string
// Due to C++11 RValue references, this doesn't copy the result string.
std::string spirvbin_t::literalString(int word) const
std::string spirvbin_t::literalString(unsigned word) const
{
std::string literal;
@ -304,7 +304,7 @@ namespace spv {
// build local Id and name maps
process(
[&](spv::Op opCode, int start) {
[&](spv::Op opCode, unsigned start) {
// remember opcodes we want to strip later
if (isStripOp(opCode))
stripInst(start);
@ -335,7 +335,7 @@ namespace spv {
// build local Id and name maps
process(
[&](spv::Op opCode, int start) {
[&](spv::Op opCode, unsigned start) {
// remember opcodes we want to strip later
if ((options & Options::STRIP) && isStripOp(opCode))
stripInst(start);
@ -358,7 +358,7 @@ namespace spv {
assert(fnRes != spv::NoResult);
if (fnStart == 0)
error("function end without function start");
fnPos[fnRes] = {fnStart, start + asWordCount(start)};
fnPos[fnRes] = range_t(fnStart, start + asWordCount(start));
fnStart = 0;
} else if (isConstOp(opCode)) {
assert(asId(start + 2) != spv::NoResult);
@ -397,7 +397,7 @@ namespace spv {
}
int spirvbin_t::processInstruction(int word, instfn_t instFn, idfn_t idFn)
int spirvbin_t::processInstruction(unsigned word, instfn_t instFn, idfn_t idFn)
{
const auto instructionStart = word;
const unsigned wordCount = asWordCount(instructionStart);
@ -501,19 +501,19 @@ namespace spv {
}
// Make a pass over all the instructions and process them given appropriate functions
spirvbin_t& spirvbin_t::process(instfn_t instFn, idfn_t idFn, int begin, int end)
spirvbin_t& spirvbin_t::process(instfn_t instFn, idfn_t idFn, unsigned begin, unsigned end)
{
// For efficiency, reserve name map space. It can grow if needed.
nameMap.reserve(32);
// If begin or end == 0, use defaults
begin = (begin == 0 ? header_size : begin);
end = (end == 0 ? int(spv.size()) : end);
end = (end == 0 ? unsigned(spv.size()) : end);
// basic parsing and InstructionDesc table borrowed from SpvDisassemble.cpp...
int nextInst = int(spv.size());
unsigned nextInst = unsigned(spv.size());
for (int word = begin; word < end; word = nextInst)
for (unsigned word = begin; word < end; word = nextInst)
nextInst = processInstruction(word, instFn, idFn);
return *this;
@ -545,12 +545,12 @@ namespace spv {
// hash values.
spv::Id fnId = spv::NoResult;
std::vector<int> instPos;
instPos.reserve(int(spv.size()) / 16); // initial estimate; can grow if needed.
std::vector<unsigned> instPos;
instPos.reserve(unsigned(spv.size()) / 16); // initial estimate; can grow if needed.
// Build local table of instruction start positions
process(
[&](spv::Op, int start) { instPos.push_back(start); return true; },
[&](spv::Op, unsigned start) { instPos.push_back(start); return true; },
op_fn_nop);
// Window size for context-sensitive canonicalization values
@ -558,10 +558,10 @@ namespace spv {
// We essentially performa a little convolution around each instruction,
// to capture the flavor of nearby code, to hopefully match to similar
// code in other modules.
static const int windowSize = 2;
static const unsigned windowSize = 2;
for (int entry = 0; entry < int(instPos.size()); ++entry) {
const int start = instPos[entry];
for (unsigned entry = 0; entry < unsigned(instPos.size()); ++entry) {
const unsigned start = instPos[entry];
const spv::Op opCode = asOpCode(start);
if (opCode == spv::OpFunction)
@ -571,20 +571,18 @@ namespace spv {
fnId = spv::NoResult;
if (fnId != spv::NoResult) { // if inside a function
const int word = start + (spv::InstructionDesc[opCode].hasType() ? 2 : 1);
const int result = spv::InstructionDesc[opCode].hasResult() ? word : -1;
if (result > 0) {
const spv::Id resId = asId(result);
if (spv::InstructionDesc[opCode].hasResult()) {
const unsigned word = start + (spv::InstructionDesc[opCode].hasType() ? 2 : 1);
const spv::Id resId = asId(word);
std::uint32_t hashval = fnId * 17; // small prime
for (int i = entry-1; i >= entry-windowSize; --i) {
for (unsigned i = entry-1; i >= entry-windowSize; --i) {
if (asOpCode(instPos[i]) == spv::OpFunction)
break;
hashval = hashval * 30103 + asOpCodeHash(instPos[i]); // 30103 = semiarbitrary prime
}
for (int i = entry; i <= entry + windowSize; ++i) {
for (unsigned i = entry; i <= entry + windowSize; ++i) {
if (asOpCode(instPos[i]) == spv::OpFunctionEnd)
break;
hashval = hashval * 30103 + asOpCodeHash(instPos[i]); // 30103 = semiarbitrary prime
@ -602,7 +600,7 @@ namespace spv {
fnId = spv::NoResult;
process(
[&](spv::Op opCode, int start) {
[&](spv::Op opCode, unsigned start) {
switch (opCode) {
case spv::OpFunction:
// Reset counters at each function
@ -658,60 +656,6 @@ namespace spv {
});
}
#ifdef NOTDEF
// remove bodies of uncalled functions
void spirvbin_t::offsetIds()
{
// Count of how many functions each ID appears within
std::unordered_map<spv::Id, int> idFnCount;
std::unordered_map<spv::Id, int> idDefinedLoc;
idset_t idsUsed; // IDs used in a given function
int instCount = 0;
// create a count of how many functions each ID is used within
process(
[&](spv::OpCode opCode, int start) {
++instCount;
switch (opCode) {
case spv::OpFunction:
for (const auto id : idsUsed)
++idFnCount[id];
idsUsed.clear();
break;
default:
{
const int word = start + (spv::InstructionDesc[opCode].hasType() ? 2 : 1);
const int result = spv::InstructionDesc[opCode].hasResult() ? word : -1;
if (result > 0)
idDefinedLoc[asId(result)] = instCount;
}
break;
}
return false;
},
[&](spv::Id& id) { idsUsed.insert(id); });
// For each ID defined in exactly one function, replace uses by
// negative offset to definitions in instructions.
static const int relOffsetLimit = 64;
instCount = 0;
process([&](spv::OpCode, int) { ++instCount; return false; },
[&](spv::Id& id) {
if (idFnCount[id] == 1 && (instCount - idDefinedLoc[id]) < relOffsetLimit)
id = idDefinedLoc[id] - instCount;
});
}
#endif
// EXPERIMENTAL: forward IO and uniform load/stores into operands
// This produces invalid Schema-0 SPIRV
void spirvbin_t::forwardLoadStores()
@ -721,7 +665,7 @@ namespace spv {
// EXPERIMENTAL: Forward input and access chain loads into consumptions
process(
[&](spv::Op opCode, int start) {
[&](spv::Op opCode, unsigned start) {
// Add inputs and uniforms to the map
if (((opCode == spv::OpVariable && asWordCount(start) == 4) || (opCode == spv::OpVariableArray)) &&
(spv[start+3] == spv::StorageClassUniform ||
@ -748,7 +692,7 @@ namespace spv {
idMap.clear();
process(
[&](spv::Op opCode, int start) {
[&](spv::Op opCode, unsigned start) {
// Add inputs and uniforms to the map
if (((opCode == spv::OpVariable && asWordCount(start) == 4) || (opCode == spv::OpVariableArray)) &&
(spv[start+3] == spv::StorageClassOutput))
@ -781,7 +725,7 @@ namespace spv {
// Find all the function local pointers stored at most once, and not via access chains
process(
[&](spv::Op opCode, int start) {
[&](spv::Op opCode, unsigned start) {
const int wordCount = asWordCount(start);
// Add local variables to the map
@ -830,7 +774,7 @@ namespace spv {
op_fn_nop);
process(
[&](spv::Op opCode, int start) {
[&](spv::Op opCode, unsigned start) {
if (opCode == spv::OpLoad && fnLocalVars.count(asId(start+3)) > 0)
idMap[asId(start+2)] = idMap[asId(start+3)];
return false;
@ -839,7 +783,7 @@ namespace spv {
// Remove the load/store/variables for the ones we've discovered
process(
[&](spv::Op opCode, int start) {
[&](spv::Op opCode, unsigned start) {
if ((opCode == spv::OpLoad && fnLocalVars.count(asId(start+3)) > 0) ||
(opCode == spv::OpStore && fnLocalVars.count(asId(start+1)) > 0) ||
(opCode == spv::OpVariable && fnLocalVars.count(asId(start+2)) > 0)) {
@ -853,7 +797,6 @@ namespace spv {
[&](spv::Id& id) { if (idMap.find(id) != idMap.end()) id = idMap[id]; }
);
strip(); // strip out data we decided to eliminate
buildLocalMaps(); // rebuild ID mapping data
}
@ -884,7 +827,7 @@ namespace spv {
// decrease counts of called functions
process(
[&](spv::Op opCode, int start) {
[&](spv::Op opCode, unsigned start) {
if (opCode == spv::Op::OpFunctionCall) {
const auto call_it = fnCalls.find(asId(start + 3));
if (call_it != fnCalls.end()) {
@ -914,7 +857,7 @@ namespace spv {
// Count function variable use
process(
[&](spv::Op opCode, int start) {
[&](spv::Op opCode, unsigned start) {
if (opCode == spv::OpVariable) { ++varUseCount[asId(start+2)]; return true; }
return false;
},
@ -924,7 +867,7 @@ namespace spv {
// Remove single-use function variables + associated decorations and names
process(
[&](spv::Op opCode, int start) {
[&](spv::Op opCode, unsigned start) {
if ((opCode == spv::OpVariable && varUseCount[asId(start+2)] == 1) ||
(opCode == spv::OpDecorate && varUseCount[asId(start+1)] == 1) ||
(opCode == spv::OpName && varUseCount[asId(start+1)] == 1)) {
@ -1041,7 +984,7 @@ namespace spv {
#endif // NOTDEF
// Return start position in SPV of given type. error if not found.
int spirvbin_t::typePos(spv::Id id) const
unsigned spirvbin_t::typePos(spv::Id id) const
{
const auto tid_it = typeConstPosR.find(id);
if (tid_it == typeConstPosR.end())
@ -1052,7 +995,7 @@ namespace spv {
// Hash types to canonical values. This can return ID collisions (it's a bit
// inevitable): it's up to the caller to handle that gracefully.
std::uint32_t spirvbin_t::hashType(int typeStart) const
std::uint32_t spirvbin_t::hashType(unsigned typeStart) const
{
const unsigned wordCount = asWordCount(typeStart);
const spv::Op opCode = asOpCode(typeStart);
@ -1160,7 +1103,7 @@ namespace spv {
// Allocate a new binary big enough to hold old binary
// We'll step this iterator through the strip ranges as we go through the binary
decltype(stripRange)::const_iterator strip_it = stripRange.begin();
auto strip_it = stripRange.begin();
int strippedPos = 0;
for (unsigned word = 0; word < unsigned(spv.size()); ++word) {
@ -1201,12 +1144,6 @@ namespace spv {
mapRemainder(); // map any unmapped IDs
applyMap(); // Now remap each shader to the new IDs we've come up with
strip(); // strip out data we decided to eliminate
#define EXPERIMENT3 0
#if (EXPERIMENT3)
// TODO: ... shortcuts for simple single-const access chains and constants,
// folded into high half of the ID space.
#endif
}
// remap from a memory image

39
SPIRV/SPVRemapper.h
View File

@ -43,7 +43,7 @@ namespace spv {
// MSVC defines __cplusplus as an older value, even when it supports almost all of 11.
// We handle that here by making our own symbol.
#if __cplusplus >= 201103L || _MSC_VER >= 1800
#if __cplusplus >= 201103L || _MSC_VER >= 1700
# define use_cpp11 1
#endif
@ -84,6 +84,7 @@ public:
void remap(std::vector<unsigned int>& /*spv*/, unsigned int /*opts = 0*/)
{
printf("Tool not compiled for C++11, which is required for SPIR-V remapping.\n");
exit(5);
}
};
@ -137,9 +138,9 @@ private:
typedef std::uint32_t spirword_t;
typedef std::pair<int, int> range_t;
typedef std::pair<unsigned, unsigned> range_t;
typedef std::function<void(spv::Id&)> idfn_t;
typedef std::function<bool(spv::Op, int start)> instfn_t;
typedef std::function<bool(spv::Op, unsigned start)> instfn_t;
// Special Values for ID map:
static const spv::Id unmapped; // unchanged from default value
@ -168,14 +169,14 @@ private:
range_t typeRange(spv::Op opCode) const;
range_t constRange(spv::Op opCode) const;
spv::Id& asId(int word) { return spv[word]; }
const spv::Id& asId(int word) const { return spv[word]; }
spv::Op asOpCode(int word) const { return opOpCode(spv[word]); }
std::uint32_t asOpCodeHash(int word);
spv::Decoration asDecoration(int word) const { return spv::Decoration(spv[word]); }
unsigned asWordCount(int word) const { return opWordCount(spv[word]); }
spv::Id asTypeConstId(int word) const { return asId(word + (isTypeOp(asOpCode(word)) ? 1 : 2)); }
int typePos(spv::Id id) const;
spv::Id& asId(unsigned word) { return spv[word]; }
const spv::Id& asId(unsigned word) const { return spv[word]; }
spv::Op asOpCode(unsigned word) const { return opOpCode(spv[word]); }
std::uint32_t asOpCodeHash(unsigned word);
spv::Decoration asDecoration(unsigned word) const { return spv::Decoration(spv[word]); }
unsigned asWordCount(unsigned word) const { return opWordCount(spv[word]); }
spv::Id asTypeConstId(unsigned word) const { return asId(word + (isTypeOp(asOpCode(word)) ? 1 : 2)); }
unsigned typePos(spv::Id id) const;
static unsigned opWordCount(spirword_t data) { return data >> spv::WordCountShift; }
static spv::Op opOpCode(spirword_t data) { return spv::Op(data & spv::OpCodeMask); }
@ -201,7 +202,7 @@ private:
inline spv::Id nextUnusedId(spv::Id id);
void buildLocalMaps();
std::string literalString(int word) const; // Return literal as a std::string
std::string literalString(unsigned word) const; // Return literal as a std::string
int literalStringWords(const std::string& str) const { return (int(str.size())+4)/4; }
bool isNewIdMapped(spv::Id newId) const { return isMapped(newId); }
@ -212,10 +213,10 @@ private:
// bool matchType(const globaltypes_t& globalTypes, spv::Id lt, spv::Id gt) const;
// spv::Id findType(const globaltypes_t& globalTypes, spv::Id lt) const;
std::uint32_t hashType(int typeStart) const;
std::uint32_t hashType(unsigned typeStart) const;
spirvbin_t& process(instfn_t, idfn_t, int begin = 0, int end = 0);
int processInstruction(int word, instfn_t, idfn_t);
spirvbin_t& process(instfn_t, idfn_t, unsigned begin = 0, unsigned end = 0);
int processInstruction(unsigned word, instfn_t, idfn_t);
void validate() const;
void mapTypeConst();
@ -251,12 +252,12 @@ private:
// Add a strip range for a given instruction starting at 'start'
// Note: avoiding brace initializers to please older versions os MSVC.
void stripInst(int start) { stripRange.push_back(std::pair<unsigned, unsigned>(start, start + asWordCount(start))); }
void stripInst(unsigned start) { stripRange.push_back(range_t(start, start + asWordCount(start))); }
// Function start and end. use unordered_map because we'll have
// many fewer functions than IDs.
std::unordered_map<spv::Id, std::pair<int, int>> fnPos;
std::unordered_map<spv::Id, std::pair<int, int>> fnPosDCE; // deleted functions
std::unordered_map<spv::Id, range_t> fnPos;
std::unordered_map<spv::Id, range_t> fnPosDCE; // deleted functions
// Which functions are called, anywhere in the module, with a call count
std::unordered_map<spv::Id, int> fnCalls;
@ -270,7 +271,7 @@ private:
spv::Id largestNewId; // biggest new ID we have mapped anything to
// Sections of the binary to strip, given as [begin,end)
std::vector<std::pair<unsigned, unsigned>> stripRange;
std::vector<range_t> stripRange;
// processing options:
std::uint32_t options;

19
StandAlone/spirv-remap.cpp
View File

@ -75,11 +75,12 @@ namespace {
}
// Read word stream from disk
void read(std::vector<SpvWord>& spv, const std::string& inFilename)
void read(std::vector<SpvWord>& spv, const std::string& inFilename, int verbosity)
{
std::ifstream fp;
std::cout << " reading: " << inFilename << std::endl;
if (verbosity > 0)
logHandler(std::string(" reading: ") + inFilename);
spv.clear();
fp.open(inFilename, std::fstream::in | std::fstream::binary);
@ -104,14 +105,15 @@ namespace {
}
}
void write(std::vector<SpvWord>& spv, const std::string& outFile)
void write(std::vector<SpvWord>& spv, const std::string& outFile, int verbosity)
{
if (outFile.empty())
errHandler("missing output filename.");
std::ofstream fp;
std::cout << " writing: " << outFile << std::endl;
if (verbosity > 0)
logHandler(std::string(" writing: ") + outFile);
fp.open(outFile, std::fstream::out | std::fstream::binary);
@ -157,12 +159,12 @@ namespace {
{
for (const auto& filename : inputFile) {
std::vector<SpvWord> spv;
read(spv, filename);
read(spv, filename, verbosity);
spv::spirvbin_t(verbosity).remap(spv, opts);
const std::string outfile = outputDir + path_sep_char() + basename(filename);
write(spv, outfile);
write(spv, outfile, verbosity);
}
if (verbosity > 0)
@ -305,17 +307,18 @@ namespace {
int main(int argc, char** argv)
{
#ifdef use_cpp11
std::vector<std::string> inputFile;
std::string outputDir;
int opts;
int verbosity;
#ifdef use_cpp11
// handle errors by exiting
spv::spirvbin_t::registerErrorHandler(errHandler);
// Log messages to std::cout
spv::spirvbin_t::registerLogHandler(logHandler);
#endif
if (argc < 2)
usage(argv[0]);
@ -330,7 +333,5 @@ int main(int argc, char** argv)
// Main operations: read, remap, and write.
execute(inputFile, outputDir, opts, verbosity);
#endif
// If we get here, everything went OK! Nothing more to be done.
}