diff --git a/README-spirv-remap.txt b/README-spirv-remap.txt new file mode 100644 index 00000000..abd924ec --- /dev/null +++ b/README-spirv-remap.txt @@ -0,0 +1,137 @@ + +VERSION +-------------------------------------------------------------------------------- +spirv-remap 0.97 + +INTRO: +-------------------------------------------------------------------------------- +spirv-remap is a utility to improve compression of SPIRV binary files via +entropy reduction, plus optional stripping of debug information and +load/store optimization. It transforms SPIRV to SPIRV, remapping IDs. The +resulting modules have an increased ID range (IDs are not as tightly packed +around zero), but will compress better when multiple modules are compressed +together, since compressor's dictionary can find better cross module +commonality. + +Remapping is accomplished via canonicalization. Thus, modules can be +compressed one at a time with no loss of quality relative to operating on +many modules at once. The command line tool operates on multiple modules +only in the trivial repetition sense, for ease of use. The remapper API +only accepts a single module at a time. + +There are two modes of use: command line, and a C++11 API. Both are +described below. + +spirv-remap is currently in an alpha state. Although there are no known +remapping defects, it has only been exercised on one real world game shader +workload. + + +FEEDBACK +-------------------------------------------------------------------------------- +Report defects, enhancements requests, code improvements, etc to: + spvremapper@lunarg.com + + +COMMAND LINE USAGE: +-------------------------------------------------------------------------------- +Examples are given with a verbosity of one (-v), but more verbosity can be +had via -vv, -vvv, etc, or an integer parameter to --verbose, such as +"--verbose 4". With no verbosity, the command is silent and returns 0 on +success, and a positive integer error on failure. + +Pre-built binaries for several OSs are available. Examples presented are +for Linux. Command line arguments can be provided in any order. + +1. Basic ID remapping + +Perform ID remapping on all shaders in "*.spv", writing new files with +the same basenames to /tmp/out_dir. + + spirv-remap --map all --input *.spv --output /tmp/out_dir + +2. Perform all possible size reductions + + spirv-remap-linux-64 --do-everything --input *.spv --output /tmp/out_dir + +Note that --do-everything is a synonym for: + + --map all --dce all --opt all --strip all + +API USAGE: +-------------------------------------------------------------------------------- + +The public interface to the remapper is defined in SPIRV/SPVRemapper.h as follows: + +namespace spv { + +class spirvbin_t +{ +public: + enum Options { ... }; + spirvbin_t(int verbose = 0); // construct + + // remap an existing binary in memory + void remap(std::vector& spv, std::uint32_t opts = Options::DO_EVERYTHING); + + // Type for error/log handler functions + typedef std::function errorfn_t; + typedef std::function logfn_t; + + // Register error/log handling functions (can be c/c++ fn, lambda fn, or functor) + static void registerErrorHandler(errorfn_t handler) { errorHandler = handler; } + static void registerLogHandler(logfn_t handler) { logHandler = handler; } +}; + +} // namespace spv + +The class definition is in SPVRemapper.cpp. + +remap() accepts an std::vector of SPIRV words, modifies them per the +request given in 'opts', and leaves the 'spv' container with the result. +It is safe to instantiate one spirvbin_t per thread and process a different +SPIRV in each. + +The "opts" parameter to remap() accepts a bit mask of desired remapping +options. See REMAPPING AND OPTIMIZATION OPTIONS. + +On error, the function supplied to registerErrorHandler() will be invoked. +This can be a standard C/C++ function, a lambda function, or a functor. +The default handler simply calls exit(5); The error handler is a static +members, so need only be set up once, not once per spirvbin_t instance. + +Log messages are supplied to registerLogHandler(). By default, log +messages are eaten silently. The log handler is also a static member. + +BUILD DEPENDENCIES: +-------------------------------------------------------------------------------- + 1. C++11 compatible compiler + 2. cmake + 3. glslang + + +BUILDING +-------------------------------------------------------------------------------- +The standalone remapper is built along side glslangValidator through its +normal build process. + + +REMAPPING AND OPTIMIZATION OPTIONS +-------------------------------------------------------------------------------- +API: + These are bits defined under spv::spirvbin_t::Options::, and can be + bitwise or-ed together as desired. + + MAP_TYPES = canonicalize type IDs + MAP_NAMES = canonicalize named data + MAP_FUNCS = canonicalize function bodies + DCE_FUNCS = remove dead functions + DCE_VARS = remove dead variables + DCE_TYPES = remove dead types + OPT_LOADSTORE = optimize unneeded load/stores + MAP_ALL = (MAP_TYPES | MAP_NAMES | MAP_FUNCS) + DCE_ALL = (DCE_FUNCS | DCE_VARS | DCE_TYPES) + OPT_ALL = (OPT_LOADSTORE) + ALL_BUT_STRIP = (MAP_ALL | DCE_ALL | OPT_ALL) + DO_EVERYTHING = (STRIP | ALL_BUT_STRIP) + diff --git a/SPIRV/SPVRemapper.cpp b/SPIRV/SPVRemapper.cpp index 97eb4a5c..d390c77d 100644 --- a/SPIRV/SPVRemapper.cpp +++ b/SPIRV/SPVRemapper.cpp @@ -32,11 +32,9 @@ //ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE //POSSIBILITY OF SUCH DAMAGE. // - -#include "SPVRemapper.h" -#include "doc.h" -/* -*-mode:c++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 3 -*- */ +#include "SPVRemapper.h" +#include "doc.h" #if !defined (use_cpp11) // ... not supported before C++11 @@ -45,1178 +43,1181 @@ #include #include -namespace spv { - -// By default, just abort on error. Can be overridden via RegisterErrorHandler -spirvbin_t::errorfn_t spirvbin_t::errorHandler = [](const std::string&) { exit(5); }; -// By default, eat log messages. Can be overridden via RegisterLogHandler -spirvbin_t::logfn_t spirvbin_t::logHandler = [](const std::string&) { }; - -// This can be overridden to provide other message behavior if needed -void spirvbin_t::msg(int minVerbosity, int indent, const std::string& txt) const -{ - if (verbose >= minVerbosity) - logHandler(std::string(indent, ' ') + txt); -} - -// hash opcode, with special handling for OpExtInst -std::uint32_t spirvbin_t::asOpCodeHash(int word) -{ - const spv::Op opCode = asOpCode(word); - - std::uint32_t offset = 0; - - switch (opCode) { - case spv::OpExtInst: - offset += asId(word + 4); break; - default: - break; - } - - return opCode * 19 + offset; // 19 = small prime -} - -spirvbin_t::range_t spirvbin_t::literalRange(spv::Op opCode) const -{ - static const int maxCount = 1<<30; - - switch (opCode) { - case spv::OpTypeFloat: // fall through... - case spv::OpTypePointer: return range_t(2, 3); - case spv::OpTypeInt: return range_t(2, 4); - case spv::OpTypeSampler: return range_t(3, 8); - case spv::OpTypeVector: // fall through - case spv::OpTypeMatrix: // ... - case spv::OpTypePipe: return range_t(3, 4); - case spv::OpConstant: return range_t(3, maxCount); - default: return range_t(0, 0); - } -} - -spirvbin_t::range_t spirvbin_t::typeRange(spv::Op opCode) const -{ - static const int maxCount = 1<<30; - - if (isConstOp(opCode)) - return range_t(1, 2); - - switch (opCode) { - case spv::OpTypeVector: // fall through - case spv::OpTypeMatrix: // ... - case spv::OpTypeSampler: // ... - case spv::OpTypeArray: // ... - case spv::OpTypeRuntimeArray: // ... - case spv::OpTypePipe: return range_t(2, 3); - case spv::OpTypeStruct: // fall through - case spv::OpTypeFunction: return range_t(2, maxCount); - case spv::OpTypePointer: return range_t(3, 4); - default: return range_t(0, 0); - } -} - -spirvbin_t::range_t spirvbin_t::constRange(spv::Op opCode) const -{ - static const int maxCount = 1<<30; - - switch (opCode) { - case spv::OpTypeArray: // fall through... - case spv::OpTypeRuntimeArray: return range_t(3, 4); - case spv::OpConstantComposite: return range_t(3, maxCount); - default: return range_t(0, 0); - } -} - -// Is this an opcode we should remove when using --strip? -bool spirvbin_t::isStripOp(spv::Op opCode) const -{ - switch (opCode) { - case spv::OpSource: - case spv::OpSourceExtension: - case spv::OpName: - case spv::OpMemberName: - case spv::OpLine: return true; - default: return false; - } -} - -bool spirvbin_t::isFlowCtrlOpen(spv::Op opCode) const -{ - switch (opCode) { - case spv::OpBranchConditional: - case spv::OpSwitch: return true; - default: return false; - } -} - -bool spirvbin_t::isFlowCtrlClose(spv::Op opCode) const -{ - switch (opCode) { - case spv::OpLoopMerge: - case spv::OpSelectionMerge: return true; - default: return false; - } -} - -bool spirvbin_t::isTypeOp(spv::Op opCode) const -{ - switch (opCode) { - case spv::OpTypeVoid: - case spv::OpTypeBool: - case spv::OpTypeInt: - case spv::OpTypeFloat: - case spv::OpTypeVector: - case spv::OpTypeMatrix: - case spv::OpTypeSampler: - case spv::OpTypeFilter: - case spv::OpTypeArray: - case spv::OpTypeRuntimeArray: - case spv::OpTypeStruct: - case spv::OpTypeOpaque: - case spv::OpTypePointer: - case spv::OpTypeFunction: - case spv::OpTypeEvent: - case spv::OpTypeDeviceEvent: - case spv::OpTypeReserveId: - case spv::OpTypeQueue: - case spv::OpTypePipe: return true; - default: return false; - } -} - -bool spirvbin_t::isConstOp(spv::Op opCode) const -{ - switch (opCode) { - case spv::OpConstantNullObject: error("unimplemented constant type"); - case spv::OpConstantSampler: error("unimplemented constant type"); - - case spv::OpConstantTrue: - case spv::OpConstantFalse: - case spv::OpConstantNullPointer: - case spv::OpConstantComposite: - case spv::OpConstant: return true; - default: return false; - } -} - -const auto inst_fn_nop = [](spv::Op, int) { return false; }; -const auto op_fn_nop = [](spv::Id&) { }; - -// g++ doesn't like these defined in the class proper in an anonymous namespace. -// Dunno why. Also MSVC doesn't like the constexpr keyword. Also dunno why. -// Defining them externally seems to please both compilers, so, here they are. -const spv::Id spirvbin_t::unmapped = spv::Id(-10000); -const spv::Id spirvbin_t::unused = spv::Id(-10001); -const int spirvbin_t::header_size = 5; - -spv::Id spirvbin_t::nextUnusedId(spv::Id id) -{ - while (isNewIdMapped(id)) // search for an unused ID - ++id; - - return id; -} - -spv::Id spirvbin_t::localId(spv::Id id, spv::Id newId) -{ - assert(id != spv::NoResult && newId != spv::NoResult); - - if (id >= idMapL.size()) - idMapL.resize(id+1, unused); +namespace spv { - if (newId != unmapped && newId != unused) { - if (isOldIdUnused(id)) - error(std::string("ID unused in module: ") + std::to_string(id)); + // By default, just abort on error. Can be overridden via RegisterErrorHandler + spirvbin_t::errorfn_t spirvbin_t::errorHandler = [](const std::string&) { exit(5); }; + // By default, eat log messages. Can be overridden via RegisterLogHandler + spirvbin_t::logfn_t spirvbin_t::logHandler = [](const std::string&) { }; - if (!isOldIdUnmapped(id)) - error(std::string("ID already mapped: ") + std::to_string(id) + " -> " + // This can be overridden to provide other message behavior if needed + void spirvbin_t::msg(int minVerbosity, int indent, const std::string& txt) const + { + if (verbose >= minVerbosity) + logHandler(std::string(indent, ' ') + txt); + } + + // hash opcode, with special handling for OpExtInst + std::uint32_t spirvbin_t::asOpCodeHash(int word) + { + const spv::Op opCode = asOpCode(word); + + std::uint32_t offset = 0; + + switch (opCode) { + case spv::OpExtInst: + offset += asId(word + 4); break; + default: + break; + } + + return opCode * 19 + offset; // 19 = small prime + } + + spirvbin_t::range_t spirvbin_t::literalRange(spv::Op opCode) const + { + static const int maxCount = 1<<30; + + switch (opCode) { + case spv::OpTypeFloat: // fall through... + case spv::OpTypePointer: return range_t(2, 3); + case spv::OpTypeInt: return range_t(2, 4); + case spv::OpTypeSampler: return range_t(3, 8); + case spv::OpTypeVector: // fall through + case spv::OpTypeMatrix: // ... + case spv::OpTypePipe: return range_t(3, 4); + case spv::OpConstant: return range_t(3, maxCount); + default: return range_t(0, 0); + } + } + + spirvbin_t::range_t spirvbin_t::typeRange(spv::Op opCode) const + { + static const int maxCount = 1<<30; + + if (isConstOp(opCode)) + return range_t(1, 2); + + switch (opCode) { + case spv::OpTypeVector: // fall through + case spv::OpTypeMatrix: // ... + case spv::OpTypeSampler: // ... + case spv::OpTypeArray: // ... + case spv::OpTypeRuntimeArray: // ... + case spv::OpTypePipe: return range_t(2, 3); + case spv::OpTypeStruct: // fall through + case spv::OpTypeFunction: return range_t(2, maxCount); + case spv::OpTypePointer: return range_t(3, 4); + default: return range_t(0, 0); + } + } + + spirvbin_t::range_t spirvbin_t::constRange(spv::Op opCode) const + { + static const int maxCount = 1<<30; + + switch (opCode) { + case spv::OpTypeArray: // fall through... + case spv::OpTypeRuntimeArray: return range_t(3, 4); + case spv::OpConstantComposite: return range_t(3, maxCount); + default: return range_t(0, 0); + } + } + + // Is this an opcode we should remove when using --strip? + bool spirvbin_t::isStripOp(spv::Op opCode) const + { + switch (opCode) { + case spv::OpSource: + case spv::OpSourceExtension: + case spv::OpName: + case spv::OpMemberName: + case spv::OpLine: return true; + default: return false; + } + } + + bool spirvbin_t::isFlowCtrlOpen(spv::Op opCode) const + { + switch (opCode) { + case spv::OpBranchConditional: + case spv::OpSwitch: return true; + default: return false; + } + } + + bool spirvbin_t::isFlowCtrlClose(spv::Op opCode) const + { + switch (opCode) { + case spv::OpLoopMerge: + case spv::OpSelectionMerge: return true; + default: return false; + } + } + + bool spirvbin_t::isTypeOp(spv::Op opCode) const + { + switch (opCode) { + case spv::OpTypeVoid: + case spv::OpTypeBool: + case spv::OpTypeInt: + case spv::OpTypeFloat: + case spv::OpTypeVector: + case spv::OpTypeMatrix: + case spv::OpTypeSampler: + case spv::OpTypeFilter: + case spv::OpTypeArray: + case spv::OpTypeRuntimeArray: + case spv::OpTypeStruct: + case spv::OpTypeOpaque: + case spv::OpTypePointer: + case spv::OpTypeFunction: + case spv::OpTypeEvent: + case spv::OpTypeDeviceEvent: + case spv::OpTypeReserveId: + case spv::OpTypeQueue: + case spv::OpTypePipe: return true; + default: return false; + } + } + + bool spirvbin_t::isConstOp(spv::Op opCode) const + { + switch (opCode) { + case spv::OpConstantNullObject: error("unimplemented constant type"); + case spv::OpConstantSampler: error("unimplemented constant type"); + + case spv::OpConstantTrue: + case spv::OpConstantFalse: + case spv::OpConstantNullPointer: + case spv::OpConstantComposite: + case spv::OpConstant: return true; + default: return false; + } + } + + const auto inst_fn_nop = [](spv::Op, int) { return false; }; + const auto op_fn_nop = [](spv::Id&) { }; + + // g++ doesn't like these defined in the class proper in an anonymous namespace. + // Dunno why. Also MSVC doesn't like the constexpr keyword. Also dunno why. + // Defining them externally seems to please both compilers, so, here they are. + const spv::Id spirvbin_t::unmapped = spv::Id(-10000); + const spv::Id spirvbin_t::unused = spv::Id(-10001); + const int spirvbin_t::header_size = 5; + + spv::Id spirvbin_t::nextUnusedId(spv::Id id) + { + while (isNewIdMapped(id)) // search for an unused ID + ++id; + + return id; + } + + spv::Id spirvbin_t::localId(spv::Id id, spv::Id newId) + { + assert(id != spv::NoResult && newId != spv::NoResult); + + if (id >= idMapL.size()) + idMapL.resize(id+1, unused); + + if (newId != unmapped && newId != unused) { + if (isOldIdUnused(id)) + error(std::string("ID unused in module: ") + std::to_string(id)); + + if (!isOldIdUnmapped(id)) + error(std::string("ID already mapped: ") + std::to_string(id) + " -> " + std::to_string(localId(id))); - if (isNewIdMapped(newId)) - error(std::string("ID already used in module: ") + std::to_string(newId)); + if (isNewIdMapped(newId)) + error(std::string("ID already used in module: ") + std::to_string(newId)); - msg(4, 4, std::string("map: ") + std::to_string(id) + " -> " + std::to_string(newId)); - setMapped(newId); - largestNewId = std::max(largestNewId, newId); - } - - return idMapL[id] = newId; -} - -// Parse a literal string from the SPIR binary and return it as an std::string -// Due to C++11 RValue references, this doesn't copy the result string. -std::string spirvbin_t::literalString(int word) const -{ - std::string literal; - - literal.reserve(16); - - const char* bytes = reinterpret_cast(spv.data() + word); - - while (bytes && *bytes) - literal += *bytes++; - - return literal; -} + msg(4, 4, std::string("map: ") + std::to_string(id) + " -> " + std::to_string(newId)); + setMapped(newId); + largestNewId = std::max(largestNewId, newId); + } + + return idMapL[id] = newId; + } + + // Parse a literal string from the SPIR binary and return it as an std::string + // Due to C++11 RValue references, this doesn't copy the result string. + std::string spirvbin_t::literalString(int word) const + { + std::string literal; + + literal.reserve(16); + + const char* bytes = reinterpret_cast(spv.data() + word); + + while (bytes && *bytes) + literal += *bytes++; + + return literal; + } -void spirvbin_t::applyMap() -{ - msg(3, 2, std::string("Applying map: ")); - - // Map local IDs through the ID map - process(inst_fn_nop, // ignore instructions - [this](spv::Id& id) { - id = localId(id); - assert(id != unused && id != unmapped); - } - ); -} - - -// Find free IDs for anything we haven't mapped -void spirvbin_t::mapRemainder() -{ - msg(3, 2, std::string("Remapping remainder: ")); + void spirvbin_t::applyMap() + { + msg(3, 2, std::string("Applying map: ")); - spv::Id unusedId = 1; // can't use 0: that's NoResult - spirword_t maxBound = 0; - - for (spv::Id id = 0; id < idMapL.size(); ++id) { - if (isOldIdUnused(id)) - continue; - - // Find a new mapping for any used but unmapped IDs - if (isOldIdUnmapped(id)) - localId(id, unusedId = nextUnusedId(unusedId)); - - if (isOldIdUnmapped(id)) - error(std::string("old ID not mapped: ") + std::to_string(id)); - - // Track max bound - maxBound = std::max(maxBound, localId(id) + 1); - } - - bound(maxBound); // reset header ID bound to as big as it now needs to be -} - -void spirvbin_t::stripDebug() -{ - if ((options & Options::STRIP) == 0) - return; - - // build local Id and name maps - process( - [&](spv::Op opCode, int start) { - // remember opcodes we want to strip later - if (isStripOp(opCode)) - stripInst(start); - return true; - }, - op_fn_nop); -} - -void spirvbin_t::buildLocalMaps() -{ - msg(2, 2, std::string("build local maps: ")); - - mapped.clear(); - idMapL.clear(); - nameMap.clear(); - fnPos.clear(); - fnPosDCE.clear(); - fnCalls.clear(); - typeConstPos.clear(); - typeConstPosR.clear(); - entryPoint = spv::NoResult; - largestNewId = 0; - - idMapL.resize(bound(), unused); - - int fnStart = 0; - spv::Id fnRes = spv::NoResult; - - // build local Id and name maps - process( - [&](spv::Op opCode, int start) { - // remember opcodes we want to strip later - if ((options & Options::STRIP) && isStripOp(opCode)) - stripInst(start); - - if (opCode == spv::Op::OpName) { - const spv::Id target = asId(start+1); - const std::string name = literalString(start+2); - nameMap[name] = target; - return true; - } else if (opCode == spv::Op::OpFunctionCall) { - ++fnCalls[asId(start + 3)]; - } else if (opCode == spv::Op::OpEntryPoint) { - entryPoint = asId(start + 2); - } else if (opCode == spv::Op::OpFunction) { - if (fnStart != 0) - error("nested function found"); - fnStart = start; - fnRes = asId(start + 2); - } else if (opCode == spv::Op::OpFunctionEnd) { - assert(fnRes != spv::NoResult); - if (fnStart == 0) - error("function end without function start"); - fnPos[fnRes] = {fnStart, start + asWordCount(start)}; - fnStart = 0; - } else if (isConstOp(opCode)) { - assert(asId(start + 2) != spv::NoResult); - typeConstPos.insert(start); - typeConstPosR[asId(start + 2)] = start; - } else if (isTypeOp(opCode)) { - assert(asId(start + 1) != spv::NoResult); - typeConstPos.insert(start); - typeConstPosR[asId(start + 1)] = start; - } - - return false; - }, - - [this](spv::Id& id) { localId(id, unmapped); } - ); -} - -// Validate the SPIR header -void spirvbin_t::validate() const -{ - msg(2, 2, std::string("validating: ")); - - if (spv.size() < header_size) - error("file too short: "); - - if (magic() != spv::MagicNumber) - error("bad magic number"); - - // field 1 = version - // field 2 = generator magic - // field 3 = result bound - - if (schemaNum() != 0) - error("bad schema, must be 0"); -} + // Map local IDs through the ID map + process(inst_fn_nop, // ignore instructions + [this](spv::Id& id) { + id = localId(id); + assert(id != unused && id != unmapped); + } + ); + } -int spirvbin_t::processInstruction(int word, instfn_t instFn, idfn_t idFn) -{ - const auto instructionStart = word; - const unsigned wordCount = asWordCount(instructionStart); - const spv::Op opCode = asOpCode(instructionStart); - const int nextInst = word++ + wordCount; + // Find free IDs for anything we haven't mapped + void spirvbin_t::mapRemainder() + { + msg(3, 2, std::string("Remapping remainder: ")); - if (nextInst > int(spv.size())) - error("spir instruction terminated too early"); + spv::Id unusedId = 1; // can't use 0: that's NoResult + spirword_t maxBound = 0; - // Base for computing number of operands; will be updated as more is learned - unsigned numOperands = wordCount - 1; + for (spv::Id id = 0; id < idMapL.size(); ++id) { + if (isOldIdUnused(id)) + continue; - if (instFn(opCode, instructionStart)) - return nextInst; + // Find a new mapping for any used but unmapped IDs + if (isOldIdUnmapped(id)) + localId(id, unusedId = nextUnusedId(unusedId)); - // Read type and result ID from instruction desc table - if (spv::InstructionDesc[opCode].hasType()) { - idFn(asId(word++)); - --numOperands; - } - - if (spv::InstructionDesc[opCode].hasResult()) { - idFn(asId(word++)); - --numOperands; - } - - // Extended instructions: currently, assume everything is an ID. - // TODO: add whatever data we need for exceptions to that - if (opCode == spv::OpExtInst) { - word += 2; // instruction set, and instruction from set - numOperands -= 2; - - for (unsigned op=0; op < numOperands; ++op) - idFn(asId(word++)); // ID - - return nextInst; - } - - // Store IDs from instruction in our map - for (int op = 0; op < spv::InstructionDesc[opCode].operands.getNum(); ++op, --numOperands) { - switch (spv::InstructionDesc[opCode].operands.getClass(op)) { - case spv::OperandId: - idFn(asId(word++)); - break; - - case spv::OperandOptionalId: - case spv::OperandVariableIds: - for (unsigned i = 0; i < numOperands; ++i) - idFn(asId(word++)); - return nextInst; - - case spv::OperandVariableLiterals: - if (opCode == spv::OpDecorate && asDecoration(word - 1) == spv::DecorationBuiltIn) { - ++word; - --numOperands; - } - word += numOperands; - return nextInst; - - case spv::OperandVariableLiteralId: - while (numOperands > 0) { - ++word; // immediate - idFn(asId(word++)); // ID - numOperands -= 2; - } - return nextInst; - - case spv::OperandLiteralString: - word += literalStringWords(literalString(word)); - return nextInst; - - // Single word operands we simply ignore, as they hold no IDs - case spv::OperandLiteralNumber: - case spv::OperandSource: - case spv::OperandExecutionModel: - case spv::OperandAddressing: - case spv::OperandMemory: - case spv::OperandExecutionMode: - case spv::OperandStorage: - case spv::OperandDimensionality: - case spv::OperandDecoration: - case spv::OperandBuiltIn: - case spv::OperandSelect: - case spv::OperandLoop: - case spv::OperandFunction: - case spv::OperandMemorySemantics: - case spv::OperandMemoryAccess: - case spv::OperandExecutionScope: - case spv::OperandGroupOperation: - case spv::OperandKernelEnqueueFlags: - case spv::OperandKernelProfilingInfo: - ++word; - break; - - default: - break; - } - } - - return nextInst; -} - -// Make a pass over all the instructions and process them given appropriate functions -spirvbin_t& spirvbin_t::process(instfn_t instFn, idfn_t idFn, int begin, int end) -{ - // For efficiency, reserve name map space. It can grow if needed. - nameMap.reserve(32); - - // If begin or end == 0, use defaults - begin = (begin == 0 ? header_size : begin); - end = (end == 0 ? int(spv.size()) : end); - - // basic parsing and InstructionDesc table borrowed from SpvDisassemble.cpp... - int nextInst = int(spv.size()); - - for (int word = begin; word < end; word = nextInst) - nextInst = processInstruction(word, instFn, idFn); - - return *this; -} - -// Apply global name mapping to a single module -void spirvbin_t::mapNames() -{ - static const std::uint32_t softTypeIdLimit = 3011; // small prime. TODO: get from options - static const std::uint32_t firstMappedID = 3019; // offset into ID space - - for (const auto& name : nameMap) { - std::uint32_t hashval = 1911; - for (const char c : name.first) - hashval = hashval * 1009 + c; - - if (isOldIdUnmapped(name.second)) - localId(name.second, nextUnusedId(hashval % softTypeIdLimit + firstMappedID)); - } -} - -// Map fn contents to IDs of similar functions in other modules -void spirvbin_t::mapFnBodies() -{ - static const std::uint32_t softTypeIdLimit = 19071; // small prime. TODO: get from options - static const std::uint32_t firstMappedID = 6203; // offset into ID space - - // Initial approach: go through some high priority opcodes first and assign them - // hash values. - - spv::Id fnId = spv::NoResult; - std::vector instPos; - instPos.reserve(int(spv.size()) / 16); // initial estimate; can grow if needed. - - // Build local table of instruction start positions - process( - [&](spv::Op, int start) { instPos.push_back(start); return true; }, - op_fn_nop); - - // Window size for context-sensitive canonicalization values - // Emperical best size from a single data set. TODO: Would be a good tunable. - // We essentially performa a little convolution around each instruction, - // to capture the flavor of nearby code, to hopefully match to similar - // code in other modules. - static const int windowSize = 2; - - for (int entry = 0; entry < int(instPos.size()); ++entry) { - const int start = instPos[entry]; - const spv::Op opCode = asOpCode(start); - - if (opCode == spv::OpFunction) - fnId = asId(start + 2); - - if (opCode == spv::OpFunctionEnd) - fnId = spv::NoResult; - - if (fnId != spv::NoResult) { // if inside a function - const int word = start + (spv::InstructionDesc[opCode].hasType() ? 2 : 1); - const int result = spv::InstructionDesc[opCode].hasResult() ? word : -1; - - if (result > 0) { - const spv::Id resId = asId(result); - std::uint32_t hashval = fnId * 17; // small prime - - for (int i = entry-1; i >= entry-windowSize; --i) { - if (asOpCode(instPos[i]) == spv::OpFunction) - break; - hashval = hashval * 30103 + asOpCodeHash(instPos[i]); // 30103 = semiarbitrary prime - } - - for (int i = entry; i <= entry + windowSize; ++i) { - if (asOpCode(instPos[i]) == spv::OpFunctionEnd) - break; - hashval = hashval * 30103 + asOpCodeHash(instPos[i]); // 30103 = semiarbitrary prime - } - - if (isOldIdUnmapped(resId)) - localId(resId, nextUnusedId(hashval % softTypeIdLimit + firstMappedID)); - } - } - } - - spv::Op thisOpCode(spv::OpNop); - std::unordered_map opCounter; - int idCounter(0); - fnId = spv::NoResult; - - process( - [&](spv::Op opCode, int start) { - switch (opCode) { - case spv::OpFunction: - // Reset counters at each function - idCounter = 0; - opCounter.clear(); - fnId = asId(start + 2); - break; - - case spv::OpTextureSample: - case spv::OpTextureSampleDref: - case spv::OpTextureSampleLod: - case spv::OpTextureSampleProj: - case spv::OpTextureSampleGrad: - case spv::OpTextureSampleOffset: - case spv::OpTextureSampleProjLod: - case spv::OpTextureSampleProjGrad: - case spv::OpTextureSampleLodOffset: - case spv::OpTextureSampleProjOffset: - case spv::OpTextureSampleGradOffset: - case spv::OpTextureSampleProjLodOffset: - case spv::OpTextureSampleProjGradOffset: - case spv::OpDot: - case spv::OpCompositeExtract: - case spv::OpCompositeInsert: - case spv::OpVectorShuffle: - case spv::OpLabel: - case spv::OpVariable: - - case spv::OpAccessChain: - case spv::OpLoad: - case spv::OpStore: - case spv::OpCompositeConstruct: - case spv::OpFunctionCall: - ++opCounter[opCode]; - idCounter = 0; - thisOpCode = opCode; - break; - default: - thisOpCode = spv::OpNop; - } - - return false; - }, - - [&](spv::Id& id) { - if (thisOpCode != spv::OpNop) { - ++idCounter; - const std::uint32_t hashval = opCounter[thisOpCode] * thisOpCode * 50047 + idCounter + fnId * 117; - - if (isOldIdUnmapped(id)) - localId(id, nextUnusedId(hashval % softTypeIdLimit + firstMappedID)); - } - }); -} - -#ifdef NOTDEF -// remove bodies of uncalled functions -void spirvbin_t::offsetIds() -{ - // Count of how many functions each ID appears within - std::unordered_map idFnCount; - std::unordered_map idDefinedLoc; - idset_t idsUsed; // IDs used in a given function - - int instCount = 0; - - // create a count of how many functions each ID is used within - process( - [&](spv::OpCode opCode, int start) { - ++instCount; - - switch (opCode) { - case spv::OpFunction: - for (const auto id : idsUsed) - ++idFnCount[id]; - idsUsed.clear(); - break; - - default: - { - const int word = start + (spv::InstructionDesc[opCode].hasType() ? 2 : 1); - const int result = spv::InstructionDesc[opCode].hasResult() ? word : -1; - - if (result > 0) - idDefinedLoc[asId(result)] = instCount; - } - break; - } - - return false; - }, - - [&](spv::Id& id) { idsUsed.insert(id); }); - - // For each ID defined in exactly one function, replace uses by - // negative offset to definitions in instructions. - - static const int relOffsetLimit = 64; - - instCount = 0; - process([&](spv::OpCode, int) { ++instCount; return false; }, - [&](spv::Id& id) { - if (idFnCount[id] == 1 && (instCount - idDefinedLoc[id]) < relOffsetLimit) - id = idDefinedLoc[id] - instCount; - }); -} -#endif - - -// EXPERIMENTAL: forward IO and uniform load/stores into operands -// This produces invalid Schema-0 SPIRV -void spirvbin_t::forwardLoadStores() -{ - idset_t fnLocalVars; // set of function local vars - idmap_t idMap; // Map of load result IDs to what they load - - // EXPERIMENTAL: Forward input and access chain loads into consumptions - process( - [&](spv::Op opCode, int start) { - // Add inputs and uniforms to the map - if (((opCode == spv::OpVariable && asWordCount(start) == 4) || (opCode == spv::OpVariableArray)) && - (spv[start+3] == spv::StorageClassUniform || - spv[start+3] == spv::StorageClassUniformConstant || - spv[start+3] == spv::StorageClassInput)) - fnLocalVars.insert(asId(start+2)); - - if (opCode == spv::OpAccessChain && fnLocalVars.count(asId(start+3)) > 0) - fnLocalVars.insert(asId(start+2)); - - if (opCode == spv::OpLoad && fnLocalVars.count(asId(start+3)) > 0) { - idMap[asId(start+2)] = asId(start+3); - stripInst(start); - } - - return false; - }, - - [&](spv::Id& id) { if (idMap.find(id) != idMap.end()) id = idMap[id]; } - ); - - // EXPERIMENTAL: Implicit output stores - fnLocalVars.clear(); - idMap.clear(); - - process( - [&](spv::Op opCode, int start) { - // Add inputs and uniforms to the map - if (((opCode == spv::OpVariable && asWordCount(start) == 4) || (opCode == spv::OpVariableArray)) && - (spv[start+3] == spv::StorageClassOutput)) - fnLocalVars.insert(asId(start+2)); - - if (opCode == spv::OpStore && fnLocalVars.count(asId(start+1)) > 0) { - idMap[asId(start+2)] = asId(start+1); - stripInst(start); - } - - return false; - }, - op_fn_nop); - - process( - inst_fn_nop, - [&](spv::Id& id) { if (idMap.find(id) != idMap.end()) id = idMap[id]; } - ); - - strip(); // strip out data we decided to eliminate - buildLocalMaps(); // rebuild ID mapping data -} - -// remove bodies of uncalled functions -void spirvbin_t::optLoadStore() -{ - idset_t fnLocalVars; - // Map of load result IDs to what they load - idmap_t idMap; - - // Find all the function local pointers stored at most once, and not via access chains - process( - [&](spv::Op opCode, int start) { - const int wordCount = asWordCount(start); - - // Add local variables to the map - if ((opCode == spv::OpVariable && spv[start+3] == spv::StorageClassFunction && asWordCount(start) == 4) || - (opCode == spv::OpVariableArray && spv[start+3] == spv::StorageClassFunction)) - fnLocalVars.insert(asId(start+2)); - - // Ignore process vars referenced via access chain - if ((opCode == spv::OpAccessChain || opCode == spv::OpInBoundsAccessChain) && fnLocalVars.count(asId(start+3)) > 0) { - fnLocalVars.erase(asId(start+3)); - idMap.erase(asId(start+3)); - } - - if (opCode == spv::OpLoad && fnLocalVars.count(asId(start+3)) > 0) { - // Avoid loads before stores (TODO: why? Crashes driver, but seems like it shouldn't). - if (idMap.find(asId(start+3)) == idMap.end()) { - fnLocalVars.erase(asId(start+3)); - idMap.erase(asId(start+3)); - } - - // don't do for volatile references - if (wordCount > 4 && (spv[start+4] & spv::MemoryAccessVolatileMask)) { - fnLocalVars.erase(asId(start+3)); - idMap.erase(asId(start+3)); - } - } - - if (opCode == spv::OpStore && fnLocalVars.count(asId(start+1)) > 0) { - if (idMap.find(asId(start+1)) == idMap.end()) { - idMap[asId(start+1)] = asId(start+2); - } else { - // Remove if it has more than one store to the same pointer - fnLocalVars.erase(asId(start+1)); - idMap.erase(asId(start+1)); - } - - // don't do for volatile references - if (wordCount > 3 && (spv[start+3] & spv::MemoryAccessVolatileMask)) { - fnLocalVars.erase(asId(start+3)); - idMap.erase(asId(start+3)); - } - } - - return true; - }, - op_fn_nop); - - process( - [&](spv::Op opCode, int start) { - if (opCode == spv::OpLoad && fnLocalVars.count(asId(start+3)) > 0) - idMap[asId(start+2)] = idMap[asId(start+3)]; - return false; - }, - op_fn_nop); - - // Remove the load/store/variables for the ones we've discovered - process( - [&](spv::Op opCode, int start) { - if ((opCode == spv::OpLoad && fnLocalVars.count(asId(start+3)) > 0) || - (opCode == spv::OpStore && fnLocalVars.count(asId(start+1)) > 0) || - (opCode == spv::OpVariable && fnLocalVars.count(asId(start+2)) > 0)) { - stripInst(start); - return true; - } - - return false; - }, - - [&](spv::Id& id) { if (idMap.find(id) != idMap.end()) id = idMap[id]; } - ); - - - strip(); // strip out data we decided to eliminate - buildLocalMaps(); // rebuild ID mapping data -} - -// remove bodies of uncalled functions -void spirvbin_t::dceFuncs() -{ - msg(3, 2, std::string("Removing Dead Functions: ")); + if (isOldIdUnmapped(id)) + error(std::string("old ID not mapped: ") + std::to_string(id)); - // TODO: There are more efficient ways to do this. - bool changed = true; - - while (changed) { - changed = false; - - for (auto fn = fnPos.begin(); fn != fnPos.end(); ) { - if (fn->first == entryPoint) { // don't DCE away the entry point! - ++fn; - continue; - } - - const auto call_it = fnCalls.find(fn->first); - - if (call_it == fnCalls.end() || call_it->second == 0) { - changed = true; - stripRange.push_back(fn->second); - fnPosDCE.insert(*fn); - - // decrease counts of called functions - process( - [&](spv::Op opCode, int start) { - if (opCode == spv::Op::OpFunctionCall) { - const auto call_it = fnCalls.find(asId(start + 3)); - if (call_it != fnCalls.end()) { - if (--call_it->second <= 0) - fnCalls.erase(call_it); - } - } - - return true; - }, - op_fn_nop, - fn->second.first, - fn->second.second); - - fn = fnPos.erase(fn); - } else ++fn; - } - } -} - -// remove unused function variables + decorations -void spirvbin_t::dceVars() -{ - msg(3, 2, std::string("DCE Vars: ")); + // Track max bound + maxBound = std::max(maxBound, localId(id) + 1); + } - std::unordered_map varUseCount; + bound(maxBound); // reset header ID bound to as big as it now needs to be + } - // Count function variable use - process( - [&](spv::Op opCode, int start) { - if (opCode == spv::OpVariable) { ++varUseCount[asId(start+2)]; return true; } - return false; - }, - - [&](spv::Id& id) { if (varUseCount[id]) ++varUseCount[id]; } - ); - - // Remove single-use function variables + associated decorations and names - process( - [&](spv::Op opCode, int start) { - if ((opCode == spv::OpVariable && varUseCount[asId(start+2)] == 1) || - (opCode == spv::OpDecorate && varUseCount[asId(start+1)] == 1) || - (opCode == spv::OpName && varUseCount[asId(start+1)] == 1)) { - stripInst(start); - } - return true; - }, - op_fn_nop); -} - -// remove unused types -void spirvbin_t::dceTypes() -{ - std::vector isType(bound(), false); - - // for speed, make O(1) way to get to type query (map is log(n)) - for (const auto typeStart : typeConstPos) - isType[asTypeConstId(typeStart)] = true; - - std::unordered_map typeUseCount; - - // Count total type usage - process(inst_fn_nop, - [&](spv::Id& id) { if (isType[id]) ++typeUseCount[id]; } - ); - - // Remove types from deleted code - for (const auto& fn : fnPosDCE) - process(inst_fn_nop, - [&](spv::Id& id) { if (isType[id]) --typeUseCount[id]; }, - fn.second.first, fn.second.second); - - // Remove single reference types - for (const auto typeStart : typeConstPos) { - const spv::Id typeId = asTypeConstId(typeStart); - if (typeUseCount[typeId] == 1) { - --typeUseCount[typeId]; - stripInst(typeStart); - } - } -} - - -#ifdef NOTDEF -bool spirvbin_t::matchType(const spirvbin_t::globaltypes_t& globalTypes, spv::Id lt, spv::Id gt) const -{ - // Find the local type id "lt" and global type id "gt" - const auto lt_it = typeConstPosR.find(lt); - if (lt_it == typeConstPosR.end()) - return false; - - const auto typeStart = lt_it->second; - - // Search for entry in global table - const auto gtype = globalTypes.find(gt); - if (gtype == globalTypes.end()) - return false; - - const auto& gdata = gtype->second; - - // local wordcount and opcode - const int wordCount = asWordCount(typeStart); - const spv::Op opCode = asOpCode(typeStart); - - // no type match if opcodes don't match, or operand count doesn't match - if (opCode != opOpCode(gdata[0]) || wordCount != opWordCount(gdata[0])) - return false; - - const unsigned numOperands = wordCount - 2; // all types have a result - - const auto cmpIdRange = [&](range_t range) { - for (int x=range.first; xsecond; -} - -// Hash types to canonical values. This can return ID collisions (it's a bit -// inevitable): it's up to the caller to handle that gracefully. -std::uint32_t spirvbin_t::hashType(int typeStart) const -{ - const unsigned wordCount = asWordCount(typeStart); - const spv::Op opCode = asOpCode(typeStart); - - switch (opCode) { - case spv::OpTypeVoid: return 0; - case spv::OpTypeBool: return 1; - case spv::OpTypeInt: return 3 + (spv[typeStart+3]); - case spv::OpTypeFloat: return 5; - case spv::OpTypeVector: - return 6 + hashType(typePos(spv[typeStart+2])) * (spv[typeStart+3] - 1); - case spv::OpTypeMatrix: - return 30 + hashType(typePos(spv[typeStart+2])) * (spv[typeStart+3] - 1); - case spv::OpTypeSampler: - return 120 + hashType(typePos(spv[typeStart+2])) + - spv[typeStart+3] + // dimensionality - spv[typeStart+4] * 8 * 16 + // content - spv[typeStart+5] * 4 * 16 + // arrayed - spv[typeStart+6] * 2 * 16 + // compare - spv[typeStart+7] * 1 * 16; // multisampled - case spv::OpTypeFilter: - return 500; - case spv::OpTypeArray: - return 501 + hashType(typePos(spv[typeStart+2])) * spv[typeStart+3]; - case spv::OpTypeRuntimeArray: - return 5000 + hashType(typePos(spv[typeStart+2])); - case spv::OpTypeStruct: - { - std::uint32_t hash = 10000; - for (unsigned w=2; w < wordCount; ++w) - hash += w * hashType(typePos(spv[typeStart+w])); - return hash; - } - - case spv::OpTypeOpaque: return 6000 + spv[typeStart+2]; - case spv::OpTypePointer: return 100000 + hashType(typePos(spv[typeStart+3])); - case spv::OpTypeFunction: - { - std::uint32_t hash = 200000; - for (unsigned w=2; w < wordCount; ++w) - hash += w * hashType(typePos(spv[typeStart+w])); - return hash; - } - - case spv::OpTypeEvent: return 300000; - case spv::OpTypeDeviceEvent: return 300001; - case spv::OpTypeReserveId: return 300002; - case spv::OpTypeQueue: return 300003; - case spv::OpTypePipe: return 300004; - - case spv::OpConstantNullObject: return 300005; - case spv::OpConstantSampler: return 300006; - - case spv::OpConstantTrue: return 300007; - case spv::OpConstantFalse: return 300008; - case spv::OpConstantNullPointer: return 300009; - case spv::OpConstantComposite: - { - std::uint32_t hash = 300011 + hashType(typePos(spv[typeStart+1])); - for (unsigned w=3; w < wordCount; ++w) - hash += w * hashType(typePos(spv[typeStart+w])); - return hash; - } - case spv::OpConstant: - { - std::uint32_t hash = 400011 + hashType(typePos(spv[typeStart+1])); - for (unsigned w=3; w < wordCount; ++w) - hash += w * spv[typeStart+w]; - return hash; - } + // build local Id and name maps + process( + [&](spv::Op opCode, int start) { + // remember opcodes we want to strip later + if (isStripOp(opCode)) + stripInst(start); + return true; + }, + op_fn_nop); + } - default: - error("unknown type opcode"); - return 0; - } -} - -void spirvbin_t::mapTypeConst() -{ - globaltypes_t globalTypeMap; + void spirvbin_t::buildLocalMaps() + { + msg(2, 2, std::string("build local maps: ")); - msg(3, 2, std::string("Remapping Consts & Types: ")); + mapped.clear(); + idMapL.clear(); + nameMap.clear(); + fnPos.clear(); + fnPosDCE.clear(); + fnCalls.clear(); + typeConstPos.clear(); + typeConstPosR.clear(); + entryPoint = spv::NoResult; + largestNewId = 0; - static const std::uint32_t softTypeIdLimit = 3011; // small prime. TODO: get from options - static const std::uint32_t firstMappedID = 8; // offset into ID space - - for (auto& typeStart : typeConstPos) { - const spv::Id resId = asTypeConstId(typeStart); - const std::uint32_t hashval = hashType(typeStart); - - if (isOldIdUnmapped(resId)) - localId(resId, nextUnusedId(hashval % softTypeIdLimit + firstMappedID)); - } -} - - -// Strip a single binary by removing ranges given in stripRange -void spirvbin_t::strip() -{ - if (stripRange.empty()) // nothing to do - return; - - // Sort strip ranges in order of traversal - std::sort(stripRange.begin(), stripRange.end()); - - // Allocate a new binary big enough to hold old binary - // We'll step this iterator through the strip ranges as we go through the binary - decltype(stripRange)::const_iterator strip_it = stripRange.begin(); - - int strippedPos = 0; - for (unsigned word = 0; word < unsigned(spv.size()); ++word) { - if (strip_it != stripRange.end() && word >= strip_it->second) - ++strip_it; - - if (strip_it == stripRange.end() || word < strip_it->first || word >= strip_it->second) - spv[strippedPos++] = spv[word]; - } - - spv.resize(strippedPos); - stripRange.clear(); -} - -// Strip a single binary by removing ranges given in stripRange -void spirvbin_t::remap(std::uint32_t opts) -{ - options = opts; - - validate(); // validate header - buildLocalMaps(); - - msg(3, 4, std::string("ID bound: ") + std::to_string(bound())); - - if (options & OPT_LOADSTORE) optLoadStore(); - if (options & OPT_FWD_LS) forwardLoadStores(); - if (options & DCE_FUNCS) dceFuncs(); - if (options & DCE_VARS) dceVars(); - if (options & DCE_TYPES) dceTypes(); - if (options & MAP_TYPES) mapTypeConst(); - if (options & MAP_NAMES) mapNames(); - if (options & MAP_FUNCS) mapFnBodies(); - // if (options & STRIP) stripDebug(); - - mapRemainder(); // map any unmapped IDs - applyMap(); // Now remap each shader to the new IDs we've come up with - strip(); // strip out data we decided to eliminate - -#define EXPERIMENT3 0 -#if (EXPERIMENT3) -// TODO: ... shortcuts for simple single-const access chains and constants, -// folded into high half of the ID space. -#endif -} - -// remap from a memory image -void spirvbin_t::remap(std::vector& in_spv, std::uint32_t opts) -{ - spv.swap(in_spv); - remap(opts); - spv.swap(in_spv); -} + idMapL.resize(bound(), unused); + + int fnStart = 0; + spv::Id fnRes = spv::NoResult; + + // build local Id and name maps + process( + [&](spv::Op opCode, int start) { + // remember opcodes we want to strip later + if ((options & Options::STRIP) && isStripOp(opCode)) + stripInst(start); + + if (opCode == spv::Op::OpName) { + const spv::Id target = asId(start+1); + const std::string name = literalString(start+2); + nameMap[name] = target; + return true; + } else if (opCode == spv::Op::OpFunctionCall) { + ++fnCalls[asId(start + 3)]; + } else if (opCode == spv::Op::OpEntryPoint) { + entryPoint = asId(start + 2); + } else if (opCode == spv::Op::OpFunction) { + if (fnStart != 0) + error("nested function found"); + fnStart = start; + fnRes = asId(start + 2); + } else if (opCode == spv::Op::OpFunctionEnd) { + assert(fnRes != spv::NoResult); + if (fnStart == 0) + error("function end without function start"); + fnPos[fnRes] = {fnStart, start + asWordCount(start)}; + fnStart = 0; + } else if (isConstOp(opCode)) { + assert(asId(start + 2) != spv::NoResult); + typeConstPos.insert(start); + typeConstPosR[asId(start + 2)] = start; + } else if (isTypeOp(opCode)) { + assert(asId(start + 1) != spv::NoResult); + typeConstPos.insert(start); + typeConstPosR[asId(start + 1)] = start; + } + + return false; + }, + + [this](spv::Id& id) { localId(id, unmapped); } + ); + } + + // Validate the SPIR header + void spirvbin_t::validate() const + { + msg(2, 2, std::string("validating: ")); + + if (spv.size() < header_size) + error("file too short: "); + + if (magic() != spv::MagicNumber) + error("bad magic number"); + + // field 1 = version + // field 2 = generator magic + // field 3 = result bound + + if (schemaNum() != 0) + error("bad schema, must be 0"); + } + + + int spirvbin_t::processInstruction(int word, instfn_t instFn, idfn_t idFn) + { + const auto instructionStart = word; + const unsigned wordCount = asWordCount(instructionStart); + const spv::Op opCode = asOpCode(instructionStart); + const int nextInst = word++ + wordCount; + + if (nextInst > int(spv.size())) + error("spir instruction terminated too early"); + + // Base for computing number of operands; will be updated as more is learned + unsigned numOperands = wordCount - 1; + + if (instFn(opCode, instructionStart)) + return nextInst; + + // Read type and result ID from instruction desc table + if (spv::InstructionDesc[opCode].hasType()) { + idFn(asId(word++)); + --numOperands; + } + + if (spv::InstructionDesc[opCode].hasResult()) { + idFn(asId(word++)); + --numOperands; + } + + // Extended instructions: currently, assume everything is an ID. + // TODO: add whatever data we need for exceptions to that + if (opCode == spv::OpExtInst) { + word += 2; // instruction set, and instruction from set + numOperands -= 2; + + for (unsigned op=0; op < numOperands; ++op) + idFn(asId(word++)); // ID + + return nextInst; + } + + // Store IDs from instruction in our map + for (int op = 0; op < spv::InstructionDesc[opCode].operands.getNum(); ++op, --numOperands) { + switch (spv::InstructionDesc[opCode].operands.getClass(op)) { + case spv::OperandId: + idFn(asId(word++)); + break; + + case spv::OperandOptionalId: + case spv::OperandVariableIds: + for (unsigned i = 0; i < numOperands; ++i) + idFn(asId(word++)); + return nextInst; + + case spv::OperandVariableLiterals: + if (opCode == spv::OpDecorate && asDecoration(word - 1) == spv::DecorationBuiltIn) { + ++word; + --numOperands; + } + word += numOperands; + return nextInst; + + case spv::OperandVariableLiteralId: + while (numOperands > 0) { + ++word; // immediate + idFn(asId(word++)); // ID + numOperands -= 2; + } + return nextInst; + + case spv::OperandLiteralString: + word += literalStringWords(literalString(word)); + return nextInst; + + // Single word operands we simply ignore, as they hold no IDs + case spv::OperandLiteralNumber: + case spv::OperandSource: + case spv::OperandExecutionModel: + case spv::OperandAddressing: + case spv::OperandMemory: + case spv::OperandExecutionMode: + case spv::OperandStorage: + case spv::OperandDimensionality: + case spv::OperandDecoration: + case spv::OperandBuiltIn: + case spv::OperandSelect: + case spv::OperandLoop: + case spv::OperandFunction: + case spv::OperandMemorySemantics: + case spv::OperandMemoryAccess: + case spv::OperandExecutionScope: + case spv::OperandGroupOperation: + case spv::OperandKernelEnqueueFlags: + case spv::OperandKernelProfilingInfo: + ++word; + break; + + default: + break; + } + } + + return nextInst; + } + + // Make a pass over all the instructions and process them given appropriate functions + spirvbin_t& spirvbin_t::process(instfn_t instFn, idfn_t idFn, int begin, int end) + { + // For efficiency, reserve name map space. It can grow if needed. + nameMap.reserve(32); + + // If begin or end == 0, use defaults + begin = (begin == 0 ? header_size : begin); + end = (end == 0 ? int(spv.size()) : end); + + // basic parsing and InstructionDesc table borrowed from SpvDisassemble.cpp... + int nextInst = int(spv.size()); + + for (int word = begin; word < end; word = nextInst) + nextInst = processInstruction(word, instFn, idFn); + + return *this; + } + + // Apply global name mapping to a single module + void spirvbin_t::mapNames() + { + static const std::uint32_t softTypeIdLimit = 3011; // small prime. TODO: get from options + static const std::uint32_t firstMappedID = 3019; // offset into ID space + + for (const auto& name : nameMap) { + std::uint32_t hashval = 1911; + for (const char c : name.first) + hashval = hashval * 1009 + c; + + if (isOldIdUnmapped(name.second)) + localId(name.second, nextUnusedId(hashval % softTypeIdLimit + firstMappedID)); + } + } + + // Map fn contents to IDs of similar functions in other modules + void spirvbin_t::mapFnBodies() + { + static const std::uint32_t softTypeIdLimit = 19071; // small prime. TODO: get from options + static const std::uint32_t firstMappedID = 6203; // offset into ID space + + // Initial approach: go through some high priority opcodes first and assign them + // hash values. + + spv::Id fnId = spv::NoResult; + std::vector instPos; + instPos.reserve(int(spv.size()) / 16); // initial estimate; can grow if needed. + + // Build local table of instruction start positions + process( + [&](spv::Op, int start) { instPos.push_back(start); return true; }, + op_fn_nop); + + // Window size for context-sensitive canonicalization values + // Emperical best size from a single data set. TODO: Would be a good tunable. + // We essentially performa a little convolution around each instruction, + // to capture the flavor of nearby code, to hopefully match to similar + // code in other modules. + static const int windowSize = 2; + + for (int entry = 0; entry < int(instPos.size()); ++entry) { + const int start = instPos[entry]; + const spv::Op opCode = asOpCode(start); + + if (opCode == spv::OpFunction) + fnId = asId(start + 2); + + if (opCode == spv::OpFunctionEnd) + fnId = spv::NoResult; + + if (fnId != spv::NoResult) { // if inside a function + const int word = start + (spv::InstructionDesc[opCode].hasType() ? 2 : 1); + const int result = spv::InstructionDesc[opCode].hasResult() ? word : -1; + + if (result > 0) { + const spv::Id resId = asId(result); + std::uint32_t hashval = fnId * 17; // small prime + + for (int i = entry-1; i >= entry-windowSize; --i) { + if (asOpCode(instPos[i]) == spv::OpFunction) + break; + hashval = hashval * 30103 + asOpCodeHash(instPos[i]); // 30103 = semiarbitrary prime + } + + for (int i = entry; i <= entry + windowSize; ++i) { + if (asOpCode(instPos[i]) == spv::OpFunctionEnd) + break; + hashval = hashval * 30103 + asOpCodeHash(instPos[i]); // 30103 = semiarbitrary prime + } + + if (isOldIdUnmapped(resId)) + localId(resId, nextUnusedId(hashval % softTypeIdLimit + firstMappedID)); + } + } + } + + spv::Op thisOpCode(spv::OpNop); + std::unordered_map opCounter; + int idCounter(0); + fnId = spv::NoResult; + + process( + [&](spv::Op opCode, int start) { + switch (opCode) { + case spv::OpFunction: + // Reset counters at each function + idCounter = 0; + opCounter.clear(); + fnId = asId(start + 2); + break; + + case spv::OpTextureSample: + case spv::OpTextureSampleDref: + case spv::OpTextureSampleLod: + case spv::OpTextureSampleProj: + case spv::OpTextureSampleGrad: + case spv::OpTextureSampleOffset: + case spv::OpTextureSampleProjLod: + case spv::OpTextureSampleProjGrad: + case spv::OpTextureSampleLodOffset: + case spv::OpTextureSampleProjOffset: + case spv::OpTextureSampleGradOffset: + case spv::OpTextureSampleProjLodOffset: + case spv::OpTextureSampleProjGradOffset: + case spv::OpDot: + case spv::OpCompositeExtract: + case spv::OpCompositeInsert: + case spv::OpVectorShuffle: + case spv::OpLabel: + case spv::OpVariable: + + case spv::OpAccessChain: + case spv::OpLoad: + case spv::OpStore: + case spv::OpCompositeConstruct: + case spv::OpFunctionCall: + ++opCounter[opCode]; + idCounter = 0; + thisOpCode = opCode; + break; + default: + thisOpCode = spv::OpNop; + } + + return false; + }, + + [&](spv::Id& id) { + if (thisOpCode != spv::OpNop) { + ++idCounter; + const std::uint32_t hashval = opCounter[thisOpCode] * thisOpCode * 50047 + idCounter + fnId * 117; + + if (isOldIdUnmapped(id)) + localId(id, nextUnusedId(hashval % softTypeIdLimit + firstMappedID)); + } + }); + } + +#ifdef NOTDEF + // remove bodies of uncalled functions + void spirvbin_t::offsetIds() + { + // Count of how many functions each ID appears within + std::unordered_map idFnCount; + std::unordered_map idDefinedLoc; + idset_t idsUsed; // IDs used in a given function + + int instCount = 0; + + // create a count of how many functions each ID is used within + process( + [&](spv::OpCode opCode, int start) { + ++instCount; + + switch (opCode) { + case spv::OpFunction: + for (const auto id : idsUsed) + ++idFnCount[id]; + idsUsed.clear(); + break; + + default: + { + const int word = start + (spv::InstructionDesc[opCode].hasType() ? 2 : 1); + const int result = spv::InstructionDesc[opCode].hasResult() ? word : -1; + + if (result > 0) + idDefinedLoc[asId(result)] = instCount; + } + break; + } + + return false; + }, + + [&](spv::Id& id) { idsUsed.insert(id); }); + + // For each ID defined in exactly one function, replace uses by + // negative offset to definitions in instructions. + + static const int relOffsetLimit = 64; + + instCount = 0; + process([&](spv::OpCode, int) { ++instCount; return false; }, + [&](spv::Id& id) { + if (idFnCount[id] == 1 && (instCount - idDefinedLoc[id]) < relOffsetLimit) + id = idDefinedLoc[id] - instCount; + }); + } +#endif + + + // EXPERIMENTAL: forward IO and uniform load/stores into operands + // This produces invalid Schema-0 SPIRV + void spirvbin_t::forwardLoadStores() + { + idset_t fnLocalVars; // set of function local vars + idmap_t idMap; // Map of load result IDs to what they load + + // EXPERIMENTAL: Forward input and access chain loads into consumptions + process( + [&](spv::Op opCode, int start) { + // Add inputs and uniforms to the map + if (((opCode == spv::OpVariable && asWordCount(start) == 4) || (opCode == spv::OpVariableArray)) && + (spv[start+3] == spv::StorageClassUniform || + spv[start+3] == spv::StorageClassUniformConstant || + spv[start+3] == spv::StorageClassInput)) + fnLocalVars.insert(asId(start+2)); + + if (opCode == spv::OpAccessChain && fnLocalVars.count(asId(start+3)) > 0) + fnLocalVars.insert(asId(start+2)); + + if (opCode == spv::OpLoad && fnLocalVars.count(asId(start+3)) > 0) { + idMap[asId(start+2)] = asId(start+3); + stripInst(start); + } + + return false; + }, + + [&](spv::Id& id) { if (idMap.find(id) != idMap.end()) id = idMap[id]; } + ); + + // EXPERIMENTAL: Implicit output stores + fnLocalVars.clear(); + idMap.clear(); + + process( + [&](spv::Op opCode, int start) { + // Add inputs and uniforms to the map + if (((opCode == spv::OpVariable && asWordCount(start) == 4) || (opCode == spv::OpVariableArray)) && + (spv[start+3] == spv::StorageClassOutput)) + fnLocalVars.insert(asId(start+2)); + + if (opCode == spv::OpStore && fnLocalVars.count(asId(start+1)) > 0) { + idMap[asId(start+2)] = asId(start+1); + stripInst(start); + } + + return false; + }, + op_fn_nop); + + process( + inst_fn_nop, + [&](spv::Id& id) { if (idMap.find(id) != idMap.end()) id = idMap[id]; } + ); + + strip(); // strip out data we decided to eliminate + buildLocalMaps(); // rebuild ID mapping data + } + + // remove bodies of uncalled functions + void spirvbin_t::optLoadStore() + { + idset_t fnLocalVars; + // Map of load result IDs to what they load + idmap_t idMap; + + // Find all the function local pointers stored at most once, and not via access chains + process( + [&](spv::Op opCode, int start) { + const int wordCount = asWordCount(start); + + // Add local variables to the map + if ((opCode == spv::OpVariable && spv[start+3] == spv::StorageClassFunction && asWordCount(start) == 4) || + (opCode == spv::OpVariableArray && spv[start+3] == spv::StorageClassFunction)) + fnLocalVars.insert(asId(start+2)); + + // Ignore process vars referenced via access chain + if ((opCode == spv::OpAccessChain || opCode == spv::OpInBoundsAccessChain) && fnLocalVars.count(asId(start+3)) > 0) { + fnLocalVars.erase(asId(start+3)); + idMap.erase(asId(start+3)); + } + + if (opCode == spv::OpLoad && fnLocalVars.count(asId(start+3)) > 0) { + // Avoid loads before stores (TODO: why? Crashes driver, but seems like it shouldn't). + if (idMap.find(asId(start+3)) == idMap.end()) { + fnLocalVars.erase(asId(start+3)); + idMap.erase(asId(start+3)); + } + + // don't do for volatile references + if (wordCount > 4 && (spv[start+4] & spv::MemoryAccessVolatileMask)) { + fnLocalVars.erase(asId(start+3)); + idMap.erase(asId(start+3)); + } + } + + if (opCode == spv::OpStore && fnLocalVars.count(asId(start+1)) > 0) { + if (idMap.find(asId(start+1)) == idMap.end()) { + idMap[asId(start+1)] = asId(start+2); + } else { + // Remove if it has more than one store to the same pointer + fnLocalVars.erase(asId(start+1)); + idMap.erase(asId(start+1)); + } + + // don't do for volatile references + if (wordCount > 3 && (spv[start+3] & spv::MemoryAccessVolatileMask)) { + fnLocalVars.erase(asId(start+3)); + idMap.erase(asId(start+3)); + } + } + + return true; + }, + op_fn_nop); + + process( + [&](spv::Op opCode, int start) { + if (opCode == spv::OpLoad && fnLocalVars.count(asId(start+3)) > 0) + idMap[asId(start+2)] = idMap[asId(start+3)]; + return false; + }, + op_fn_nop); + + // Remove the load/store/variables for the ones we've discovered + process( + [&](spv::Op opCode, int start) { + if ((opCode == spv::OpLoad && fnLocalVars.count(asId(start+3)) > 0) || + (opCode == spv::OpStore && fnLocalVars.count(asId(start+1)) > 0) || + (opCode == spv::OpVariable && fnLocalVars.count(asId(start+2)) > 0)) { + stripInst(start); + return true; + } + + return false; + }, + + [&](spv::Id& id) { if (idMap.find(id) != idMap.end()) id = idMap[id]; } + ); + + + strip(); // strip out data we decided to eliminate + buildLocalMaps(); // rebuild ID mapping data + } + + // remove bodies of uncalled functions + void spirvbin_t::dceFuncs() + { + msg(3, 2, std::string("Removing Dead Functions: ")); + + // TODO: There are more efficient ways to do this. + bool changed = true; + + while (changed) { + changed = false; + + for (auto fn = fnPos.begin(); fn != fnPos.end(); ) { + if (fn->first == entryPoint) { // don't DCE away the entry point! + ++fn; + continue; + } + + const auto call_it = fnCalls.find(fn->first); + + if (call_it == fnCalls.end() || call_it->second == 0) { + changed = true; + stripRange.push_back(fn->second); + fnPosDCE.insert(*fn); + + // decrease counts of called functions + process( + [&](spv::Op opCode, int start) { + if (opCode == spv::Op::OpFunctionCall) { + const auto call_it = fnCalls.find(asId(start + 3)); + if (call_it != fnCalls.end()) { + if (--call_it->second <= 0) + fnCalls.erase(call_it); + } + } + + return true; + }, + op_fn_nop, + fn->second.first, + fn->second.second); + + fn = fnPos.erase(fn); + } else ++fn; + } + } + } + + // remove unused function variables + decorations + void spirvbin_t::dceVars() + { + msg(3, 2, std::string("DCE Vars: ")); + + std::unordered_map varUseCount; + + // Count function variable use + process( + [&](spv::Op opCode, int start) { + if (opCode == spv::OpVariable) { ++varUseCount[asId(start+2)]; return true; } + return false; + }, + + [&](spv::Id& id) { if (varUseCount[id]) ++varUseCount[id]; } + ); + + // Remove single-use function variables + associated decorations and names + process( + [&](spv::Op opCode, int start) { + if ((opCode == spv::OpVariable && varUseCount[asId(start+2)] == 1) || + (opCode == spv::OpDecorate && varUseCount[asId(start+1)] == 1) || + (opCode == spv::OpName && varUseCount[asId(start+1)] == 1)) { + stripInst(start); + } + return true; + }, + op_fn_nop); + } + + // remove unused types + void spirvbin_t::dceTypes() + { + std::vector isType(bound(), false); + + // for speed, make O(1) way to get to type query (map is log(n)) + for (const auto typeStart : typeConstPos) + isType[asTypeConstId(typeStart)] = true; + + std::unordered_map typeUseCount; + + // Count total type usage + process(inst_fn_nop, + [&](spv::Id& id) { if (isType[id]) ++typeUseCount[id]; } + ); + + // Remove types from deleted code + for (const auto& fn : fnPosDCE) + process(inst_fn_nop, + [&](spv::Id& id) { if (isType[id]) --typeUseCount[id]; }, + fn.second.first, fn.second.second); + + // Remove single reference types + for (const auto typeStart : typeConstPos) { + const spv::Id typeId = asTypeConstId(typeStart); + if (typeUseCount[typeId] == 1) { + --typeUseCount[typeId]; + stripInst(typeStart); + } + } + } + + +#ifdef NOTDEF + bool spirvbin_t::matchType(const spirvbin_t::globaltypes_t& globalTypes, spv::Id lt, spv::Id gt) const + { + // Find the local type id "lt" and global type id "gt" + const auto lt_it = typeConstPosR.find(lt); + if (lt_it == typeConstPosR.end()) + return false; + + const auto typeStart = lt_it->second; + + // Search for entry in global table + const auto gtype = globalTypes.find(gt); + if (gtype == globalTypes.end()) + return false; + + const auto& gdata = gtype->second; + + // local wordcount and opcode + const int wordCount = asWordCount(typeStart); + const spv::Op opCode = asOpCode(typeStart); + + // no type match if opcodes don't match, or operand count doesn't match + if (opCode != opOpCode(gdata[0]) || wordCount != opWordCount(gdata[0])) + return false; + + const unsigned numOperands = wordCount - 2; // all types have a result + + const auto cmpIdRange = [&](range_t range) { + for (int x=range.first; xsecond; + } + + // Hash types to canonical values. This can return ID collisions (it's a bit + // inevitable): it's up to the caller to handle that gracefully. + std::uint32_t spirvbin_t::hashType(int typeStart) const + { + const unsigned wordCount = asWordCount(typeStart); + const spv::Op opCode = asOpCode(typeStart); + + switch (opCode) { + case spv::OpTypeVoid: return 0; + case spv::OpTypeBool: return 1; + case spv::OpTypeInt: return 3 + (spv[typeStart+3]); + case spv::OpTypeFloat: return 5; + case spv::OpTypeVector: + return 6 + hashType(typePos(spv[typeStart+2])) * (spv[typeStart+3] - 1); + case spv::OpTypeMatrix: + return 30 + hashType(typePos(spv[typeStart+2])) * (spv[typeStart+3] - 1); + case spv::OpTypeSampler: + return 120 + hashType(typePos(spv[typeStart+2])) + + spv[typeStart+3] + // dimensionality + spv[typeStart+4] * 8 * 16 + // content + spv[typeStart+5] * 4 * 16 + // arrayed + spv[typeStart+6] * 2 * 16 + // compare + spv[typeStart+7] * 1 * 16; // multisampled + case spv::OpTypeFilter: + return 500; + case spv::OpTypeArray: + return 501 + hashType(typePos(spv[typeStart+2])) * spv[typeStart+3]; + case spv::OpTypeRuntimeArray: + return 5000 + hashType(typePos(spv[typeStart+2])); + case spv::OpTypeStruct: + { + std::uint32_t hash = 10000; + for (unsigned w=2; w < wordCount; ++w) + hash += w * hashType(typePos(spv[typeStart+w])); + return hash; + } + + case spv::OpTypeOpaque: return 6000 + spv[typeStart+2]; + case spv::OpTypePointer: return 100000 + hashType(typePos(spv[typeStart+3])); + case spv::OpTypeFunction: + { + std::uint32_t hash = 200000; + for (unsigned w=2; w < wordCount; ++w) + hash += w * hashType(typePos(spv[typeStart+w])); + return hash; + } + + case spv::OpTypeEvent: return 300000; + case spv::OpTypeDeviceEvent: return 300001; + case spv::OpTypeReserveId: return 300002; + case spv::OpTypeQueue: return 300003; + case spv::OpTypePipe: return 300004; + + case spv::OpConstantNullObject: return 300005; + case spv::OpConstantSampler: return 300006; + + case spv::OpConstantTrue: return 300007; + case spv::OpConstantFalse: return 300008; + case spv::OpConstantNullPointer: return 300009; + case spv::OpConstantComposite: + { + std::uint32_t hash = 300011 + hashType(typePos(spv[typeStart+1])); + for (unsigned w=3; w < wordCount; ++w) + hash += w * hashType(typePos(spv[typeStart+w])); + return hash; + } + case spv::OpConstant: + { + std::uint32_t hash = 400011 + hashType(typePos(spv[typeStart+1])); + for (unsigned w=3; w < wordCount; ++w) + hash += w * spv[typeStart+w]; + return hash; + } + + default: + error("unknown type opcode"); + return 0; + } + } + + void spirvbin_t::mapTypeConst() + { + globaltypes_t globalTypeMap; + + msg(3, 2, std::string("Remapping Consts & Types: ")); + + static const std::uint32_t softTypeIdLimit = 3011; // small prime. TODO: get from options + static const std::uint32_t firstMappedID = 8; // offset into ID space + + for (auto& typeStart : typeConstPos) { + const spv::Id resId = asTypeConstId(typeStart); + const std::uint32_t hashval = hashType(typeStart); + + if (isOldIdUnmapped(resId)) + localId(resId, nextUnusedId(hashval % softTypeIdLimit + firstMappedID)); + } + } + + + // Strip a single binary by removing ranges given in stripRange + void spirvbin_t::strip() + { + if (stripRange.empty()) // nothing to do + return; + + // Sort strip ranges in order of traversal + std::sort(stripRange.begin(), stripRange.end()); + + // Allocate a new binary big enough to hold old binary + // We'll step this iterator through the strip ranges as we go through the binary + decltype(stripRange)::const_iterator strip_it = stripRange.begin(); + + int strippedPos = 0; + for (unsigned word = 0; word < unsigned(spv.size()); ++word) { + if (strip_it != stripRange.end() && word >= strip_it->second) + ++strip_it; + + if (strip_it == stripRange.end() || word < strip_it->first || word >= strip_it->second) + spv[strippedPos++] = spv[word]; + } + + spv.resize(strippedPos); + stripRange.clear(); + } + + // Strip a single binary by removing ranges given in stripRange + void spirvbin_t::remap(std::uint32_t opts) + { + options = opts; + + // Set up opcode tables from SpvDoc + spv::Parameterize(); + + validate(); // validate header + buildLocalMaps(); + + msg(3, 4, std::string("ID bound: ") + std::to_string(bound())); + + if (options & OPT_LOADSTORE) optLoadStore(); + if (options & OPT_FWD_LS) forwardLoadStores(); + if (options & DCE_FUNCS) dceFuncs(); + if (options & DCE_VARS) dceVars(); + if (options & DCE_TYPES) dceTypes(); + if (options & MAP_TYPES) mapTypeConst(); + if (options & MAP_NAMES) mapNames(); + if (options & MAP_FUNCS) mapFnBodies(); + // if (options & STRIP) stripDebug(); + + mapRemainder(); // map any unmapped IDs + applyMap(); // Now remap each shader to the new IDs we've come up with + strip(); // strip out data we decided to eliminate + +#define EXPERIMENT3 0 +#if (EXPERIMENT3) + // TODO: ... shortcuts for simple single-const access chains and constants, + // folded into high half of the ID space. +#endif + } + + // remap from a memory image + void spirvbin_t::remap(std::vector& in_spv, std::uint32_t opts) + { + spv.swap(in_spv); + remap(opts); + spv.swap(in_spv); + } } // namespace SPV - + #endif // defined (use_cpp11) - + diff --git a/StandAlone/CMakeLists.txt b/StandAlone/CMakeLists.txt index 6d291075..a5592c82 100644 --- a/StandAlone/CMakeLists.txt +++ b/StandAlone/CMakeLists.txt @@ -10,8 +10,10 @@ else(WIN32) endif(WIN32) set(SOURCES StandAlone.cpp) +set(REMAPPER_SOURCES spirv-remap.cpp) add_executable(glslangValidator ${SOURCES}) +add_executable(spirv-remap ${REMAPPER_SOURCES}) set(LIBRARIES glslang @@ -26,6 +28,7 @@ elseif(UNIX) endif(WIN32) target_link_libraries(glslangValidator ${LIBRARIES}) +target_link_libraries(spirv-remap ${LIBRARIES}) if(WIN32) source_group("Source" FILES ${SOURCES}) @@ -33,3 +36,6 @@ endif(WIN32) install(TARGETS glslangValidator RUNTIME DESTINATION bin) + +install(TARGETS spirv-remap + RUNTIME DESTINATION bin) diff --git a/StandAlone/spirv-remap.cpp b/StandAlone/spirv-remap.cpp new file mode 100644 index 00000000..e14c8b56 --- /dev/null +++ b/StandAlone/spirv-remap.cpp @@ -0,0 +1,336 @@ +// +//Copyright (C) 2015 LunarG, Inc. +// +//All rights reserved. +// +//Redistribution and use in source and binary forms, with or without +//modification, are permitted provided that the following conditions +//are met: +// +// Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided +// with the distribution. +// +// Neither the name of 3Dlabs Inc. Ltd. nor the names of its +// contributors may be used to endorse or promote products derived +// from this software without specific prior written permission. +// +//THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +//"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +//LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +//FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +//COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +//INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +//BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +//LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +//CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +//LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN +//ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +//POSSIBILITY OF SUCH DAMAGE. +// + +#include +#include +#include +#include + +#include "../SPIRV/SPVRemapper.h" + +namespace { + + typedef unsigned int SpvWord; + + // Poor man's basename: given a complete path, return file portion. + // E.g: + // Linux: /foo/bar/test -> test + // Win: c:\foo\bar\test -> test + // It's not very efficient, but that doesn't matter for our minimal-duty use. + // Using boost::filesystem would be better in many ways, but want to avoid that dependency. + + // OS dependent path separator (avoiding boost::filesystem dependency) +#if defined(_WIN32) + char path_sep_char() { return '\\'; } +#else + char path_sep_char() { return '/'; } +#endif + + std::string basename(const std::string filename) + { + const size_t sepLoc = filename.find_last_of(path_sep_char()); + + return (sepLoc == filename.npos) ? filename : filename.substr(sepLoc+1); + } + + void errHandler(const std::string& str) { + std::cout << str << std::endl; + exit(5); + } + + void logHandler(const std::string& str) { + std::cout << str << std::endl; + } + + // Read word stream from disk + void read(std::vector& spv, const std::string& inFilename) + { + std::ifstream fp; + + std::cout << " reading: " << inFilename << std::endl; + + spv.clear(); + fp.open(inFilename, std::fstream::in | std::fstream::binary); + + if (fp.fail()) + errHandler("error opening file for read: "); + + // Reserve space (for efficiency, not for correctness) + fp.seekg(0, fp.end); + spv.reserve(size_t(fp.tellg()) / sizeof(SpvWord)); + fp.seekg(0, fp.beg); + + while (!fp.eof()) { + SpvWord inWord; + fp.read((char *)&inWord, sizeof(inWord)); + + if (!fp.eof()) { + spv.push_back(inWord); + if (fp.fail()) + errHandler(std::string("error reading file: ") + inFilename); + } + } + } + + void write(std::vector& spv, const std::string& outFile) + { + if (outFile.empty()) + errHandler("missing output filename."); + + std::ofstream fp; + + std::cout << " writing: " << outFile << std::endl; + + fp.open(outFile, std::fstream::out | std::fstream::binary); + + if (fp.fail()) + errHandler(std::string("error opening file for write: ") + outFile); + + for (auto word : spv) { + fp.write((char *)&word, sizeof(word)); + if (fp.fail()) + errHandler(std::string("error writing file: ") + outFile); + } + + // file is closed by destructor + } + + // Print helpful usage message to stdout, and exit + void usage(const char* const name, const char* const msg = 0) + { + if (msg) + std::cout << msg << std::endl << std::endl; + + std::cout << "Usage: " << std::endl; + + std::cout << " " << basename(name) + << " [-v[v[...]] | --verbose [int]]" + << " [--map (all|types|names|funcs)]" + << " [--dce (all|types|funcs)]" + << " [--opt (all|loadstore)]" + << " [--strip-all | --strip all | -s]" + << " [--do-everything]" + << " --input | -i file1 [file2...] --output|-o DESTDIR" + << std::endl; + + std::cout << " " << basename(name) << " [--version | -V]" << std::endl; + std::cout << " " << basename(name) << " [--help | -?]" << std::endl; + + exit(5); + } + + // grind through each SPIR in turn + void execute(const std::vector& inputFile, const std::string& outputDir, + int opts, int verbosity) + { + for (const auto& filename : inputFile) { + std::vector spv; + read(spv, filename); + spv::spirvbin_t(verbosity).remap(spv, opts); + + const std::string outfile = outputDir + path_sep_char() + basename(filename); + + write(spv, outfile); + } + + if (verbosity > 0) + std::cout << "Done: " << inputFile.size() << " file(s) processed" << std::endl; + } + + // Parse command line options + void parseCmdLine(int argc, char** argv, std::vector& inputFile, + std::string& outputDir, + int& options, + int& verbosity) + { + if (argc < 2) + usage(argv[0]); + + verbosity = 0; + options = spv::spirvbin_t::Options::NONE; + + // Parse command line. + // boost::program_options would be quite a bit nicer, but we don't want to + // introduce a dependency on boost. + for (int a=1; a= argc) + usage(argv[0], "--output requires an argument"); + if (!outputDir.empty()) + usage(argv[0], "--output can be provided only once"); + + outputDir = argv[a++]; + + // Remove trailing directory separator characters + while (!outputDir.empty() && outputDir.back() == path_sep_char()) + outputDir.pop_back(); + + } + else if (arg == "-vv") { verbosity = 2; ++a; } // verbosity shortcuts + else if (arg == "-vvv") { verbosity = 3; ++a; } // ... + else if (arg == "-vvvv") { verbosity = 4; ++a; } // ... + else if (arg == "-vvvvv") { verbosity = 5; ++a; } // ... + + else if (arg == "--verbose" || arg == "-v") { + ++a; + verbosity = 1; + + if (a < argc) { + try { + verbosity = std::stoi(argv[a]); + ++a; + } catch (const std::invalid_argument&) { } // ok to have no numeric value + } + } + else if (arg == "--version" || arg == "-V") { + std::cout << basename(argv[0]) << " version 0.97 " << __DATE__ << " " << __TIME__ << std::endl; + exit(0); + } else if (arg == "--input" || arg == "-i") { + // Collect input files + for (++a; a < argc && argv[a][0] != '-'; ++a) + inputFile.push_back(argv[a]); + } else if (arg == "--do-everything") { + ++a; + options = options | spv::spirvbin_t::Options::DO_EVERYTHING; + } else if (arg == "--strip-all" || arg == "-s") { + ++a; + options = options | spv::spirvbin_t::Options::STRIP; + } else if (arg == "--strip") { + ++a; + if (strncmp(argv[a], "all", 3) == 0) { + options = options | spv::spirvbin_t::Options::STRIP; + ++a; + } + } else if (arg == "--dce") { + // Parse comma (or colon, etc) separated list of things to dce + ++a; + for (const char* c = argv[a]; *c; ++c) { + if (strncmp(c, "all", 3) == 0) { + options = (options | spv::spirvbin_t::Options::DCE_ALL); + c += 3; + } else if (strncmp(c, "*", 1) == 0) { + options = (options | spv::spirvbin_t::Options::DCE_ALL); + c += 1; + } else if (strncmp(c, "funcs", 5) == 0) { + options = (options | spv::spirvbin_t::Options::DCE_FUNCS); + c += 5; + } else if (strncmp(c, "types", 5) == 0) { + options = (options | spv::spirvbin_t::Options::DCE_TYPES); + c += 5; + } + } + ++a; + } else if (arg == "--map") { + // Parse comma (or colon, etc) separated list of things to map + ++a; + for (const char* c = argv[a]; *c; ++c) { + if (strncmp(c, "all", 3) == 0) { + options = (options | spv::spirvbin_t::Options::MAP_ALL); + c += 3; + } else if (strncmp(c, "*", 1) == 0) { + options = (options | spv::spirvbin_t::Options::MAP_ALL); + c += 1; + } else if (strncmp(c, "types", 5) == 0) { + options = (options | spv::spirvbin_t::Options::MAP_TYPES); + c += 5; + } else if (strncmp(c, "names", 5) == 0) { + options = (options | spv::spirvbin_t::Options::MAP_NAMES); + c += 5; + } else if (strncmp(c, "funcs", 5) == 0) { + options = (options | spv::spirvbin_t::Options::MAP_FUNCS); + c += 5; + } + } + ++a; + } else if (arg == "--opt") { + ++a; + for (const char* c = argv[a]; *c; ++c) { + if (strncmp(c, "all", 3) == 0) { + options = (options | spv::spirvbin_t::Options::OPT_ALL); + c += 3; + } else if (strncmp(c, "*", 1) == 0) { + options = (options | spv::spirvbin_t::Options::OPT_ALL); + c += 1; + } else if (strncmp(c, "loadstore", 9) == 0) { + options = (options | spv::spirvbin_t::Options::OPT_LOADSTORE); + c += 9; + } + } + ++a; + } else if (arg == "--help" || arg == "-?") { + usage(argv[0]); + } else { + usage(argv[0], "Unknown command line option"); + } + } + } + +} // namespace + + +int main(int argc, char** argv) +{ +#ifdef use_cpp11 + std::vector inputFile; + std::string outputDir; + int opts; + int verbosity; + + // handle errors by exiting + spv::spirvbin_t::registerErrorHandler(errHandler); + + // Log messages to std::cout + spv::spirvbin_t::registerLogHandler(logHandler); + + if (argc < 2) + usage(argv[0]); + + parseCmdLine(argc, argv, inputFile, outputDir, opts, verbosity); + + if (outputDir.empty()) + usage(argv[0], "Output directory required"); + + std::string errmsg; + + // Main operations: read, remap, and write. + execute(inputFile, outputDir, opts, verbosity); + +#endif + + // If we get here, everything went OK! Nothing more to be done. +}