SPIR-V compression: Requires rerunning CMake. Adds a standalone tool for running the SPV compression.
git-svn-id: https://cvs.khronos.org/svn/repos/ogl/trunk/ecosystem/public/sdk/tools/glslang@31232 e7fa87d3-cd2b-0410-9028-fcbf551c1848
This commit is contained in:
John Kessenich
parent
40e391184c
commit
3c4a276282
137
README-spirv-remap.txt
Normal file
137
README-spirv-remap.txt
Normal file
@ -0,0 +1,137 @@
|
||||
|
||||
VERSION
|
||||
--------------------------------------------------------------------------------
|
||||
spirv-remap 0.97
|
||||
|
||||
INTRO:
|
||||
--------------------------------------------------------------------------------
|
||||
spirv-remap is a utility to improve compression of SPIRV binary files via
|
||||
entropy reduction, plus optional stripping of debug information and
|
||||
load/store optimization. It transforms SPIRV to SPIRV, remapping IDs. The
|
||||
resulting modules have an increased ID range (IDs are not as tightly packed
|
||||
around zero), but will compress better when multiple modules are compressed
|
||||
together, since compressor's dictionary can find better cross module
|
||||
commonality.
|
||||
|
||||
Remapping is accomplished via canonicalization. Thus, modules can be
|
||||
compressed one at a time with no loss of quality relative to operating on
|
||||
many modules at once. The command line tool operates on multiple modules
|
||||
only in the trivial repetition sense, for ease of use. The remapper API
|
||||
only accepts a single module at a time.
|
||||
|
||||
There are two modes of use: command line, and a C++11 API. Both are
|
||||
described below.
|
||||
|
||||
spirv-remap is currently in an alpha state. Although there are no known
|
||||
remapping defects, it has only been exercised on one real world game shader
|
||||
workload.
|
||||
|
||||
|
||||
FEEDBACK
|
||||
--------------------------------------------------------------------------------
|
||||
Report defects, enhancements requests, code improvements, etc to:
|
||||
spvremapper@lunarg.com
|
||||
|
||||
|
||||
COMMAND LINE USAGE:
|
||||
--------------------------------------------------------------------------------
|
||||
Examples are given with a verbosity of one (-v), but more verbosity can be
|
||||
had via -vv, -vvv, etc, or an integer parameter to --verbose, such as
|
||||
"--verbose 4". With no verbosity, the command is silent and returns 0 on
|
||||
success, and a positive integer error on failure.
|
||||
|
||||
Pre-built binaries for several OSs are available. Examples presented are
|
||||
for Linux. Command line arguments can be provided in any order.
|
||||
|
||||
1. Basic ID remapping
|
||||
|
||||
Perform ID remapping on all shaders in "*.spv", writing new files with
|
||||
the same basenames to /tmp/out_dir.
|
||||
|
||||
spirv-remap --map all --input *.spv --output /tmp/out_dir
|
||||
|
||||
2. Perform all possible size reductions
|
||||
|
||||
spirv-remap-linux-64 --do-everything --input *.spv --output /tmp/out_dir
|
||||
|
||||
Note that --do-everything is a synonym for:
|
||||
|
||||
--map all --dce all --opt all --strip all
|
||||
|
||||
API USAGE:
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
The public interface to the remapper is defined in SPIRV/SPVRemapper.h as follows:
|
||||
|
||||
namespace spv {
|
||||
|
||||
class spirvbin_t
|
||||
{
|
||||
public:
|
||||
enum Options { ... };
|
||||
spirvbin_t(int verbose = 0); // construct
|
||||
|
||||
// remap an existing binary in memory
|
||||
void remap(std::vector<std::uint32_t>& spv, std::uint32_t opts = Options::DO_EVERYTHING);
|
||||
|
||||
// Type for error/log handler functions
|
||||
typedef std::function<void(const std::string&)> errorfn_t;
|
||||
typedef std::function<void(const std::string&)> logfn_t;
|
||||
|
||||
// Register error/log handling functions (can be c/c++ fn, lambda fn, or functor)
|
||||
static void registerErrorHandler(errorfn_t handler) { errorHandler = handler; }
|
||||
static void registerLogHandler(logfn_t handler) { logHandler = handler; }
|
||||
};
|
||||
|
||||
} // namespace spv
|
||||
|
||||
The class definition is in SPVRemapper.cpp.
|
||||
|
||||
remap() accepts an std::vector of SPIRV words, modifies them per the
|
||||
request given in 'opts', and leaves the 'spv' container with the result.
|
||||
It is safe to instantiate one spirvbin_t per thread and process a different
|
||||
SPIRV in each.
|
||||
|
||||
The "opts" parameter to remap() accepts a bit mask of desired remapping
|
||||
options. See REMAPPING AND OPTIMIZATION OPTIONS.
|
||||
|
||||
On error, the function supplied to registerErrorHandler() will be invoked.
|
||||
This can be a standard C/C++ function, a lambda function, or a functor.
|
||||
The default handler simply calls exit(5); The error handler is a static
|
||||
members, so need only be set up once, not once per spirvbin_t instance.
|
||||
|
||||
Log messages are supplied to registerLogHandler(). By default, log
|
||||
messages are eaten silently. The log handler is also a static member.
|
||||
|
||||
BUILD DEPENDENCIES:
|
||||
--------------------------------------------------------------------------------
|
||||
1. C++11 compatible compiler
|
||||
2. cmake
|
||||
3. glslang
|
||||
|
||||
|
||||
BUILDING
|
||||
--------------------------------------------------------------------------------
|
||||
The standalone remapper is built along side glslangValidator through its
|
||||
normal build process.
|
||||
|
||||
|
||||
REMAPPING AND OPTIMIZATION OPTIONS
|
||||
--------------------------------------------------------------------------------
|
||||
API:
|
||||
These are bits defined under spv::spirvbin_t::Options::, and can be
|
||||
bitwise or-ed together as desired.
|
||||
|
||||
MAP_TYPES = canonicalize type IDs
|
||||
MAP_NAMES = canonicalize named data
|
||||
MAP_FUNCS = canonicalize function bodies
|
||||
DCE_FUNCS = remove dead functions
|
||||
DCE_VARS = remove dead variables
|
||||
DCE_TYPES = remove dead types
|
||||
OPT_LOADSTORE = optimize unneeded load/stores
|
||||
MAP_ALL = (MAP_TYPES | MAP_NAMES | MAP_FUNCS)
|
||||
DCE_ALL = (DCE_FUNCS | DCE_VARS | DCE_TYPES)
|
||||
OPT_ALL = (OPT_LOADSTORE)
|
||||
ALL_BUT_STRIP = (MAP_ALL | DCE_ALL | OPT_ALL)
|
||||
DO_EVERYTHING = (STRIP | ALL_BUT_STRIP)
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@ -10,8 +10,10 @@ else(WIN32)
|
||||
endif(WIN32)
|
||||
|
||||
set(SOURCES StandAlone.cpp)
|
||||
set(REMAPPER_SOURCES spirv-remap.cpp)
|
||||
|
||||
add_executable(glslangValidator ${SOURCES})
|
||||
add_executable(spirv-remap ${REMAPPER_SOURCES})
|
||||
|
||||
set(LIBRARIES
|
||||
glslang
|
||||
@ -26,6 +28,7 @@ elseif(UNIX)
|
||||
endif(WIN32)
|
||||
|
||||
target_link_libraries(glslangValidator ${LIBRARIES})
|
||||
target_link_libraries(spirv-remap ${LIBRARIES})
|
||||
|
||||
if(WIN32)
|
||||
source_group("Source" FILES ${SOURCES})
|
||||
@ -33,3 +36,6 @@ endif(WIN32)
|
||||
|
||||
install(TARGETS glslangValidator
|
||||
RUNTIME DESTINATION bin)
|
||||
|
||||
install(TARGETS spirv-remap
|
||||
RUNTIME DESTINATION bin)
|
||||
|
||||
336
StandAlone/spirv-remap.cpp
Normal file
336
StandAlone/spirv-remap.cpp
Normal file
@ -0,0 +1,336 @@
|
||||
//
|
||||
//Copyright (C) 2015 LunarG, Inc.
|
||||
//
|
||||
//All rights reserved.
|
||||
//
|
||||
//Redistribution and use in source and binary forms, with or without
|
||||
//modification, are permitted provided that the following conditions
|
||||
//are met:
|
||||
//
|
||||
// Redistributions of source code must retain the above copyright
|
||||
// notice, this list of conditions and the following disclaimer.
|
||||
//
|
||||
// Redistributions in binary form must reproduce the above
|
||||
// copyright notice, this list of conditions and the following
|
||||
// disclaimer in the documentation and/or other materials provided
|
||||
// with the distribution.
|
||||
//
|
||||
// Neither the name of 3Dlabs Inc. Ltd. nor the names of its
|
||||
// contributors may be used to endorse or promote products derived
|
||||
// from this software without specific prior written permission.
|
||||
//
|
||||
//THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
//"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
//LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
|
||||
//FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
|
||||
//COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
|
||||
//INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
//BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
||||
//LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
//CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
||||
//LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
|
||||
//ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
//POSSIBILITY OF SUCH DAMAGE.
|
||||
//
|
||||
|
||||
#include <iostream>
|
||||
#include <fstream>
|
||||
#include <cstring>
|
||||
#include <stdexcept>
|
||||
|
||||
#include "../SPIRV/SPVRemapper.h"
|
||||
|
||||
namespace {
|
||||
|
||||
typedef unsigned int SpvWord;
|
||||
|
||||
// Poor man's basename: given a complete path, return file portion.
|
||||
// E.g:
|
||||
// Linux: /foo/bar/test -> test
|
||||
// Win: c:\foo\bar\test -> test
|
||||
// It's not very efficient, but that doesn't matter for our minimal-duty use.
|
||||
// Using boost::filesystem would be better in many ways, but want to avoid that dependency.
|
||||
|
||||
// OS dependent path separator (avoiding boost::filesystem dependency)
|
||||
#if defined(_WIN32)
|
||||
char path_sep_char() { return '\\'; }
|
||||
#else
|
||||
char path_sep_char() { return '/'; }
|
||||
#endif
|
||||
|
||||
std::string basename(const std::string filename)
|
||||
{
|
||||
const size_t sepLoc = filename.find_last_of(path_sep_char());
|
||||
|
||||
return (sepLoc == filename.npos) ? filename : filename.substr(sepLoc+1);
|
||||
}
|
||||
|
||||
void errHandler(const std::string& str) {
|
||||
std::cout << str << std::endl;
|
||||
exit(5);
|
||||
}
|
||||
|
||||
void logHandler(const std::string& str) {
|
||||
std::cout << str << std::endl;
|
||||
}
|
||||
|
||||
// Read word stream from disk
|
||||
void read(std::vector<SpvWord>& spv, const std::string& inFilename)
|
||||
{
|
||||
std::ifstream fp;
|
||||
|
||||
std::cout << " reading: " << inFilename << std::endl;
|
||||
|
||||
spv.clear();
|
||||
fp.open(inFilename, std::fstream::in | std::fstream::binary);
|
||||
|
||||
if (fp.fail())
|
||||
errHandler("error opening file for read: ");
|
||||
|
||||
// Reserve space (for efficiency, not for correctness)
|
||||
fp.seekg(0, fp.end);
|
||||
spv.reserve(size_t(fp.tellg()) / sizeof(SpvWord));
|
||||
fp.seekg(0, fp.beg);
|
||||
|
||||
while (!fp.eof()) {
|
||||
SpvWord inWord;
|
||||
fp.read((char *)&inWord, sizeof(inWord));
|
||||
|
||||
if (!fp.eof()) {
|
||||
spv.push_back(inWord);
|
||||
if (fp.fail())
|
||||
errHandler(std::string("error reading file: ") + inFilename);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void write(std::vector<SpvWord>& spv, const std::string& outFile)
|
||||
{
|
||||
if (outFile.empty())
|
||||
errHandler("missing output filename.");
|
||||
|
||||
std::ofstream fp;
|
||||
|
||||
std::cout << " writing: " << outFile << std::endl;
|
||||
|
||||
fp.open(outFile, std::fstream::out | std::fstream::binary);
|
||||
|
||||
if (fp.fail())
|
||||
errHandler(std::string("error opening file for write: ") + outFile);
|
||||
|
||||
for (auto word : spv) {
|
||||
fp.write((char *)&word, sizeof(word));
|
||||
if (fp.fail())
|
||||
errHandler(std::string("error writing file: ") + outFile);
|
||||
}
|
||||
|
||||
// file is closed by destructor
|
||||
}
|
||||
|
||||
// Print helpful usage message to stdout, and exit
|
||||
void usage(const char* const name, const char* const msg = 0)
|
||||
{
|
||||
if (msg)
|
||||
std::cout << msg << std::endl << std::endl;
|
||||
|
||||
std::cout << "Usage: " << std::endl;
|
||||
|
||||
std::cout << " " << basename(name)
|
||||
<< " [-v[v[...]] | --verbose [int]]"
|
||||
<< " [--map (all|types|names|funcs)]"
|
||||
<< " [--dce (all|types|funcs)]"
|
||||
<< " [--opt (all|loadstore)]"
|
||||
<< " [--strip-all | --strip all | -s]"
|
||||
<< " [--do-everything]"
|
||||
<< " --input | -i file1 [file2...] --output|-o DESTDIR"
|
||||
<< std::endl;
|
||||
|
||||
std::cout << " " << basename(name) << " [--version | -V]" << std::endl;
|
||||
std::cout << " " << basename(name) << " [--help | -?]" << std::endl;
|
||||
|
||||
exit(5);
|
||||
}
|
||||
|
||||
// grind through each SPIR in turn
|
||||
void execute(const std::vector<std::string>& inputFile, const std::string& outputDir,
|
||||
int opts, int verbosity)
|
||||
{
|
||||
for (const auto& filename : inputFile) {
|
||||
std::vector<SpvWord> spv;
|
||||
read(spv, filename);
|
||||
spv::spirvbin_t(verbosity).remap(spv, opts);
|
||||
|
||||
const std::string outfile = outputDir + path_sep_char() + basename(filename);
|
||||
|
||||
write(spv, outfile);
|
||||
}
|
||||
|
||||
if (verbosity > 0)
|
||||
std::cout << "Done: " << inputFile.size() << " file(s) processed" << std::endl;
|
||||
}
|
||||
|
||||
// Parse command line options
|
||||
void parseCmdLine(int argc, char** argv, std::vector<std::string>& inputFile,
|
||||
std::string& outputDir,
|
||||
int& options,
|
||||
int& verbosity)
|
||||
{
|
||||
if (argc < 2)
|
||||
usage(argv[0]);
|
||||
|
||||
verbosity = 0;
|
||||
options = spv::spirvbin_t::Options::NONE;
|
||||
|
||||
// Parse command line.
|
||||
// boost::program_options would be quite a bit nicer, but we don't want to
|
||||
// introduce a dependency on boost.
|
||||
for (int a=1; a<argc; ) {
|
||||
const std::string arg = argv[a];
|
||||
|
||||
if (arg == "--output" || arg == "-o") {
|
||||
// Output directory
|
||||
if (++a >= argc)
|
||||
usage(argv[0], "--output requires an argument");
|
||||
if (!outputDir.empty())
|
||||
usage(argv[0], "--output can be provided only once");
|
||||
|
||||
outputDir = argv[a++];
|
||||
|
||||
// Remove trailing directory separator characters
|
||||
while (!outputDir.empty() && outputDir.back() == path_sep_char())
|
||||
outputDir.pop_back();
|
||||
|
||||
}
|
||||
else if (arg == "-vv") { verbosity = 2; ++a; } // verbosity shortcuts
|
||||
else if (arg == "-vvv") { verbosity = 3; ++a; } // ...
|
||||
else if (arg == "-vvvv") { verbosity = 4; ++a; } // ...
|
||||
else if (arg == "-vvvvv") { verbosity = 5; ++a; } // ...
|
||||
|
||||
else if (arg == "--verbose" || arg == "-v") {
|
||||
++a;
|
||||
verbosity = 1;
|
||||
|
||||
if (a < argc) {
|
||||
try {
|
||||
verbosity = std::stoi(argv[a]);
|
||||
++a;
|
||||
} catch (const std::invalid_argument&) { } // ok to have no numeric value
|
||||
}
|
||||
}
|
||||
else if (arg == "--version" || arg == "-V") {
|
||||
std::cout << basename(argv[0]) << " version 0.97 " << __DATE__ << " " << __TIME__ << std::endl;
|
||||
exit(0);
|
||||
} else if (arg == "--input" || arg == "-i") {
|
||||
// Collect input files
|
||||
for (++a; a < argc && argv[a][0] != '-'; ++a)
|
||||
inputFile.push_back(argv[a]);
|
||||
} else if (arg == "--do-everything") {
|
||||
++a;
|
||||
options = options | spv::spirvbin_t::Options::DO_EVERYTHING;
|
||||
} else if (arg == "--strip-all" || arg == "-s") {
|
||||
++a;
|
||||
options = options | spv::spirvbin_t::Options::STRIP;
|
||||
} else if (arg == "--strip") {
|
||||
++a;
|
||||
if (strncmp(argv[a], "all", 3) == 0) {
|
||||
options = options | spv::spirvbin_t::Options::STRIP;
|
||||
++a;
|
||||
}
|
||||
} else if (arg == "--dce") {
|
||||
// Parse comma (or colon, etc) separated list of things to dce
|
||||
++a;
|
||||
for (const char* c = argv[a]; *c; ++c) {
|
||||
if (strncmp(c, "all", 3) == 0) {
|
||||
options = (options | spv::spirvbin_t::Options::DCE_ALL);
|
||||
c += 3;
|
||||
} else if (strncmp(c, "*", 1) == 0) {
|
||||
options = (options | spv::spirvbin_t::Options::DCE_ALL);
|
||||
c += 1;
|
||||
} else if (strncmp(c, "funcs", 5) == 0) {
|
||||
options = (options | spv::spirvbin_t::Options::DCE_FUNCS);
|
||||
c += 5;
|
||||
} else if (strncmp(c, "types", 5) == 0) {
|
||||
options = (options | spv::spirvbin_t::Options::DCE_TYPES);
|
||||
c += 5;
|
||||
}
|
||||
}
|
||||
++a;
|
||||
} else if (arg == "--map") {
|
||||
// Parse comma (or colon, etc) separated list of things to map
|
||||
++a;
|
||||
for (const char* c = argv[a]; *c; ++c) {
|
||||
if (strncmp(c, "all", 3) == 0) {
|
||||
options = (options | spv::spirvbin_t::Options::MAP_ALL);
|
||||
c += 3;
|
||||
} else if (strncmp(c, "*", 1) == 0) {
|
||||
options = (options | spv::spirvbin_t::Options::MAP_ALL);
|
||||
c += 1;
|
||||
} else if (strncmp(c, "types", 5) == 0) {
|
||||
options = (options | spv::spirvbin_t::Options::MAP_TYPES);
|
||||
c += 5;
|
||||
} else if (strncmp(c, "names", 5) == 0) {
|
||||
options = (options | spv::spirvbin_t::Options::MAP_NAMES);
|
||||
c += 5;
|
||||
} else if (strncmp(c, "funcs", 5) == 0) {
|
||||
options = (options | spv::spirvbin_t::Options::MAP_FUNCS);
|
||||
c += 5;
|
||||
}
|
||||
}
|
||||
++a;
|
||||
} else if (arg == "--opt") {
|
||||
++a;
|
||||
for (const char* c = argv[a]; *c; ++c) {
|
||||
if (strncmp(c, "all", 3) == 0) {
|
||||
options = (options | spv::spirvbin_t::Options::OPT_ALL);
|
||||
c += 3;
|
||||
} else if (strncmp(c, "*", 1) == 0) {
|
||||
options = (options | spv::spirvbin_t::Options::OPT_ALL);
|
||||
c += 1;
|
||||
} else if (strncmp(c, "loadstore", 9) == 0) {
|
||||
options = (options | spv::spirvbin_t::Options::OPT_LOADSTORE);
|
||||
c += 9;
|
||||
}
|
||||
}
|
||||
++a;
|
||||
} else if (arg == "--help" || arg == "-?") {
|
||||
usage(argv[0]);
|
||||
} else {
|
||||
usage(argv[0], "Unknown command line option");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
|
||||
int main(int argc, char** argv)
|
||||
{
|
||||
#ifdef use_cpp11
|
||||
std::vector<std::string> inputFile;
|
||||
std::string outputDir;
|
||||
int opts;
|
||||
int verbosity;
|
||||
|
||||
// handle errors by exiting
|
||||
spv::spirvbin_t::registerErrorHandler(errHandler);
|
||||
|
||||
// Log messages to std::cout
|
||||
spv::spirvbin_t::registerLogHandler(logHandler);
|
||||
|
||||
if (argc < 2)
|
||||
usage(argv[0]);
|
||||
|
||||
parseCmdLine(argc, argv, inputFile, outputDir, opts, verbosity);
|
||||
|
||||
if (outputDir.empty())
|
||||
usage(argv[0], "Output directory required");
|
||||
|
||||
std::string errmsg;
|
||||
|
||||
// Main operations: read, remap, and write.
|
||||
execute(inputFile, outputDir, opts, verbosity);
|
||||
|
||||
#endif
|
||||
|
||||
// If we get here, everything went OK! Nothing more to be done.
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user