Merge pull request #632 from steve-lunarg/structure-splitting

HLSL: inter-stage structure splitting.

hlsl/hlslGrammar.cpp

@@ -478,7 +478,7 @@ bool HlslGrammar::acceptFullySpecifiedType(TType& type)
     // type_specifier
     if (! acceptType(type)) {
         // If this is not a type, we may have inadvertently gone down a wrong path
-        // py parsing "sample", which can be treated like either an identifier or a
+        // by parsing "sample", which can be treated like either an identifier or a
         // qualifier.  Back it out, if we did.
         if (qualifier.sample)
             recedeToken();

hlsl/hlslParseHelper.cpp

@@ -59,9 +59,12 @@ HlslParseContext::HlslParseContext(TSymbolTable& symbolTable, TIntermediate& int
     loopNestingLevel(0), annotationNestingLevel(0), structNestingLevel(0), controlFlowNestingLevel(0),
     postEntryPointReturn(false),
     limits(resources.limits),
+    inEntryPoint(false),
     entryPointOutput(nullptr),
     nextInLocation(0), nextOutLocation(0),
-    sourceEntryPointName(sourceEntryPointName)
+    sourceEntryPointName(sourceEntryPointName),
+    builtInIoIndex(nullptr),
+    builtInIoBase(nullptr)
 {
     globalUniformDefaults.clear();
     globalUniformDefaults.layoutMatrix = ElmRowMajor;
@@ -656,11 +659,13 @@ TIntermTyped* HlslParseContext::handleBracketDereference(const TSourceLoc& loc,
 
         if (base->getAsSymbolNode() && (wasFlattened(base) || shouldFlatten(base->getType()))) {
             if (index->getQualifier().storage != EvqConst)
-                error(loc, "Invalid variable index to flattened uniform array", base->getAsSymbolNode()->getName().c_str(), "");
+                error(loc, "Invalid variable index to flattened array", base->getAsSymbolNode()->getName().c_str(), "");
 
-            result = flattenAccess(loc, base, indexValue);
+            result = flattenAccess(base, indexValue);
             flattened = (result != base);
         } else {
+            splitAccessArray(loc, base, index);
+
             if (index->getQualifier().storage == EvqConst) {
                 if (base->getType().isImplicitlySizedArray())
                     updateImplicitArraySize(loc, base, indexValue);
@@ -765,11 +770,9 @@ TIntermTyped* HlslParseContext::handleDotDereference(const TSourceLoc& loc, TInt
         }
     } else if (field == "Append" ||
                field == "RestartStrip") {
-        // These methods only valid on stage in variables
-        // TODO: ... which are stream out types, if there's any way to test that here.
-        if (base->getType().getQualifier().storage == EvqVaryingOut) {
-            return intermediate.addMethod(base, TType(EbtVoid), &field, loc);
-        }
+        // We cannot check the type here: it may be sanitized if we're not compiling a geometry shader, but
+        // the code is around in the shader source.
+        return intermediate.addMethod(base, TType(EbtVoid), &field, loc);
     }
 
     // It's not .length() if we get to here.
@@ -834,15 +837,21 @@ TIntermTyped* HlslParseContext::handleDotDereference(const TSourceLoc& loc, TInt
             }
         }
         if (fieldFound) {
-            if (base->getAsSymbolNode() && (wasFlattened(base) || shouldFlatten(base->getType())))
-                result = flattenAccess(loc, base, member);
-            else {
-                if (base->getType().getQualifier().storage == EvqConst)
-                    result = intermediate.foldDereference(base, member, loc);
-                else {
-                    TIntermTyped* index = intermediate.addConstantUnion(member, loc);
-                    result = intermediate.addIndex(EOpIndexDirectStruct, base, index, loc);
-                    result->setType(*(*fields)[member].type);
+            if (base->getAsSymbolNode() && (wasFlattened(base) || shouldFlatten(base->getType()))) {
+                result = flattenAccess(base, member);
+            } else {
+                // Update the base and member to access if this was a split structure.
+                result = splitAccessStruct(loc, base, member);
+                fields = base->getType().getStruct();
+
+                if (result == nullptr) {
+                    if (base->getType().getQualifier().storage == EvqConst)
+                        result = intermediate.foldDereference(base, member, loc);
+                    else {
+                        TIntermTyped* index = intermediate.addConstantUnion(member, loc);
+                        result = intermediate.addIndex(EOpIndexDirectStruct, base, index, loc);
+                        result->setType(*(*fields)[member].type);
+                    }
                 }
             }
         } else
@@ -853,6 +862,102 @@ TIntermTyped* HlslParseContext::handleDotDereference(const TSourceLoc& loc, TInt
     return result;
 }
 
+// Determine whether we should split this type
+bool HlslParseContext::shouldSplit(const TType& type)
+{
+    if (! inEntryPoint)
+        return false;
+
+    const TStorageQualifier qualifier = type.getQualifier().storage;
+
+    // If it contains interstage IO, but not ONLY interstage IO, split the struct.
+    return type.isStruct() && type.containsBuiltInInterstageIO() &&
+        (qualifier == EvqVaryingIn || qualifier == EvqVaryingOut);
+}
+
+// Split the type of the given node into two structs:
+//   1. interstage IO
+//   2. everything else
+// IO members are put into the ioStruct.  The type is modified to remove them.
+void HlslParseContext::split(TIntermTyped* node)
+{
+    if (node == nullptr)
+        return;
+
+    TIntermSymbol* symNode = node->getAsSymbolNode();
+
+    if (symNode == nullptr)
+        return;
+
+    // Create a new variable:
+    TType& splitType = split(*symNode->getType().clone(), symNode->getName());
+
+    splitIoVars[symNode->getId()] = makeInternalVariable(symNode->getName(), splitType);
+}
+
+// Split the type of the given variable into two structs:
+void HlslParseContext::split(const TVariable& variable)
+{
+    const TType& type = variable.getType();
+
+    TString name = (&variable == entryPointOutput) ? "" : variable.getName();
+
+    // Create a new variable:
+    TType& splitType = split(*type.clone(), name);
+
+    splitIoVars[variable.getUniqueId()] = makeInternalVariable(variable.getName(), splitType);
+}
+
+// Recursive implementation of split(const TVariable& variable).
+// Returns reference to the modified type.
+TType& HlslParseContext::split(TType& type, TString name, const TType* outerStructType)
+{
+    const TArraySizes* arraySizes = nullptr;
+
+    // At the outer-most scope, remember the struct type so we can examine its storage class
+    // at deeper levels.
+    if (outerStructType == nullptr)
+        outerStructType = &type;
+
+    if (type.isArray())
+        arraySizes = &type.getArraySizes();
+
+    // We can ignore arrayness: it's uninvolved.
+    if (type.isStruct()) {
+        TTypeList* userStructure = type.getWritableStruct();
+ 
+        // Get iterator to (now at end) set of builtin iterstage IO members
+        const auto firstIo = std::stable_partition(userStructure->begin(), userStructure->end(),
+                                                   [](const TTypeLoc& t) {return !t.type->isBuiltInInterstageIO();});
+
+        // Move those to the builtin IO.  However, we also propagate arrayness (just one level is handled
+        // now) to this variable.
+        for (auto ioType = firstIo; ioType != userStructure->end(); ++ioType) {
+            const TType& memberType = *ioType->type;
+            TVariable* ioVar = makeInternalVariable(name + (name.empty() ? "" : ".") + memberType.getFieldName(), memberType);
+
+            if (arraySizes)
+                ioVar->getWritableType().newArraySizes(*arraySizes);
+
+            interstageBuiltInIo[tInterstageIoData(memberType, *outerStructType)] = ioVar;
+
+            // Merge qualifier from the user structure
+            mergeQualifiers(ioVar->getWritableType().getQualifier(), outerStructType->getQualifier());
+        }
+
+        // Erase the IO vars from the user structure.
+        userStructure->erase(firstIo, userStructure->end());
+
+        // Recurse further into the members.
+        for (unsigned int i = 0; i < userStructure->size(); ++i)
+            split(*(*userStructure)[i].type,
+                  name + (name.empty() ? "" : ".") + (*userStructure)[i].type->getFieldName(),
+                  outerStructType);
+    }
+
+    return type;
+}
+
 // Determine whether we should flatten an arbitrary type.
 bool HlslParseContext::shouldFlatten(const TType& type) const
 {
@@ -868,9 +973,11 @@ bool HlslParseContext::shouldFlattenIO(const TType& type) const
 
     const TStorageQualifier qualifier = type.getQualifier().storage;
 
-    return type.isStruct() &&
-           (qualifier == EvqVaryingIn ||
-            qualifier == EvqVaryingOut);
+    if (!type.isStruct())
+        return false;
+
+    return ((language == EShLangVertex   && qualifier == EvqVaryingIn) ||
+            (language == EShLangFragment && qualifier == EvqVaryingOut));
 }
 
 // Is this a uniform array which should be flattened?
@@ -891,7 +998,7 @@ void HlslParseContext::flatten(const TSourceLoc& loc, const TVariable& variable)
     // emplace gives back a pair whose .first is an iterator to the item...
     auto entry = flattenMap.emplace(variable.getUniqueId(), 
                                     TFlattenData(type.getQualifier().layoutBinding));
-        
+
     // ... and the item is a map pair, so first->second is the TFlattenData itself.
     flatten(loc, variable, type, entry.first->second, "");
 }
@@ -926,13 +1033,6 @@ void HlslParseContext::flatten(const TSourceLoc& loc, const TVariable& variable)
 int HlslParseContext::flatten(const TSourceLoc& loc, const TVariable& variable, const TType& type,
                               TFlattenData& flattenData, TString name)
 {
-    // TODO: when struct splitting is in place we can remove this restriction.
-    if (language == EShLangGeometry) {
-        const TType derefType(type, 0);
-        if (!isFinalFlattening(derefType) && type.getQualifier().storage == EvqVaryingIn)
-            error(loc, "recursive type not yet supported in GS input", variable.getName().c_str(), "");
-    }
-
     // If something is an arrayed struct, the array flattener will recursively call flatten()
     // to then flatten the struct, so this is an "if else": we don't do both.
     if (type.isArray())
@@ -953,7 +1053,7 @@ int HlslParseContext::addFlattenedMember(const TSourceLoc& loc,
 {
     if (isFinalFlattening(type)) {
         // This is as far as we flatten.  Insert the variable.
-        TVariable* memberVariable = makeInternalVariable(memberName.c_str(), type);
+        TVariable* memberVariable = makeInternalVariable(memberName, type);
         mergeQualifiers(memberVariable->getWritableType().getQualifier(), variable.getType().getQualifier());
 
         if (flattenData.nextBinding != TQualifier::layoutBindingEnd)
@@ -1043,16 +1143,22 @@ int HlslParseContext::flattenArray(const TSourceLoc& loc, const TVariable& varia
 // Return true if we have flattened this node.
 bool HlslParseContext::wasFlattened(const TIntermTyped* node) const
 {
-    return node != nullptr &&
-        node->getAsSymbolNode() != nullptr &&
+    return node != nullptr && node->getAsSymbolNode() != nullptr &&
         wasFlattened(node->getAsSymbolNode()->getId());
 }
 
+// Return true if we have split this structure
+bool HlslParseContext::wasSplit(const TIntermTyped* node) const
+{
+    return node != nullptr && node->getAsSymbolNode() != nullptr &&
+        wasSplit(node->getAsSymbolNode()->getId());
+}
+
 
 // Turn an access into an aggregate that was flattened to instead be
 // an access to the individual variable the member was flattened to.
 // Assumes shouldFlatten() or equivalent was called first.
-TIntermTyped* HlslParseContext::flattenAccess(const TSourceLoc&, TIntermTyped* base, int member)
+TIntermTyped* HlslParseContext::flattenAccess(TIntermTyped* base, int member)
 {
     const TType dereferencedType(base->getType(), member);  // dereferenced type
 
@@ -1078,6 +1184,115 @@ TIntermTyped* HlslParseContext::flattenAccess(const TSourceLoc&, TIntermTyped* b
     }
 }
 
+// Find and return the split IO TVariable for id, or nullptr if none.
+TVariable* HlslParseContext::getSplitIoVar(int id) const
+{
+    const auto splitIoVar = splitIoVars.find(id);
+
+    if (splitIoVar == splitIoVars.end())
+        return nullptr;
+
+    return splitIoVar->second;
+}
+
+// Find and return the split IO TVariable for variable, or nullptr if none.
+TVariable* HlslParseContext::getSplitIoVar(const TVariable* var) const
+{
+    if (var == nullptr)
+        return nullptr;
+
+    return getSplitIoVar(var->getUniqueId());
+}
+
+// Find and return the split IO TVariable for symbol in this node, or nullptr if none.
+TVariable* HlslParseContext::getSplitIoVar(const TIntermTyped* node) const
+{
+    if (node == nullptr)
+        return nullptr;
+
+    const TIntermSymbol* symbolNode = node->getAsSymbolNode();
+
+    if (symbolNode == nullptr)
+        return nullptr;
+
+    return getSplitIoVar(symbolNode->getId());
+}
+
+// Remember the index used to dereference into this structure, in case it has to be moved to a
+// split-off builtin IO member.
+void HlslParseContext::splitAccessArray(const TSourceLoc& loc, TIntermTyped* base, TIntermTyped* index)
+{
+    const TVariable* splitIoVar = getSplitIoVar(base);
+
+    // Not a split structure
+    if (splitIoVar == nullptr)
+        return;
+
+    if (builtInIoBase) {
+        error(loc, "only one array dimension supported for builtIn IO variable", "", "");
+        return;
+    }
+
+    builtInIoBase  = base;
+    builtInIoIndex = index;
+}
+
+
+// Turn an access into an struct that was split to instead be an
+// access to either the modified structure, or a direct reference to
+// one of the split member variables.
+TIntermTyped* HlslParseContext::splitAccessStruct(const TSourceLoc& loc, TIntermTyped*& base, int& member)
+{
+    // nothing to do
+    if (base == nullptr)
+        return nullptr;
+
+    // We have a pending bracket reference to an outer struct that we may want to move to an inner member.
+    if (builtInIoBase)
+        base = builtInIoBase;
+
+    const TVariable* splitIoVar = getSplitIoVar(base);
+
+    if (splitIoVar == nullptr)
+        return nullptr;
+
+    const TTypeList& members = *base->getType().getStruct();
+
+    const TType& memberType = *members[member].type;
+
+    if (memberType.isBuiltInInterstageIO()) {
+        // It's one of the interstage IO variables we split off.
+        TIntermTyped* builtIn = intermediate.addSymbol(*interstageBuiltInIo[tInterstageIoData(memberType, base->getType())], loc);
+
+        // If there's an array reference to an outer split struct, we re-apply it here.
+        if (builtInIoIndex != nullptr) {
+            if (builtInIoIndex->getQualifier().storage == EvqConst)
+                builtIn = intermediate.addIndex(EOpIndexDirect, builtIn, builtInIoIndex, loc);
+            else
+                builtIn = intermediate.addIndex(EOpIndexIndirect, builtIn, builtInIoIndex, loc);
+
+            builtIn->setType(memberType);
+
+            builtInIoIndex = nullptr;
+            builtInIoBase  = nullptr;
+        }
+
+        return builtIn;
+    } else {
+        // It's not an IO variable.  Find the equivalent index into the new variable.
+        base = intermediate.addSymbol(*splitIoVar, loc);
+
+        int newMember = 0;
+        for (int m=0; m<member; ++m)
+            if (!members[m].type->isBuiltInInterstageIO())
+                ++newMember;
+
+        member = newMember;
+
+        return nullptr;
+    }
+}
+
 // Variables that correspond to the user-interface in and out of a stage
 // (not the built-in interface) are assigned locations and
 // registered as a linkage node (part of the stage's external interface).
@@ -1105,8 +1320,17 @@ void HlslParseContext::assignLocations(TVariable& variable)
         auto& memberList = flattenMap[variable.getUniqueId()].members;
         for (auto member = memberList.begin(); member != memberList.end(); ++member)
             assignLocation(**member);
-    } else
+    } else if (wasSplit(variable.getUniqueId())) {
+        TVariable* splitIoVar = getSplitIoVar(&variable);
+        const TTypeList* structure = splitIoVar->getType().getStruct();
+        // Struct splitting can produce empty structures if the only members of the
+        // struct were builtin interstage IO types.  Only assign locations if it
+        // isn't a struct, or is a non-empty struct.
+        if (structure == nullptr || !structure->empty())
+            assignLocation(*splitIoVar);
+    } else {
         assignLocation(variable);
+    }
 }
 
 //
@@ -1150,6 +1374,23 @@ TFunction& HlslParseContext::handleFunctionDeclarator(const TSourceLoc& loc, TFu
     return function;
 }
 
+
+// Add interstage IO variables to the linkage in canonical order.
+void HlslParseContext::addInterstageIoToLinkage()
+{
+    std::vector<tInterstageIoData> io;
+    io.reserve(interstageBuiltInIo.size());
+
+    for (auto ioVar = interstageBuiltInIo.begin(); ioVar != interstageBuiltInIo.end(); ++ioVar)
+        io.push_back(ioVar->first);
+
+    // Our canonical order is the TBuiltInVariable numeric order.
+    std::sort(io.begin(), io.end());
+
+    for (int idx = 0; idx < int(io.size()); ++idx)
+        trackLinkageDeferred(*interstageBuiltInIo[io[idx]]);
+}
+
 //
 // Handle seeing the function prototype in front of a function definition in the grammar.  
 // The body is handled after this function returns.
@@ -1188,6 +1429,8 @@ TIntermAggregate* HlslParseContext::handleFunctionDefinition(const TSourceLoc& l
         if (entryPointOutput) {
             if (shouldFlatten(entryPointOutput->getType()))
                 flatten(loc, *entryPointOutput);
+            if (shouldSplit(entryPointOutput->getType()))
+                split(*entryPointOutput);
             assignLocations(*entryPointOutput);
         }
     } else
@@ -1215,7 +1458,15 @@ TIntermAggregate* HlslParseContext::handleFunctionDefinition(const TSourceLoc& l
     for (int i = 0; i < function.getParamCount(); i++) {
         TParameter& param = function[i];
         if (param.name != nullptr) {
-            TVariable *variable = new TVariable(param.name, *param.type);
+            TType* sanitizedType;
+
+            // If we're not in the entry point, parameters are sanitized types.
+            if (inEntryPoint)
+                sanitizedType = param.type;
+            else
+                sanitizedType = sanitizeType(param.type);
+
+            TVariable *variable = new TVariable(param.name, *sanitizedType);
 
             // Insert the parameters with name in the symbol table.
             if (! symbolTable.insert(*variable))
@@ -1225,6 +1476,8 @@ TIntermAggregate* HlslParseContext::handleFunctionDefinition(const TSourceLoc& l
                 if (inEntryPoint) {
                     if (shouldFlatten(*param.type))
                         flatten(loc, *variable);
+                    if (shouldSplit(*param.type))
+                        split(*variable);
                     assignLocations(*variable);
                 }
 
@@ -1239,6 +1492,7 @@ TIntermAggregate* HlslParseContext::handleFunctionDefinition(const TSourceLoc& l
         } else
             paramNodes = intermediate.growAggregate(paramNodes, intermediate.addSymbol(*param.type, loc), loc);
     }
+
     intermediate.setAggregateOperator(paramNodes, EOpParameters, TType(EbtVoid), loc);
     loopNestingLevel = 0;
     controlFlowNestingLevel = 0;
@@ -1373,10 +1627,12 @@ TIntermNode* HlslParseContext::handleReturnValue(const TSourceLoc& loc, TIntermT
         return intermediate.addBranch(EOpReturn, value, loc);
 }
 
-void HlslParseContext::handleFunctionArgument(TFunction* function, TIntermTyped*& arguments, TIntermTyped* newArg)
+void HlslParseContext::handleFunctionArgument(TFunction* function,
+                                              TIntermTyped*& arguments, TIntermTyped* newArg)
 {
     TParameter param = { 0, new TType, nullptr };
     param.type->shallowCopy(newArg->getType());
+
     function->addParameter(param);
     if (arguments)
         arguments = intermediate.growAggregate(arguments, newArg);
@@ -1392,14 +1648,15 @@ TIntermTyped* HlslParseContext::handleAssign(const TSourceLoc& loc, TOperator op
     if (left == nullptr || right == nullptr)
         return nullptr;
 
-    const auto mustFlatten = [&](const TIntermTyped& node) {
-        return wasFlattened(&node) && node.getAsSymbolNode() &&
-               flattenMap.find(node.getAsSymbolNode()->getId()) != flattenMap.end();
-    };
+    const bool isSplitLeft    = wasSplit(left);
+    const bool isSplitRight   = wasSplit(right);
 
-    const bool flattenLeft = mustFlatten(*left);
-    const bool flattenRight = mustFlatten(*right);
-    if (! flattenLeft && ! flattenRight)
+    const bool isFlattenLeft  = wasFlattened(left);
+    const bool isFlattenRight = wasFlattened(right);
+
+    // OK to do a single assign if both are split, or both are unsplit.  But if one is and the other
+    // isn't, we fall back to a memberwise copy.
+    if (! isFlattenLeft && ! isFlattenRight && !isSplitLeft && !isSplitRight)
         return intermediate.addAssign(op, left, right, loc);
 
     TIntermAggregate* assignList = nullptr;
@@ -1425,10 +1682,10 @@ TIntermTyped* HlslParseContext::handleAssign(const TSourceLoc& loc, TOperator op
     if (left->getType().isArray())
         memberCount = left->getType().getCumulativeArraySize();
 
-    if (flattenLeft)
+    if (isFlattenLeft)
         leftVariables = &flattenMap.find(left->getAsSymbolNode()->getId())->second.members;
 
-    if (flattenRight) {
+    if (isFlattenRight) {
         rightVariables = &flattenMap.find(right->getAsSymbolNode()->getId())->second.members;
     } else {
         // The RHS is not flattened.  There are several cases:
@@ -1456,73 +1713,118 @@ TIntermTyped* HlslParseContext::handleAssign(const TSourceLoc& loc, TOperator op
 
     int memberIdx = 0;
 
-    const auto getMember = [&](bool flatten, TIntermTyped* node,
-                               const TVector<TVariable*>& memberVariables, int member,
-                               TOperator op, const TType& memberType) -> TIntermTyped * {
+    // We track the outer-most aggregate, so that we can use its storage class later.
+    const TIntermTyped* outerLeft  = left;
+    const TIntermTyped* outerRight = right;
+
+    const auto getMember = [&](bool isLeft, TIntermTyped* node, int member, TIntermTyped* splitNode, int splitMember) -> TIntermTyped * {
         TIntermTyped* subTree;
-        if (flatten && isFinalFlattening(memberType)) {
-            subTree = intermediate.addSymbol(*memberVariables[memberIdx++]);
+
+        const bool flattened      = isLeft ? isFlattenLeft : isFlattenRight;
+        const bool split          = isLeft ? isSplitLeft : isSplitRight;
+        const TIntermTyped* outer = isLeft ? outerLeft   : outerRight;
+        const TVector<TVariable*>& flatVariables      = isLeft ? *leftVariables : *rightVariables;
+        const TOperator op = node->getType().isArray() ? EOpIndexDirect : EOpIndexDirectStruct;
+        const TType derefType(node->getType(), member);
+
+        if (split && derefType.isBuiltInInterstageIO()) {
+            // copy from interstage IO builtin if needed
+            subTree = intermediate.addSymbol(*interstageBuiltInIo.find(tInterstageIoData(derefType, outer->getType()))->second);
+        } else if (flattened && isFinalFlattening(derefType)) {
+            subTree = intermediate.addSymbol(*flatVariables[memberIdx++]);
         } else {
-            subTree = intermediate.addIndex(op, node, intermediate.addConstantUnion(member, loc), loc);
-            subTree->setType(memberType);
+            const TType splitDerefType(splitNode->getType(), splitMember);
+
+            subTree = intermediate.addIndex(op, splitNode, intermediate.addConstantUnion(splitMember, loc), loc);
+            subTree->setType(splitDerefType);
         }
 
         return subTree;
     };
 
-    // Cannot use auto here, because this is recursive, and auto can't work out the type without seeing the
-    // whole thing.  So, we'll resort to an explicit type via std::function.
-    const std::function<void(TIntermTyped* left, TIntermTyped* right)>
-    traverse = [&](TIntermTyped* left, TIntermTyped* right) -> void {
-        // If we get here, we are assigning to or from a whole array or struct that must be
-        // flattened, so have to do member-by-member assignment:
-
-        if (left->getType().isArray()) {
-            // array case
-            const TType dereferencedType(left->getType(), 0);
-
-            for (int element=0; element < left->getType().getOuterArraySize(); ++element) {
-                    // Add a new AST symbol node if we have a temp variable holding a complex RHS.
-                TIntermTyped* subRight = getMember(flattenRight, right, *rightVariables, element,
-                                                   EOpIndexDirect, dereferencedType);
-                TIntermTyped* subLeft = getMember(flattenLeft, left, *leftVariables, element,
-                                                  EOpIndexDirect, dereferencedType);
-
-                if (isFinalFlattening(dereferencedType))
-                    assignList = intermediate.growAggregate(assignList, intermediate.addAssign(op, subLeft, subRight, loc), loc);
-                else
-                    traverse(subLeft, subRight);
-            }
-        } else if (left->getType().isStruct()) {
-            // struct case
-            const auto& members = *left->getType().getStruct();
-
-            for (int member = 0; member < (int)members.size(); ++member) {
-                TIntermTyped* subRight = getMember(flattenRight, right, *rightVariables, member,
-                                                   EOpIndexDirectStruct, *members[member].type);
-                TIntermTyped* subLeft = getMember(flattenLeft, left, *leftVariables, member,
-                                                  EOpIndexDirectStruct, *members[member].type);
-
-                if (isFinalFlattening(*members[member].type))
-                    assignList = intermediate.growAggregate(assignList, intermediate.addAssign(op, subLeft, subRight, loc), loc);
-                else
-                    traverse(subLeft, subRight);
-            }
-        } else {
-            assert(0);  // we should never be called on a non-flattenable thing, because
-                        // that case bails out above to a simple copy.
-        }
-
-    };
-
     // Use the proper RHS node: a new symbol from a TVariable, copy
     // of an TIntermSymbol node, or sometimes the right node directly.
     right = rhsTempVar   ? intermediate.addSymbol(*rhsTempVar, loc) :
             cloneSymNode ? intermediate.addSymbol(*cloneSymNode) :
             right;
 
+    // Cannot use auto here, because this is recursive, and auto can't work out the type without seeing the
+    // whole thing.  So, we'll resort to an explicit type via std::function.
+    const std::function<void(TIntermTyped* left, TIntermTyped* right, TIntermTyped* splitLeft, TIntermTyped* splitRight)>
+    traverse = [&](TIntermTyped* left, TIntermTyped* right, TIntermTyped* splitLeft, TIntermTyped* splitRight) -> void {
+        // If we get here, we are assigning to or from a whole array or struct that must be
+        // flattened, so have to do member-by-member assignment:
+
+        if (left->getType().isArray()) {
+            const TType dereferencedType(left->getType(), 0);
+
+            // array case
+            for (int element=0; element < left->getType().getOuterArraySize(); ++element) {
+                    // Add a new AST symbol node if we have a temp variable holding a complex RHS.
+                TIntermTyped* subLeft  = getMember(true,  left,  element, left, element);
+                TIntermTyped* subRight = getMember(false, right, element, right, element);
+
+                if (isFinalFlattening(dereferencedType))
+                    assignList = intermediate.growAggregate(assignList, intermediate.addAssign(op, subLeft, subRight, loc), loc);
+                else
+                    traverse(subLeft, subRight, splitLeft, splitRight);
+            }
+        } else if (left->getType().isStruct()) {
+            // struct case
+            const auto& membersL = *left->getType().getStruct();
+            const auto& membersR = *right->getType().getStruct();
+
+            // These track the members in the split structures corresponding to the same in the unsplit structures,
+            // which we traverse in parallel.
+            int memberL = 0;
+            int memberR = 0;
+
+            for (int member = 0; member < int(membersL.size()); ++member) {
+                const TType& typeL = *membersL[member].type;
+                const TType& typeR = *membersR[member].type;
+
+                TIntermTyped* subLeft  = getMember(true,  left, member, left, member);
+                TIntermTyped* subRight = getMember(false, right, member, right, member);
+
+                // If there is no splitting, use the same values to avoid inefficiency.
+                TIntermTyped* subSplitLeft =  isSplitLeft  ? getMember(true,  left,  member, splitLeft, memberL) : subLeft;
+                TIntermTyped* subSplitRight = isSplitRight ? getMember(false, right, member, splitRight, memberR) : subRight;
+
+                // If this is the final flattening (no nested types below to flatten) we'll copy the member, else
+                // recurse into the type hierarchy.  However, if splitting the struct, that means we can copy a whole
+                // subtree here IFF it does not itself contain any interstage built-in IO variables, so we only have to
+                // recurse into it if there's something for splitting to do.  That can save a lot of AST verbosity for
+                // a bunch of memberwise copies.
+                if (isFinalFlattening(typeL) || (!isFlattenLeft && !isFlattenRight && 
+                                                 !typeL.containsBuiltInInterstageIO() && !typeR.containsBuiltInInterstageIO())) {
+                    assignList = intermediate.growAggregate(assignList, intermediate.addAssign(op, subSplitLeft, subSplitRight, loc), loc);
+                } else {
+                    traverse(subLeft, subRight, subSplitLeft, subSplitRight);
+                }
+
+                memberL += (typeL.isBuiltInInterstageIO() ? 0 : 1);
+                memberR += (typeR.isBuiltInInterstageIO() ? 0 : 1);
+            }
+        } else {
+            assert(0);  // we should never be called on a non-flattenable thing, because
+                        // that case bails out above to a simple copy.
+        }
+
+    };
+
+    TIntermTyped* splitLeft  = left;
+    TIntermTyped* splitRight = right;
+
+    // If either left or right was a split structure, we must read or write it, but still have to
+    // parallel-recurse through the unsplit structure to identify the builtin IO vars.
+    if (isSplitLeft)
+        splitLeft = intermediate.addSymbol(*getSplitIoVar(left), loc);
+
+    if (isSplitRight)
+        splitRight = intermediate.addSymbol(*getSplitIoVar(right), loc);
+
     // This makes the whole assignment, recursing through subtypes as needed.
-    traverse(left, right);
+    traverse(left, right, splitLeft, splitRight);
 
     assert(assignList != nullptr);
     assignList->setOperator(EOpSequence);
@@ -2214,9 +2516,9 @@ void HlslParseContext::decomposeGeometryMethods(const TSourceLoc& loc, TIntermTy
             emit->setType(TType(EbtVoid));
 
             sequence = intermediate.growAggregate(sequence,
-                                                  intermediate.addAssign(EOpAssign, 
-                                                                         argAggregate->getSequence()[0]->getAsTyped(),
-                                                                         argAggregate->getSequence()[1]->getAsTyped(), loc),
+                                                  handleAssign(loc, EOpAssign, 
+                                                               argAggregate->getSequence()[0]->getAsTyped(),
+                                                               argAggregate->getSequence()[1]->getAsTyped()),
                                                   loc);
 
             sequence = intermediate.growAggregate(sequence, emit);
@@ -2840,7 +3142,7 @@ void HlslParseContext::addInputArgumentConversions(const TFunction& function, TI
             else
                 error(arg->getLoc(), "cannot convert input argument, argument", "", "%d", i);
         } else {
-            if (wasFlattened(arg)) {
+            if (wasFlattened(arg) || wasSplit(arg)) {
                 // Will make a two-level subtree.
                 // The deepest will copy member-by-member to build the structure to pass.
                 // The level above that will be a two-operand EOpComma sequence that follows the copy by the
@@ -4781,6 +5083,43 @@ void HlslParseContext::declareTypedef(const TSourceLoc& loc, TString& identifier
         error(loc, "name already defined", "typedef", identifier.c_str());
 }
 
+// Type sanitization: return existing sanitized (temporary) type if there is one, else make new one.
+TType* HlslParseContext::sanitizeType(TType* type)
+{
+    // We only do this for structs.
+    if (!type->isStruct())
+        return type;
+
+    // Type sanitization: if this is declaring a variable of a type that contains
+    // interstage IO, we want to make it a temporary.
+    const auto sanitizedTypeIter = sanitizedTypeMap.find(type->getStruct());
+
+    if (sanitizedTypeIter != sanitizedTypeMap.end()) {
+        // We've sanitized this before.  Use that one.
+        TType* sanitizedType = new TType();
+        sanitizedType->shallowCopy(*sanitizedTypeIter->second);
+
+        // Arrayness is not part of the sanitized type.  Use the input type's arrayness.
+        if (type->isArray())
+            sanitizedType->newArraySizes(type->getArraySizes());
+        else
+            sanitizedType->clearArraySizes();
+        return sanitizedType;
+    } else {
+        if (type->containsBuiltInInterstageIO()) {
+            // This means the type contains interstage IO, but we've never encountered it before.
+            // Copy it, sanitize it, and remember it in the sanitizedTypeMap
+            TType* sanitizedType = type->clone();
+            sanitizedType->makeTemporary();
+            sanitizedTypeMap[type->getStruct()] = sanitizedType;
+            return sanitizedType;
+        } else {
+            // This means the type has no interstage IO, so we can use it as is.
+            return type;
+        }
+    }
+}
+
 //
 // Do everything necessary to handle a variable (non-block) declaration.
 // Either redeclaring a variable, or making a new one, updating the symbol
@@ -4806,15 +5145,20 @@ TIntermNode* HlslParseContext::declareVariable(const TSourceLoc& loc, TString& i
     inheritGlobalDefaults(type.getQualifier());
 
     const bool flattenVar = shouldFlatten(type);
+    const bool splitVar   = shouldSplit(type);
+
+    // Type sanitization: if this is declaring a variable of a type that contains
+    // interstage IO, we want to make it a temporary.
+    TType* sanitizedType = sanitizeType(&type);
 
     // Declare the variable
     if (type.isArray()) {
         // array case
-        declareArray(loc, identifier, type, symbol, !flattenVar);
+        declareArray(loc, identifier, *sanitizedType, symbol, !flattenVar);
     } else {
         // non-array case
         if (! symbol)
-            symbol = declareNonArray(loc, identifier, type, !flattenVar);
+            symbol = declareNonArray(loc, identifier, *sanitizedType, !flattenVar);
         else if (type != symbol->getType())
             error(loc, "cannot change the type of", "redeclaration", symbol->getName().c_str());
     }
@@ -4822,6 +5166,9 @@ TIntermNode* HlslParseContext::declareVariable(const TSourceLoc& loc, TString& i
     if (flattenVar)
         flatten(loc, *symbol->getAsVariable());
 
+    if (splitVar)
+        split(*symbol->getAsVariable());
+
     if (! symbol)
         return nullptr;
 
@@ -5863,4 +6210,13 @@ void HlslParseContext::renameShaderFunction(TString*& name) const
         name = new TString(intermediate.getEntryPointName().c_str());
 }
 
+// post-processing
+void HlslParseContext::finish()
+{
+    addInterstageIoToLinkage();
+
+    TParseContextBase::finish();
+}
+
+
 } // end namespace glslang

hlsl/hlslParseHelper.h

@@ -186,6 +186,9 @@ protected:
     void fixConstInit(const TSourceLoc&, TString& identifier, TType& type, TIntermTyped*& initializer);
     void inheritGlobalDefaults(TQualifier& dst) const;
     TVariable* makeInternalVariable(const char* name, const TType&) const;
+    TVariable* makeInternalVariable(const TString& name, const TType& type) const {
+        return makeInternalVariable(name.c_str(), type);
+    }
     TVariable* declareNonArray(const TSourceLoc&, TString& identifier, TType&, bool track);
     void declareArray(const TSourceLoc&, TString& identifier, const TType&, TSymbol*&, bool track);
     TIntermNode* executeInitializer(const TSourceLoc&, TIntermTyped* initializer, TVariable* variable);
@@ -198,7 +201,7 @@ protected:
 
     // Array and struct flattening
     bool shouldFlatten(const TType& type) const;
-    TIntermTyped* flattenAccess(const TSourceLoc&, TIntermTyped* base, int member);
+    TIntermTyped* flattenAccess(TIntermTyped* base, int member);
     bool shouldFlattenIO(const TType&) const;
     bool shouldFlattenUniform(const TType&) const;
     bool wasFlattened(const TIntermTyped* node) const;
@@ -206,11 +209,30 @@ protected:
     int  addFlattenedMember(const TSourceLoc& loc, const TVariable&, const TType&, TFlattenData&, const TString& name, bool track);
     bool isFinalFlattening(const TType& type) const { return !(type.isStruct() || type.isArray()); }
 
+    // Structure splitting (splits interstage builtin types into its own struct)
+    bool shouldSplit(const TType&);
+    TIntermTyped* splitAccessStruct(const TSourceLoc& loc, TIntermTyped*& base, int& member);
+    void splitAccessArray(const TSourceLoc& loc, TIntermTyped* base, TIntermTyped* index);
+    TType& split(TType& type, TString name, const TType* outerStructType = nullptr);
+    void split(TIntermTyped*);
+    void split(const TVariable&);
+    bool wasSplit(const TIntermTyped* node) const;
+    bool wasSplit(int id) const { return splitIoVars.find(id) != splitIoVars.end(); }
+    TVariable* getSplitIoVar(const TIntermTyped* node) const;
+    TVariable* getSplitIoVar(const TVariable* var) const;
+    TVariable* getSplitIoVar(int id) const;
+    void addInterstageIoToLinkage();
+
     void flatten(const TSourceLoc& loc, const TVariable& variable);
     int flatten(const TSourceLoc& loc, const TVariable& variable, const TType&, TFlattenData&, TString name);
     int flattenStruct(const TSourceLoc& loc, const TVariable& variable, const TType&, TFlattenData&, TString name);
     int flattenArray(const TSourceLoc& loc, const TVariable& variable, const TType&, TFlattenData&, TString name);
 
+    // Type sanitization: return existing sanitized (temporary) type if there is one, else make new one.
+    TType* sanitizeType(TType*);
+
+    void finish(); // post-processing
+
     // Current state of parsing
     struct TPragma contextPragma;
     int loopNestingLevel;        // 0 if outside all loops
@@ -276,6 +298,38 @@ protected:
     TVector<int> flattenLevel;  // nested postfix operator level for flattening
     TVector<int> flattenOffset; // cumulative offset for flattening
 
+    // Sanitized type map.  During declarations we use the sanitized form of the type
+    // if it exists.
+    TMap<const TTypeList*, TType*> sanitizedTypeMap;
+
+    // Structure splitting data:
+    TMap<int, TVariable*>              splitIoVars;  // variables with the builtin interstage IO removed, indexed by unique ID.
+
+    // The builtin interstage IO map considers e.g, EvqPosition on input and output separately, so that we
+    // can build the linkage correctly if position appears on both sides.  Otherwise, multiple positions
+    // are considered identical.
+    struct tInterstageIoData {
+        tInterstageIoData(const TType& memberType, const TType& storageType) :
+            builtIn(memberType.getQualifier().builtIn),
+            storage(storageType.getQualifier().storage) { }
+
+        TBuiltInVariable  builtIn;
+        TStorageQualifier storage;
+
+        // ordering for maps
+        bool operator<(const tInterstageIoData d) const {
+            return (builtIn != d.builtIn) ? (builtIn < d.builtIn) : (storage < d.storage);
+        }
+    };
+
+    TMap<tInterstageIoData, TVariable*> interstageBuiltInIo; // individual builtin interstage IO vars, inxed by builtin type.
+
+    // We have to move array references to structs containing builtin interstage IO to the split variables.
+    // This is only handled for one level.  This stores the index, because we'll need it in the future, since
+    // unlike normal array references, here the index happens before we discover what it applies to.
+    TIntermTyped* builtInIoIndex;
+    TIntermTyped* builtInIoBase;
+
     unsigned int nextInLocation;
     unsigned int nextOutLocation;