[Maya python] vertex color deformer not updating

Hey guys,

i’m just starting to learn maya’s python scripting and i’m having problems with vertex color information updating in a deformer.

at the end of the deform function i’m calling:

mesh.setVertexColors(vertex_color_list, vertex_index_list)

and everything works fine except the result doesnt show up untill i use some tool on the mesh. Any ideas why is that happening?

Here’s the code:

import sys
import maya.OpenMaya as OpenMaya
import maya.OpenMayaMPx as OpenMayaMPx
# Helper functions

def clamp(min_value,max_value,value):
    return max(min(value, max_value), min_value)
def saturate(value):
    return clamp(0, 1, value)
def mapValue(min_value, max_value, value):
    return saturate((value - min_value)/(max_value - min_value))
def cutout(min_value, value):
    if value < min_value:
        return 0
    else:
        return value

# Plugin information

plugin_node_name = 'SM_Damage'
plugin_node_id = OpenMaya.MTypeId(0x1C3B0476)

maya_api_version = OpenMaya.MGlobal.apiVersion()

# get the input and geometry based on the maya version
if maya_api_version < 201600:
    input_attr = OpenMayaMPx.cvar.MPxDeformerNode_input
    input_geom_attr = OpenMayaMPx.cvar.MPxDeformerNode_inputGeom
    output_geom_attr = OpenMayaMPx.cvar.MPxDeformerNode_outputGeom
else:
    input_attr = OpenMayaMPx.cvar.MPxGeometryFilter_input
    input_geom_attr = OpenMayaMPx.cvar.MPxGeometryFilter_inputGeom
    output_geom_attr = OpenMayaMPx.cvar.MPxGeometryFilter_outputGeom

# get the envelope based on the maya version
if maya_api_version < 201600:
    envelope_attr = OpenMayaMPx.cvar.MPxDeformerNode_envelope
else:
    envelope_attr = OpenMayaMPx.cvar.MPxGeometryFilter_envelope


class SM_DamageDeformerNode(OpenMayaMPx.MPxDeformerNode):

    rust_exponent_attr = OpenMaya.MObject()
    rust_black_point_attr = OpenMaya.MObject()
    rust_multiplier_attr = OpenMaya.MObject()
    # metal_attributes
    metal_exponent_attr = OpenMaya.MObject()
    metal_black_point_attr = OpenMaya.MObject()
    metal_multiplier_attr = OpenMaya.MObject()
    # AO_attributes
    AO_exponent_attr = OpenMaya.MObject()
    AO_black_point_attr = OpenMaya.MObject()
    AO_multiplier_attr = OpenMaya.MObject()

    def __init__(self):
        ''' Constructor. '''
        # (!) Make sure you call the base class's constructor.
        OpenMayaMPx.MPxDeformerNode.__init__(self)
    

    def clamp(min_value, max_value, value):
        return max(min(value, max_value), min_value)

    def get_input_geom(self, data, geom_idx):
        input_handle = data.outputArrayValue(input_attr)
        input_handle.jumpToElement(geom_idx)
        input_geom_obj = input_handle.outputValue().child(input_geom_attr).asMesh()
        return input_geom_obj
    #END

    def deform(self, data, geom_it, local_to_world_mat, geom_idx):
        
        rust_exponent_handle = data.inputValue(SM_DamageDeformerNode.rust_exponent_attr)
        rust_black_point_handle = data.inputValue(SM_DamageDeformerNode.rust_black_point_attr)
        rust_multiplier_handle = data.inputValue(SM_DamageDeformerNode.rust_multiplier_attr)
        # metal_handles
        metal_exponent_handle = data.inputValue(SM_DamageDeformerNode.metal_exponent_attr)
        metal_black_point_handle = data.inputValue(SM_DamageDeformerNode.metal_black_point_attr)
        metal_multiplier_handle = data.inputValue(SM_DamageDeformerNode.metal_multiplier_attr)
        # AO_handles
        AO_exponent_handle = data.inputValue(SM_DamageDeformerNode.AO_exponent_attr)
        AO_black_point_handle = data.inputValue(SM_DamageDeformerNode.AO_black_point_attr)
        AO_multiplier_handle = data.inputValue(SM_DamageDeformerNode.AO_multiplier_attr)
        
        # set final variables
        rust_exponent = rust_exponent_handle.asFloat()
        rust_black_point = rust_black_point_handle.asFloat()
        rust_multiplier = rust_multiplier_handle.asFloat()
        # metal_attributes
        metal_exponent = metal_exponent_handle.asFloat()
        metal_black_point = metal_black_point_handle.asFloat()
        metal_multiplier = metal_multiplier_handle.asFloat()
        # AO_attributes
        AO_exponent = AO_exponent_handle.asFloat()
        AO_black_point = AO_black_point_handle.asFloat()
        AO_multiplier = AO_multiplier_handle.asFloat()

        
        # creates OpenMaya mesh object
        input_geom_obj = self.get_input_geom(data, geom_idx)
        mesh = OpenMaya.MFnMesh(input_geom_obj)
        mesh_iterator = OpenMaya.MItMeshVertex(input_geom_obj)
        vertex_count = mesh_iterator.count()

        # Retrieve mesh data

        fnComponent = OpenMaya.MFnSingleIndexedComponent()
        fullComponent = fnComponent.create(OpenMaya.MFn.kMeshVertComponent)
        fnComponent.setCompleteData(vertex_count)
        vertex_index_list = OpenMaya.MIntArray()

        # get vertex index list
        fnComponent.getElements(vertex_index_list)
        # get vertex normals
        normals = OpenMaya.MFloatVectorArray()
        space = OpenMaya.MSpace.kObject
        mesh.getVertexNormals(True, normals, space)

        vertex_color_list = OpenMaya.MColorArray()
        for i in range(vertex_count):
            vertex_color_list.setLength(vertex_count)
            vertex_color_list.set(i, 0, 0, 0)

        # iterate over all vertices

        while not mesh_iterator.isDone():

            # collect current vertex data
            index = mesh_iterator.index()
            # get current vertex normal
            normal = OpenMaya.MVector(normals[index])
            vertex_color = OpenMaya.MColor()
            # get current vertex position
            position_a = mesh_iterator.position(space)

            # retrieve connected indices
            connected_vertices = OpenMaya.MIntArray()
            mesh_iterator.getConnectedVertices(connected_vertices)

            dot_product_min = 1.0
            dot_product_max = -1.0

            # calculate concavity/convexity data
            space = OpenMaya.MSpace.kObject
            # print "vertex ", index
            # print "position x", "%.3f" % position_a.x, ", y", "%.3f" % position_a.y, ", z", "%.3f" % position_a.z
            # print "N: x", "%.3f" % normal.x, ", y", "%.3f" % normal.y, ", z","%.3f" %  normal.z

            for k in connected_vertices:

                position_b = OpenMaya.MPoint()
                mesh.getPoint(k, position_b, space)
                # create edge vector
                edge_vector = position_b - position_a
                edge_vector.normalize()
                # calculate dot product
                dot_product = edge_vector * normal

                if dot_product < dot_product_min:
                    dot_product_min = dot_product
                if dot_product > dot_product_max:
                    dot_product_max = dot_product
                    # END IF

            # END FOR
            # print "factor: ", "%.3f" % dot_product_min


            def calculate_value(change_sign, exponent, black_point, final_multiplier):
                # print color_value
                if change_sign:
                    color_value = dot_product_min
                    color_value *= -1
                else:
                    color_value = dot_product_min
                color_value = saturate(color_value)
                color_value = pow(color_value, exponent)
                color_value = cutout(black_point, color_value)
                color_value = color_value * final_multiplier
                color_value = saturate(color_value)

                return color_value

            # print rust_value

            vertex_color.r = calculate_value(False, AO_exponent, AO_black_point, AO_multiplier)
            vertex_color.g = calculate_value(True, rust_exponent, rust_black_point, rust_multiplier)
            vertex_color.b = calculate_value(True, metal_exponent, metal_black_point, metal_multiplier)

            vertex_color_list.set(vertex_color, index)

            mesh_iterator.next()

        mesh.setVertexColors(vertex_color_list, vertex_index_list)
#END deform

def nodeCreator():
    return OpenMayaMPx.asMPxPtr(SM_DamageDeformerNode())
#END

def nodeInitializer():

    num_attr = OpenMaya.MFnNumericAttribute()

    # setup attributes

    # rust black point
    SM_DamageDeformerNode.rust_black_point_attr = num_attr.create('rust_black_point', 'rbp',
                                                             OpenMaya.MFnNumericData.kFloat, 0.0)
    num_attr.setMin(0.0)
    num_attr.setMax(1.0)
    num_attr.setChannelBox(True)
    num_attr.setStorable(True)
    num_attr.setWritable(True)
    num_attr.setReadable(False)
    SM_DamageDeformerNode.addAttribute(SM_DamageDeformerNode.rust_black_point_attr)

    # metal black point
    SM_DamageDeformerNode.metal_black_point_attr = num_attr.create('metal_black_point', 'mbp',
                                                              OpenMaya.MFnNumericData.kFloat, 0.0)
    num_attr.setMin(0.0)
    num_attr.setMax(1.0)
    num_attr.setChannelBox(True)
    num_attr.setStorable(True)
    num_attr.setWritable(True)
    num_attr.setReadable(False)
    SM_DamageDeformerNode.addAttribute(SM_DamageDeformerNode.metal_black_point_attr)

    # AO black point
    SM_DamageDeformerNode.AO_black_point_attr = num_attr.create('AO_black_point', 'aobp',
                                                           OpenMaya.MFnNumericData.kFloat, 0.0)
    num_attr.setMin(0.0)
    num_attr.setMax(1.0)
    num_attr.setChannelBox(True)
    num_attr.setStorable(True)
    num_attr.setWritable(True)
    num_attr.setReadable(False)
    SM_DamageDeformerNode.addAttribute(SM_DamageDeformerNode.AO_black_point_attr)

    # rust exponent
    SM_DamageDeformerNode.rust_exponent_attr = num_attr.create('rust_exponent', 'rexp',
                                                          OpenMaya.MFnNumericData.kFloat, 1.0)
    num_attr.setMin(0.0)
    num_attr.setMax(10)
    num_attr.setChannelBox(True)
    num_attr.setStorable(True)
    num_attr.setWritable(True)
    num_attr.setReadable(False)
    SM_DamageDeformerNode.addAttribute(SM_DamageDeformerNode.rust_exponent_attr)

    # metal exponent
    SM_DamageDeformerNode.metal_exponent_attr = num_attr.create('metal_exponent', 'mexp',
                                                          OpenMaya.MFnNumericData.kFloat, 1.0)
    num_attr.setMin(0.0)
    num_attr.setMax(10)
    num_attr.setChannelBox(True)
    num_attr.setStorable(True)
    num_attr.setWritable(True)
    num_attr.setReadable(False)
    SM_DamageDeformerNode.addAttribute(SM_DamageDeformerNode.metal_exponent_attr)

    # AO exponent
    SM_DamageDeformerNode.AO_exponent_attr = num_attr.create('AO_exponent', 'aoexp',
                                                          OpenMaya.MFnNumericData.kFloat, 1.0)
    num_attr.setMin(0.0)
    num_attr.setMax(10)
    num_attr.setChannelBox(True)
    num_attr.setStorable(True)
    num_attr.setWritable(True)
    num_attr.setReadable(False)
    SM_DamageDeformerNode.addAttribute(SM_DamageDeformerNode.AO_exponent_attr)

    # rust multiplier
    SM_DamageDeformerNode.rust_multiplier_attr = num_attr.create('rust_multiplier', 'rmul',
                                                          OpenMaya.MFnNumericData.kFloat, 1.0)
    num_attr.setMin(0.0)
    num_attr.setMax(10)
    num_attr.setChannelBox(True)
    num_attr.setStorable(True)
    num_attr.setWritable(True)
    num_attr.setReadable(False)
    SM_DamageDeformerNode.addAttribute(SM_DamageDeformerNode.rust_multiplier_attr)

    # metal_multiplier
    SM_DamageDeformerNode.metal_multiplier_attr = num_attr.create('metal_multiplier', 'mmul',
                                                          OpenMaya.MFnNumericData.kFloat, 1.0)
    num_attr.setMin(0.0)
    num_attr.setMax(10)
    num_attr.setChannelBox(True)
    num_attr.setStorable(True)
    num_attr.setWritable(True)
    num_attr.setReadable(False)
    SM_DamageDeformerNode.addAttribute(SM_DamageDeformerNode.metal_multiplier_attr)

    # AO_multiplier
    SM_DamageDeformerNode.AO_multiplier_attr = num_attr.create('AO_multiplier', 'aomul',
                                                          OpenMaya.MFnNumericData.kFloat, 1.0)
    num_attr.setMin(0.0)
    num_attr.setMax(10)
    num_attr.setChannelBox(True)
    num_attr.setStorable(True)
    num_attr.setWritable(True)
    num_attr.setReadable(False)
    SM_DamageDeformerNode.addAttribute(SM_DamageDeformerNode.AO_multiplier_attr)

    print dir(OpenMayaMPx.cvar)
    # set Attribute affects

    # black points
    SM_DamageDeformerNode.attributeAffects(SM_DamageDeformerNode.metal_black_point_attr,output_geom_attr)
    SM_DamageDeformerNode.attributeAffects(SM_DamageDeformerNode.AO_black_point_attr,output_geom_attr)
    SM_DamageDeformerNode.attributeAffects(SM_DamageDeformerNode.rust_black_point_attr,output_geom_attr)
    # exponents
    SM_DamageDeformerNode.attributeAffects(SM_DamageDeformerNode.metal_exponent_attr,output_geom_attr)
    SM_DamageDeformerNode.attributeAffects(SM_DamageDeformerNode.AO_exponent_attr,output_geom_attr)
    SM_DamageDeformerNode.attributeAffects(SM_DamageDeformerNode.rust_exponent_attr,output_geom_attr)
    # multipliers
    SM_DamageDeformerNode.attributeAffects(SM_DamageDeformerNode.metal_multiplier_attr,output_geom_attr)
    SM_DamageDeformerNode.attributeAffects(SM_DamageDeformerNode.AO_multiplier_attr,output_geom_attr)
    SM_DamageDeformerNode.attributeAffects(SM_DamageDeformerNode.rust_multiplier_attr,output_geom_attr)
#END


def initializePlugin(mobject):
    mplugin = OpenMayaMPx.MFnPlugin(mobject)
    try:
        mplugin.registerNode(plugin_node_name, plugin_node_id, nodeCreator, nodeInitializer, OpenMayaMPx.MPxNode.kDeformerNode)
    except:
        sys.stderr.write('Failed to register node: ' + plugin_node_name)
        raise
#END

def uninitializePlugin(mobject):
    ''' Uninitializes the plug-in '''
    mplugin = OpenMayaMPx.MFnPlugin(mobject)
    try:
        mplugin.deregisterNode(plugin_node_id)
    except:
        sys.stderr.write('Failed to deregister node: ' + plugin_node_name)
        raise

    

If i add actual vertex transform in the deform function it works fine. So the Deformer itself is correct. But why isn’t it updating vertex color in the viewport? The calculated values are written into the colorSet but it doesn’t show up until some other tool is used on the object.

I’ve seen other people using MPxNode class for vertex color and implementing compute function. Since MPxDeformerNode is a subclass of MPxNode it should have the same functionality but because my script doesn’t work i’m assuming that deform only tells maya (viewport?) to update position and normal but not vertex color. Is there a way around it?

It’s all a bit confusing for me.