Output tensors of a Functional model must be the output of a TensorFlow `Layer`

So I’m trying to expand the Roberta Pretrained Model and I was doing a basic model for testing but I’m getting this error from TensorFlow: ValueError: Output tensors of a Functional model must be the output of a TensorFlow Layer. which is from the Model api of Keras but I don’t exactly know what’s causing it.

Code:

LEN_SEQ = 64
BATCH_SIZE = 16
TEST_TRAIN_SPLIT = 0.9
TRANSFORMER = 'roberta-base'

df = pd.read_csv('train-processed.csv')
df = df.head(100)
samples_count = len(df)

# Create labels
target = df['first_Sentiment'].values.astype(int)
labels = np.zeros((samples_count, target.max() + 1))
labels[np.arange(samples_count), target] = 1

tokenizer = AutoTokenizer.from_pretrained(TRANSFORMER)
tokens = tokenizer(
    df['first_Phrase'].tolist(),
    max_length=LEN_SEQ,
    truncation=True,
    padding='max_length',
    add_special_tokens=True,
    return_tensors='tf'
)

base_model = TFAutoModel.from_pretrained(TRANSFORMER)

embedding = base_model.roberta(input_ids=tokens['input_ids'], attention_mask=tokens['attention_mask'])
embedding.trainable = False

# Define inputs
input_ids = Input(shape=(LEN_SEQ,), name='input_ids', dtype='int32')
input_mask = Input(shape=(LEN_SEQ,), name='input_mask', dtype='int32')

# Define hidden layers
layer = Dense(LEN_SEQ * 2, activation='relu')(embedding[1])
layer = Dense(LEN_SEQ, activation='relu')(layer)

# Define output
output = Dense(target.max() + 1, activation='softmax', name='output')(layer)

model = Model(inputs=[input_ids, input_mask], outputs=[output])

Full error traceback:

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-80-9a6ccb1b4ca8> in <module>
     10 output = Dense(target.max() + 1, activation='softmax', name='output')(layer)
     11 
---> 12 model = Model(inputs=[input_ids, input_mask], outputs=[output])
     13 
     14 model.compile(

/usr/local/lib/python3.8/site-packages/tensorflow/python/training/tracking/base.py in _method_wrapper(self, *args, **kwargs)
    528     self._self_setattr_tracking = False  # pylint: disable=protected-access
    529     try:
--> 530       result = method(self, *args, **kwargs)
    531     finally:
    532       self._self_setattr_tracking = previous_value  # pylint: disable=protected-access

/usr/local/lib/python3.8/site-packages/keras/engine/functional.py in __init__(self, inputs, outputs, name, trainable, **kwargs)
    107     generic_utils.validate_kwargs(kwargs, {})
    108     super(Functional, self).__init__(name=name, trainable=trainable)
--> 109     self._init_graph_network(inputs, outputs)
    110 
    111   @tf.__internal__.tracking.no_automatic_dependency_tracking

/usr/local/lib/python3.8/site-packages/tensorflow/python/training/tracking/base.py in _method_wrapper(self, *args, **kwargs)
    528     self._self_setattr_tracking = False  # pylint: disable=protected-access
    529     try:
--> 530       result = method(self, *args, **kwargs)
    531     finally:
    532       self._self_setattr_tracking = previous_value  # pylint: disable=protected-access

/usr/local/lib/python3.8/site-packages/keras/engine/functional.py in _init_graph_network(self, inputs, outputs)
    144         base_layer_utils.create_keras_history(self._nested_outputs)
    145 
--> 146     self._validate_graph_inputs_and_outputs()
    147 
    148     # A Network does not create weights of its own, thus it is already

/usr/local/lib/python3.8/site-packages/keras/engine/functional.py in _validate_graph_inputs_and_outputs(self)
    719       if not hasattr(x, '_keras_history'):
    720         cls_name = self.__class__.__name__
--> 721         raise ValueError('Output tensors of a ' + cls_name + ' model must be '
    722                          'the output of a TensorFlow `Layer` '
    723                          '(thus holding past layer metadata). Found: ' + str(x))

ValueError: Output tensors of a Functional model must be the output of a TensorFlow `Layer` (thus holding past layer metadata). Found: tf.Tensor(
[[0.18333092 0.18954797 0.22477032 0.21039596 0.1919548 ]
 [0.18219706 0.1903447  0.2256843  0.20587942 0.19589448]
 [0.18239683 0.1907878  0.22491893 0.20824413 0.19365236]
 [0.18193942 0.1898969  0.2259874  0.20646562 0.1957107 ]
 [0.18132213 0.1893883  0.22565623 0.21005587 0.1935775 ]
 [0.18237704 0.18789911 0.22692119 0.20759228 0.1952104 ]
 [0.18217668 0.18732095 0.22601548 0.21063867 0.19384828]
 [0.1817196  0.18970788 0.22175607 0.21405536 0.19276106]
 [0.18154216 0.18738106 0.22770867 0.2091297  0.19423842]
 [0.1839993  0.19110405 0.2193769  0.2124882  0.19303149]
 [0.18009834 0.19029345 0.22552258 0.21138497 0.19270067]
 [0.18262982 0.18932794 0.22548872 0.20995376 0.19259974]
 [0.18132062 0.1894746  0.22257458 0.21089785 0.19573237]
 [0.18127124 0.18927224 0.2275273  0.2061446  0.19578464]
 [0.18001163 0.1883382  0.22915907 0.20934238 0.19314879]
 [0.18409619 0.19204247 0.22269006 0.20967877 0.19149245]
 [0.18143429 0.18780865 0.22895294 0.21044146 0.1913626 ]
 [0.18210162 0.18980804 0.22135185 0.21205473 0.1946838 ]
 [0.18077913 0.18933856 0.22730026 0.2079047  0.19467732]
 [0.18248595 0.19133545 0.2252994  0.20402898 0.1968502 ]
 [0.18053354 0.18830904 0.22379933 0.21369977 0.19365832]
 [0.18100418 0.1889128  0.22656825 0.21134934 0.19216539]
 [0.18219638 0.18901002 0.22543809 0.20894748 0.194408  ]
 [0.17991781 0.18693839 0.23250549 0.21227528 0.18836297]
 [0.18322821 0.1881207  0.22497904 0.20976694 0.19390512]
 [0.17972894 0.18888594 0.2251662  0.21268585 0.19353302]
 [0.1822505  0.18769115 0.22729188 0.21127912 0.19148737]
 [0.18432644 0.18830952 0.22477935 0.20987424 0.19271052]
 [0.1801894  0.18920776 0.22684936 0.20734173 0.19641179]
 [0.181594   0.1880084  0.22798598 0.20937674 0.19303486]
 [0.18252885 0.19045824 0.22497422 0.207161   0.19487773]
 [0.18196142 0.18878765 0.22479571 0.2105628  0.19389246]
 [0.18600896 0.18686578 0.2283819  0.21188499 0.18685843]
 [0.18056509 0.18865508 0.22694935 0.21080662 0.19302382]
 [0.18446274 0.1887065  0.22405164 0.21271324 0.19006592]
 [0.1812612  0.18995184 0.22384171 0.20790772 0.19703752]
 [0.1861402  0.189157   0.2236694  0.21078445 0.19024895]
 [0.18149142 0.18862149 0.2255336  0.20888737 0.19546609]
 [0.18088317 0.1882689  0.22780944 0.20749897 0.19553955]
 [0.1824722  0.18926203 0.22691077 0.2071967  0.1941583 ]
 [0.18111941 0.18773855 0.22366299 0.21535842 0.19212064]
 [0.18248987 0.18920848 0.22602491 0.20733926 0.19493747]
 [0.18306294 0.19167435 0.22505572 0.21000686 0.19020009]
 [0.18466519 0.1885763  0.22352514 0.21257839 0.19065501]
 [0.18297954 0.18976018 0.2262897  0.20864752 0.19232307]
 [0.18216778 0.18953851 0.22490299 0.21057723 0.1928135 ]
 [0.18181367 0.19077264 0.2232015  0.21115994 0.1930523 ]
 [0.18345618 0.18753015 0.22660162 0.20830849 0.1941036 ]
 [0.18212378 0.18797131 0.2247642  0.21066691 0.19447377]
 [0.18199605 0.19106121 0.22245005 0.21217921 0.19231346]
 [0.18243583 0.18764758 0.22628336 0.21369886 0.18993443]
 [0.18162242 0.18957089 0.22591078 0.20930369 0.19359224]
 [0.18090473 0.18757755 0.22858356 0.20813066 0.19480348]
 [0.17951688 0.18841572 0.22520997 0.21235934 0.19449812]
 [0.1850496  0.18895829 0.22575855 0.20854111 0.1916925 ]
 [0.18254244 0.18938984 0.22754729 0.20879866 0.19172177]
 [0.1816532  0.18972425 0.22676478 0.20679341 0.19506434]
 [0.18303266 0.19159187 0.22373216 0.20538329 0.19625996]
 [0.18126963 0.18750906 0.2258774  0.21198079 0.1933631 ]
 [0.18387978 0.18828613 0.22228165 0.21189795 0.19365448]
 [0.1834729  0.18976368 0.22469373 0.20830937 0.19376035]
 [0.18359789 0.18833868 0.22379532 0.21078889 0.19347927]
 [0.18039297 0.18886234 0.22411437 0.2105467  0.19608359]
 [0.17980678 0.18979622 0.2266618  0.20471531 0.19901991]
 [0.18554561 0.19003332 0.22477089 0.21021138 0.18943883]
 [0.18349187 0.18941568 0.22224301 0.21004184 0.19480757]
 [0.18351436 0.19169463 0.22155108 0.21009424 0.19314568]
 [0.18123321 0.18985166 0.22660086 0.21186577 0.19044854]
 [0.18183744 0.192495   0.22091088 0.21275932 0.1919973 ]
 [0.18028514 0.18943599 0.22416686 0.21241388 0.19369814]
 [0.18061554 0.18873625 0.22677769 0.21073307 0.19313747]
 [0.18186866 0.18851075 0.22588421 0.21183755 0.19189876]
 [0.18126652 0.18949142 0.22501452 0.20897155 0.19525598]
 [0.1835434  0.19079022 0.22333461 0.21146008 0.19087164]
 [0.18269798 0.19171126 0.22150221 0.21224435 0.19184415]
 [0.17996274 0.19000672 0.22470033 0.2105299  0.19480029]
 [0.18345153 0.19032337 0.2239142  0.21167503 0.19063583]
 [0.18224017 0.19025423 0.22567508 0.2087501  0.19308044]
 [0.18233515 0.18966553 0.22833474 0.20635706 0.1933075 ]
 [0.18210347 0.18650064 0.22770585 0.21101129 0.19267873]
 [0.18199693 0.19086935 0.22255068 0.20988034 0.19470267]
 [0.18119748 0.18983872 0.22518982 0.20845842 0.19531558]
 [0.18367417 0.19071157 0.22310348 0.21277103 0.18973975]
 [0.17965038 0.18936628 0.22479466 0.21279414 0.19339451]
 [0.18141513 0.18989322 0.22380653 0.21031635 0.19456872]
 [0.18295668 0.19067182 0.22385122 0.20624346 0.1962768 ]
 [0.17981796 0.18981294 0.22544417 0.21043345 0.19449154]
 [0.18068986 0.1897383  0.22433658 0.21027999 0.1949553 ]
 [0.18146665 0.18844193 0.22996067 0.20703284 0.19309792]
 [0.18278767 0.18972701 0.22451803 0.20893572 0.19403161]
 [0.18077034 0.1892612  0.2236769  0.21081012 0.19548143]
 [0.18254872 0.19220418 0.22300169 0.20895892 0.19328652]
 [0.18032935 0.19029863 0.22319157 0.21000609 0.19617435]
 [0.18328631 0.18907256 0.22911799 0.20782094 0.19070214]
 [0.17863902 0.18771355 0.23066713 0.21065918 0.19232109]
 [0.18178153 0.19022569 0.22538401 0.20857622 0.1940325 ]
 [0.18072292 0.18907587 0.22616044 0.21096109 0.19307965]
 [0.18215105 0.18966101 0.22436853 0.21200544 0.191814  ]
 [0.18104836 0.18830387 0.22495148 0.21120267 0.19449359]
 [0.18192047 0.18981694 0.22512193 0.2107065  0.19243418]], shape=(100, 5), dtype=float32)

Data example:

Any help appreciated. I’m new to transformers so please feel free to point any extra considerations.

Answer

You need to pass a list of [input_ids , input_mask] to base_model.

# !pip install transformers 
from transformers import TFAutoModel
import tensorflow as tf


LEN_SEQ = 64

# Define inputs
input_ids = tf.keras.layers.Input(shape=(LEN_SEQ,), name='input_ids', dtype='int32')
input_mask = tf.keras.layers.Input(shape=(LEN_SEQ,), name='input_mask', dtype='int32')

base_model = TFAutoModel.from_pretrained('roberta-base')
for layer in base_model.layers:
    layer.trainable = False
# Check summary of tf_roberta_model
base_model.summary()

embedding = base_model([input_ids, input_mask])[1]
# Or
# embedding = base_model([input_ids, input_mask]).pooler_output


# Define hidden layers
layer = tf.keras.layers.Dense(LEN_SEQ * 2, activation='relu')(embedding)
layer = tf.keras.layers.Dense(LEN_SEQ, activation='relu')(layer)

# Define output
output = tf.keras.layers.Dense(2, activation='softmax', name='output')(layer)


model = tf.keras.models.Model(inputs=[input_ids, input_mask], outputs=[output])
model.summary()

Output:

Model: "tf_roberta_model_2"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 roberta (TFRobertaMainLayer  multiple                 124645632 
 )                                                               
                                                                 
=================================================================
Total params: 124,645,632
Trainable params: 0
Non-trainable params: 124,645,632
_________________________________________________________________


Model: "model"
__________________________________________________________________________________________________
 Layer (type)                   Output Shape         Param #     Connected to                     
==================================================================================================
 input_ids (InputLayer)         [(None, 64)]         0           []                               
                                                                                                  
 input_mask (InputLayer)        [(None, 64)]         0           []                               
                                                                                                  
 tf_roberta_model_2 (TFRobertaM  TFBaseModelOutputWi  124645632  ['input_ids[0][0]',              
 odel)                          thPoolingAndCrossAt               'input_mask[0][0]']             
                                tentions(last_hidde                                               
                                n_state=(None, 64,                                                
                                768),                                                             
                                 pooler_output=(Non                                               
                                e, 768),                                                          
                                 past_key_values=No                                               
                                ne, hidden_states=N                                               
                                one, attentions=Non                                               
                                e, cross_attentions                                               
                                =None)                                                            
                                                                                                  
 dense (Dense)                  (None, 128)          98432       ['tf_roberta_model_2[0][1]']     
                                                                                                  
 dense_1 (Dense)                (None, 64)           8256        ['dense[0][0]']                  
                                                                                                  
 output (Dense)                 (None, 2)            130         ['dense_1[0][0]']                
                                                                                                  
==================================================================================================
Total params: 124,752,450
Trainable params: 106,818
Non-trainable params: 124,645,632
__________________________________________________________________________________________________