可以在GRU层后面加入多头注意力机制,使用Keras的MultiHeadAttention实现,代码如下:
```python
from keras.layers import Input, Conv1D, CuDNNGRU, Dropout, concatenate, Dense, Lambda, add, Activation, Permute, Reshape, multiply, MultiHeadAttention
def LSTNet(trainX1,trainX2,trainY,config):
input1 = Input(shape=(trainX1.shape[1], trainX1.shape[2]))
conv1 = Conv1D(filters=48, kernel_size=6, strides=1, activation='relu') # for input1
# It's a probelm that I can't find any way to use the same Conv1D layer to train the two inputs,
conv2 = Conv1D(filters=48, kernel_size=6 , strides=1, activation='relu') # for input2
conv2.set_weights(conv1.get_weights()) # at least use same weight
conv1out = conv1(input1)
gru1out = CuDNNGRU(64)(conv1out)
gru1out = Dropout(config.dropout)(gru1out)
input2 = Input(shape=(trainX2.shape[1], trainX2.shape[2]))
conv2out = conv2(input2)
gru2out = CuDNNGRU(64)(conv2out)
gru2out = Dropout(config.dropout)(gru2out)
# 多头注意力机制
head_num = 4
attention_size = gru1out.shape[-1]
query = Dense(attention_size)(gru1out)
key = Dense(attention_size)(gru2out)
value = Dense(attention_size)(gru2out)
attention_output = MultiHeadAttention(num_heads=head_num, key_dim=attention_size // head_num)(query, key, value)
attention_output = Dropout(config.dropout)(attention_output)
gru_out = concatenate([gru1out, attention_output])
output = Dense(trainY.shape[1])(gru_out)
#highway 使用Dense模拟AR自回归过程,为预测添加线性成份,同时使输出可以响应输入的尺度变化