Transformer自2017年推出之后，已经横扫NLP领域，成为当之无愧的state-of-the-art。原始paper “Attention is All you Need”中对attention提出了通用的query/key/value抽象，开始时觉得很难理解，后来随着读的文献更多，慢慢体会到了其中的意思。关于Transformer和attention的各种解释类文章有很多，不再赘述，本文仅就其中的核心，MultiHeadAttention的实现进行源码剖析。

Transformer的实现有很多，但我看到实现得最clean的还是 # The Annotated Transformer。它的实现是一个最基本的版本，但五脏俱全，理解原理再合适不过。

class MultiHeadedAttention(nn.Module):
def __init__(self, h, d_model, dropout=0.1):
"Take in model size and number of heads."
assert d_model % h == 0
# We assume d_v always equals d_k
self.d_k = d_model // h
self.h = h
self.linears = clones(nn.Linear(d_model, d_model), 4)
self.attn = None
self.dropout = nn.Dropout(p=dropout)

def forward(self, query, key, value, mask=None):
"Implements Figure 2"
nbatches = query.size(0)

# 1) Do all the linear projections in batch from d_model => h x d_k
query, key, value = \
[l(x).view(nbatches, -1, self.h, self.d_k).transpose(1, 2)
for l, x in zip(self.linears, (query, key, value))]

# 2) Apply attention on all the projected vectors in batch.
dropout=self.dropout)

# 3) "Concat" using a view and apply a final linear.
x = x.transpose(1, 2).contiguous() \
.view(nbatches, -1, self.h * self.d_k)
return self.linears[-1](x)

# 1) Do all the linear projections in batch from d_model => h x d_k
query, key, value = \
[l(x).view(nbatches, -1, self.h, self.d_k).transpose(1, 2)
for l, x in zip(self.linears, (query, key, value))]

query, key, value = [l(x) for l, x in zip(self.linears, (query, key, value))]
query, key, value = [x.view(nbatches, -1, self.h, self.d_k).transpose(1, 2)
for x in (query, key, value)]

h = 8
d_model = 512
batch_size = 1
seq_length = 10

query = torch.randn([batch_size, seq_length, d_model])
key = query
value = query

print ('Input size: ' + str(query.size()))

import torch
import torch.nn as nn
import torch.nn.functional as F
import math, copy

def clones(module, N):
"Produce N identical layers."
return nn.ModuleList([copy.deepcopy(module) for _ in range(N)])

def attention(query, key, value, mask=None, dropout=None):
"Compute 'Scaled Dot Product Attention'"
d_k = query.size(-1)
scores = torch.matmul(query, key.transpose(-2, -1)) / math.sqrt(d_k)
p_attn = F.softmax(scores, dim = -1)
if dropout is not None:
p_attn = dropout(p_attn)

def __init__(self, h, d_model, dropout=0.1):
"Take in model size and number of heads."
assert d_model % h == 0
# We assume d_v always equals d_k
self.d_k = d_model // h
self.h = h
self.linears = clones(nn.Linear(d_model, d_model), 4) # create 4 linear layers
self.attn = None
self.dropout = nn.Dropout(p=dropout)

def forward(self, query, key, value, mask=None):
batch_size = query.size(0)

print ('Before transform query: ' + str(query.size())) # (batch_size, seq_length, d_model)

query, key, value = [l(x) for l, x in zip(self.linears, (query, key, value))] # (batch_size, seq_length, d_model), use first 3 self.linears
query, key, value = [x.view(nbatches, -1, self.h, self.d_k).transpose(1, 2)
for x in (query, key, value)] # (batch_size, h, seq_length, d_k)

print ('After transform query: ' + str(query.size()))

# 2) Apply attention on all the projected vectors in batch.

# 3) "Concat" using a view and apply a final linear.
x = x.transpose(1, 2).contiguous().view(batch_size, -1, self.h * self.d_k)
return self.linears[-1](x)

h = 8
d_model = 512
batch_size = 1
seq_length = 10

query = torch.randn([batch_size, seq_length, d_model])
key = query
value = query

print ('Input size: ' + str(query.size()))

m = model(query, key, value)

print ('Output size: ' + str(m.size()))

Input size: torch.Size([1, 10, 512])
Before transform query: torch.Size([1, 10, 512])
After transform query: torch.Size([1, 8, 10, 64])
Output size: torch.Size([1, 10, 512])