# 熵(Entropy)

## 代码与实例

import torch
import numpy as np
from torch.distributions import Categorical, kl

# Entropy
p = [0.1, 0.2, 0.3, 0.4]

Hp = -sum([p[i] * np.log(p[i]) for i in range(len(p))])
print (f"H(p) = {Hp}")
dist_p = Categorical(torch.tensor(p))
print (f"Torch H(p) = {dist_p.entropy().item()}")

H(p) = 1.2798542258336676
Torch H(p) = 1.2798542976379395

# 相对熵(Relative Entropy)

p(x)，q(x)为随机变量x的两个概率分布，定义p对q的相对熵(KL散度)为： $D(p||q)=\sum_{x}p(x) \log \frac{p(x)}{q(x)}$

KL散度在p(x)和q(x)相同时取到最小值0，两个概率分布越相似，则KL散度越小。

## 代码与实例

import torch
import numpy as np
from torch.distributions import Categorical, kl

# KL divergence
p = [0.1, 0.2, 0.3, 0.4]
q = [0.1, 0.1, 0.7, 0.1]

Dpq = sum([p[i] * np.log(p[i] / q[i]) for i in range(len(p))])
print (f"D(p, q) = {Dpq}")
dist_p = Categorical(torch.tensor(p))
dist_q = Categorical(torch.tensor(q))
print (f"Torch D(p, q) = {kl.kl_divergence(dist_p, dist_q)}")

D(p, q) = 0.43895782244378423
Torch D(p, q) = 0.4389578104019165

# 交叉熵(Cross Entropy)

## 代码与实例

import torch
import numpy as np
from torch.distributions import Categorical, kl

# Cross entropy
p = [0.1, 0.2, 0.3, 0.4]
q = [0.1, 0.1, 0.7, 0.1]

Hpq = -sum([p[i] * np.log(q[i]) for i in range(len(p))])
print (f"H(p, q) = {Hpq}")

H(p, q) = 1.7188120482774516

# 交叉熵损失函数

torch.nn.CrossEntropyLoss(weight: Optional[torch.Tensor] = None, size_average=None, ignore_index: int = -100, reduce=None, reduction: str = 'mean')

import torch
import numpy as np
from torch.distributions import Categorical, kl
from torch.nn import CrossEntropyLoss

# Cross entropy loss
p = [1, 2, 3, 4]
q = [1] # [0, 1, 0, 0] = torch.nn.functional.one_hot(torch.tensor(q), len(p))

celoss = -p[q[0]] + np.log(sum([np.exp(i) for i in p]))
print (f"Cross Entropy Loss: {celoss}")

loss = CrossEntropyLoss()
tensor_p = torch.FloatTensor(p).unsqueeze(0)
tensor_q = torch.tensor(q)
output = loss(tensor_p, tensor_q)
print (f"Torch Cross Entropy Loss: {output.item()}")

Cross Entropy Loss: 2.4401896985611957
Torch Cross Entropy Loss: 2.4401895999908447

$loss(p, tl) = -x[tl] + \log {\sum_{i} exp(x[i])}$

# 详解机器学习中的熵、条件熵、相对熵和交叉熵

# 如何通俗的解释交叉熵与相对熵？

# KL散度(相對熵)、交叉熵的解析

# 完整代码

import torch
import numpy as np
from torch.distributions import Categorical, kl
from torch.nn import CrossEntropyLoss

# Entropy
p = [0.1, 0.2, 0.3, 0.4]

Hp = -sum([p[i] * np.log(p[i]) for i in range(len(p))])
print (f"H(p) = {Hp}")
dist_p = Categorical(torch.tensor(p))
print (f"Torch H(p) = {dist_p.entropy().item()}")

# KL divergence
p = [0.1, 0.2, 0.3, 0.4]
q = [0.1, 0.1, 0.7, 0.1]

Dpq = sum([p[i] * np.log(p[i] / q[i]) for i in range(len(p))])
print (f"D(p, q) = {Dpq}")
dist_p = Categorical(torch.tensor(p))
dist_q = Categorical(torch.tensor(q))
print (f"Torch D(p, q) = {kl.kl_divergence(dist_p, dist_q)}")

# Cross entropy
p = [0.1, 0.2, 0.3, 0.4]
q = [0.1, 0.1, 0.7, 0.1]

Hpq = -sum([p[i] * np.log(q[i]) for i in range(len(p))])
print (f"H(p, q) = {Hpq}")

# Cross entropy loss
p = [1, 2, 3, 4]
q = [1] # [0, 1, 0, 0] = torch.nn.functional.one_hot(torch.tensor(q), len(p))

celoss = -p[q[0]] + np.log(sum([np.exp(i) for i in p]))
print (f"Cross Entropy Loss: {celoss}")

loss = CrossEntropyLoss()
tensor_p = torch.FloatTensor(p).unsqueeze(0)
tensor_q = torch.tensor(q)
output = loss(tensor_p, tensor_q)
print (f"Torch Cross Entropy Loss: {output.item()}")