Construindo uma IA de Go do Zero
Este artigo guia voce passo a passo na implementacao de uma IA de Go simplificada no estilo AlphaGo Zero, abrangendo logica do jogo, rede neural, MCTS e fluxo de treinamento.
Construir uma IA de Go do zero exige quatro modulos centrais: a logica do tabuleiro e das regras, a rede neural (com cabeca de politica e cabeca de valor), a busca em arvore de Monte Carlo (MCTS) e o ciclo de treinamento por reforco com auto-jogo. Este artigo implementa uma versao simplificada da arquitetura AlphaGo Zero em um tabuleiro 9x9, capaz de treinar uma IA com nivel de jogo amador iniciante.
Objetivo de Aprendizado
Apos completar este tutorial, voce tera uma IA de Go capaz de:
- Jogar contra si mesma em tabuleiro 9x9
- Melhorar continuamente atraves de aprendizado por reforco
- Atingir nivel de jogo amador iniciante
Arquitetura do Projeto
mini-alphago/
├── game/
│ ├── __init__.py
│ ├── board.py # Logica do tabuleiro
│ ├── rules.py # Implementacao de regras
│ └── state.py # Estado do jogo
├── model/
│ ├── __init__.py
│ ├── network.py # Rede neural
│ └── features.py # Codificacao de recursos
├── mcts/
│ ├── __init__.py
│ ├── node.py # No MCTS
│ └── search.py # Busca MCTS
├── training/
│ ├── __init__.py
│ ├── self_play.py # Auto-jogo
│ └── trainer.py # Treinador
├── main.py # Programa principal
└── requirements.txt
Passo 1: Tabuleiro e Regras
Implementacao do Tabuleiro
# game/board.py
import numpy as np
class Board:
"""Tabuleiro de Go"""
EMPTY = 0
BLACK = 1
WHITE = 2
def __init__(self, size=9):
self.size = size
self.board = np.zeros((size, size), dtype=np.int8)
self.current_player = self.BLACK
self.ko_point = None
self.history = []
def copy(self):
"""Copia o tabuleiro"""
new_board = Board(self.size)
new_board.board = self.board.copy()
new_board.current_player = self.current_player
new_board.ko_point = self.ko_point
new_board.history = self.history.copy()
return new_board
def get_opponent(self, player):
"""Obtem o oponente"""
return self.WHITE if player == self.BLACK else self.BLACK
def is_on_board(self, x, y):
"""Verifica se esta no tabuleiro"""
return 0 <= x < self.size and 0 <= y < self.size
def get_neighbors(self, x, y):
"""Obtem pontos adjacentes"""
neighbors = []
for dx, dy in [(-1, 0), (1, 0), (0, -1), (0, 1)]:
nx, ny = x + dx, y + dy
if self.is_on_board(nx, ny):
neighbors.append((nx, ny))
return neighbors
def get_group(self, x, y):
"""Obtem grupo (pedras conectadas da mesma cor)"""
color = self.board[x, y]
if color == self.EMPTY:
return set(), set()
group = set()
liberties = set()
stack = [(x, y)]
while stack:
cx, cy = stack.pop()
if (cx, cy) in group:
continue
group.add((cx, cy))
for nx, ny in self.get_neighbors(cx, cy):
if self.board[nx, ny] == self.EMPTY:
liberties.add((nx, ny))
elif self.board[nx, ny] == color and (nx, ny) not in group:
stack.append((nx, ny))
return group, liberties
def count_liberties(self, x, y):
"""Conta liberdades"""
_, liberties = self.get_group(x, y)
return len(liberties)
def remove_group(self, group):
"""Remove grupo"""
for x, y in group:
self.board[x, y] = self.EMPTY
def is_legal(self, x, y, player=None):
"""Verifica se e jogada legal"""
if player is None:
player = self.current_player
# Verifica se e ponto vazio
if self.board[x, y] != self.EMPTY:
return False
# Verifica se e Ko
if self.ko_point == (x, y):
return False
# Simula jogada
test_board = self.copy()
test_board.board[x, y] = player
# Primeiro verifica se pode capturar
opponent = self.get_opponent(player)
captured = []
for nx, ny in self.get_neighbors(x, y):
if test_board.board[nx, ny] == opponent:
group, liberties = test_board.get_group(nx, ny)
if len(liberties) == 0:
captured.extend(group)
if captured:
return True
# Verifica suicidio
_, liberties = test_board.get_group(x, y)
if len(liberties) == 0:
return False
return True
def play(self, x, y):
"""Joga uma pedra"""
if not self.is_legal(x, y):
return False
player = self.current_player
opponent = self.get_opponent(player)
# Coloca pedra
self.board[x, y] = player
# Captura
captured = []
for nx, ny in self.get_neighbors(x, y):
if self.board[nx, ny] == opponent:
group, liberties = self.get_group(nx, ny)
if len(liberties) == 0:
captured.extend(group)
self.remove_group(group)
# Define Ko
if len(captured) == 1:
cx, cy = list(captured)[0]
_, my_liberties = self.get_group(x, y)
if len(my_liberties) == 1:
self.ko_point = (cx, cy)
else:
self.ko_point = None
else:
self.ko_point = None
# Registra historico
self.history.append((x, y, player))
# Troca jogador
self.current_player = opponent
return True
def pass_move(self):
"""Pass (jogada virtual)"""
self.history.append((-1, -1, self.current_player))
self.current_player = self.get_opponent(self.current_player)
self.ko_point = None
def is_game_over(self):
"""Verifica se o jogo acabou"""
if len(self.history) < 2:
return False
# Ambos passaram consecutivamente
return (self.history[-1][0] == -1 and
self.history[-2][0] == -1)
def get_legal_moves(self):
"""Obtem todas as jogadas legais"""
moves = []
for x in range(self.size):
for y in range(self.size):
if self.is_legal(x, y):
moves.append((x, y))
moves.append((-1, -1)) # pass
return moves
def score(self):
"""Calcula pontuacao (metodo de contagem de area simplificado)"""
black_score = np.sum(self.board == self.BLACK)
white_score = np.sum(self.board == self.WHITE)
# Calculo de territorio simplificado
for x in range(self.size):
for y in range(self.size):
if self.board[x, y] == self.EMPTY:
neighbors = self.get_neighbors(x, y)
colors = set(self.board[nx, ny] for nx, ny in neighbors)
colors.discard(self.EMPTY)
if len(colors) == 1:
if self.BLACK in colors:
black_score += 1
else:
white_score += 1
komi = 5.5 if self.size == 9 else 7.5
return black_score - white_score - komi
Passo 2: Codificacao de Recursos
Recursos de Entrada
# model/features.py
import numpy as np
def encode_board(board):
"""
Codifica o tabuleiro como entrada para a rede neural
Planos de recursos:
0: Pedras proprias
1: Pedras do oponente
2: Pontos vazios
3: Posicao da ultima jogada
4: Posicao da penultima jogada
5: Posicoes de jogadas legais
6: Preto a jogar (todo 1 ou todo 0)
"""
size = board.size
features = np.zeros((7, size, size), dtype=np.float32)
current = board.current_player
opponent = board.get_opponent(current)
# Posicoes basicas das pedras
features[0] = (board.board == current).astype(np.float32)
features[1] = (board.board == opponent).astype(np.float32)
features[2] = (board.board == board.EMPTY).astype(np.float32)
# Jogadas recentes
if len(board.history) >= 1:
x, y, _ = board.history[-1]
if x >= 0:
features[3, x, y] = 1.0
if len(board.history) >= 2:
x, y, _ = board.history[-2]
if x >= 0:
features[4, x, y] = 1.0
# Jogadas legais
for x in range(size):
for y in range(size):
if board.is_legal(x, y):
features[5, x, y] = 1.0
# De quem e a vez
if current == board.BLACK:
features[6] = np.ones((size, size), dtype=np.float32)
return features
Passo 3: Rede Neural
Arquitetura de Rede com Duas Cabecas
# model/network.py
import torch
import torch.nn as nn
import torch.nn.functional as F
class ResidualBlock(nn.Module):
"""Bloco residual"""
def __init__(self, channels):
super().__init__()
self.conv1 = nn.Conv2d(channels, channels, 3, padding=1)
self.bn1 = nn.BatchNorm2d(channels)
self.conv2 = nn.Conv2d(channels, channels, 3, padding=1)
self.bn2 = nn.BatchNorm2d(channels)
def forward(self, x):
residual = x
x = F.relu(self.bn1(self.conv1(x)))
x = self.bn2(self.conv2(x))
x = F.relu(x + residual)
return x
class PolicyValueNetwork(nn.Module):
"""Rede de duas cabecas Policy-Value"""
def __init__(self, board_size=9, input_channels=7, num_filters=64, num_blocks=4):
super().__init__()
self.board_size = board_size
# Convolucao inicial
self.conv_input = nn.Conv2d(input_channels, num_filters, 3, padding=1)
self.bn_input = nn.BatchNorm2d(num_filters)
# Blocos residuais
self.residual_blocks = nn.ModuleList([
ResidualBlock(num_filters) for _ in range(num_blocks)
])
# Policy Head
self.policy_conv = nn.Conv2d(num_filters, 2, 1)
self.policy_bn = nn.BatchNorm2d(2)
self.policy_fc = nn.Linear(2 * board_size * board_size, board_size * board_size + 1)
# Value Head
self.value_conv = nn.Conv2d(num_filters, 1, 1)
self.value_bn = nn.BatchNorm2d(1)
self.value_fc1 = nn.Linear(board_size * board_size, 64)
self.value_fc2 = nn.Linear(64, 1)
def forward(self, x):
# Trunk compartilhado
x = F.relu(self.bn_input(self.conv_input(x)))
for block in self.residual_blocks:
x = block(x)
# Policy Head
policy = F.relu(self.policy_bn(self.policy_conv(x)))
policy = policy.view(policy.size(0), -1)
policy = self.policy_fc(policy)
policy = F.log_softmax(policy, dim=1)
# Value Head
value = F.relu(self.value_bn(self.value_conv(x)))
value = value.view(value.size(0), -1)
value = F.relu(self.value_fc1(value))
value = torch.tanh(self.value_fc2(value))
return policy, value
def create_network(board_size=9):
"""Cria a rede"""
return PolicyValueNetwork(
board_size=board_size,
input_channels=7,
num_filters=64,
num_blocks=4
)
Passo 4: Implementacao do MCTS
Classe de No
# mcts/node.py
import numpy as np
class MCTSNode:
"""No MCTS"""
def __init__(self, prior=0.0):
self.visit_count = 0
self.value_sum = 0.0
self.prior = prior
self.children = {}
@property
def value(self):
if self.visit_count == 0:
return 0.0
return self.value_sum / self.visit_count
def expand(self, policy, legal_moves):
"""Expande o no"""
for move in legal_moves:
if move not in self.children:
idx = move[0] * 9 + move[1] if move[0] >= 0 else 81
self.children[move] = MCTSNode(prior=np.exp(policy[idx]))
def select_child(self, c_puct=1.5):
"""Seleciona no filho usando PUCT"""
best_score = -float('inf')
best_move = None
best_child = None
sqrt_total = np.sqrt(max(1, self.visit_count))
for move, child in self.children.items():
if child.visit_count > 0:
q_value = child.value
else:
q_value = 0.0
u_value = c_puct * child.prior * sqrt_total / (1 + child.visit_count)
score = q_value + u_value
if score > best_score:
best_score = score
best_move = move
best_child = child
return best_move, best_child
Implementacao da Busca
# mcts/search.py
import numpy as np
import torch
from .node import MCTSNode
class MCTS:
"""Busca em Arvore de Monte Carlo"""
def __init__(self, network, board_size=9, num_simulations=100, c_puct=1.5):
self.network = network
self.board_size = board_size
self.num_simulations = num_simulations
self.c_puct = c_puct
def search(self, board, add_noise=False):
"""Executa busca MCTS"""
root = MCTSNode()
# Avalia no raiz
policy, value = self.evaluate(board)
legal_moves = board.get_legal_moves()
root.expand(policy, legal_moves)
# Adiciona ruido de Dirichlet (durante treinamento)
if add_noise:
self.add_dirichlet_noise(root)
# Executa simulacoes
for _ in range(self.num_simulations):
node = root
scratch_board = board.copy()
path = [node]
# Selection
while node.children and scratch_board.get_legal_moves():
move, node = node.select_child(self.c_puct)
if move[0] >= 0:
scratch_board.play(move[0], move[1])
else:
scratch_board.pass_move()
path.append(node)
if scratch_board.is_game_over():
break
# Expansion + Evaluation
if not scratch_board.is_game_over():
policy, value = self.evaluate(scratch_board)
legal_moves = scratch_board.get_legal_moves()
if legal_moves:
node.expand(policy, legal_moves)
# Calcula valor da perspectiva do ponto de partida da busca
if scratch_board.is_game_over():
score = scratch_board.score()
value = 1.0 if score > 0 else (-1.0 if score < 0 else 0.0)
if board.current_player != scratch_board.BLACK:
value = -value
# Backpropagation
for node in reversed(path):
node.visit_count += 1
node.value_sum += value
value = -value
return root
def evaluate(self, board):
"""Avalia usando rede neural"""
from model.features import encode_board
features = encode_board(board)
features = torch.tensor(features).unsqueeze(0)
self.network.eval()
with torch.no_grad():
policy, value = self.network(features)
return policy[0].numpy(), value[0].item()
def add_dirichlet_noise(self, root, alpha=0.3, epsilon=0.25):
"""Adiciona ruido de exploracao"""
noise = np.random.dirichlet([alpha] * len(root.children))
for i, child in enumerate(root.children.values()):
child.prior = (1 - epsilon) * child.prior + epsilon * noise[i]
def get_policy(self, root, temperature=1.0):
"""Obtem politica do resultado da busca"""
visits = np.zeros(self.board_size ** 2 + 1)
for move, child in root.children.items():
idx = move[0] * self.board_size + move[1] if move[0] >= 0 else self.board_size ** 2
visits[idx] = child.visit_count
if temperature == 0:
policy = np.zeros_like(visits)
policy[np.argmax(visits)] = 1.0
else:
visits = visits ** (1 / temperature)
policy = visits / visits.sum()
return policy
def select_move(self, root, temperature=1.0):
"""Seleciona jogada"""
policy = self.get_policy(root, temperature)
idx = np.random.choice(len(policy), p=policy)
if idx == self.board_size ** 2:
return (-1, -1)
else:
return (idx // self.board_size, idx % self.board_size)
Passo 5: Auto-jogo
# training/self_play.py
import numpy as np
from game.board import Board
from model.features import encode_board
def self_play_game(mcts, temperature=1.0, temp_threshold=30):
"""Executa uma partida de auto-jogo"""
board = Board(size=9)
game_history = []
move_count = 0
while not board.is_game_over() and move_count < 200:
# Busca MCTS
root = mcts.search(board, add_noise=True)
# Obtem politica
temp = temperature if move_count < temp_threshold else 0.0
policy = mcts.get_policy(root, temp)
# Registra dados de treinamento
features = encode_board(board)
game_history.append({
'features': features,
'policy': policy,
'player': board.current_player
})
# Seleciona e executa jogada
move = mcts.select_move(root, temp)
if move[0] >= 0:
board.play(move[0], move[1])
else:
board.pass_move()
move_count += 1
# Calcula resultado
score = board.score()
winner = Board.BLACK if score > 0 else (Board.WHITE if score < 0 else 0)
# Marca valores
for data in game_history:
if winner == 0:
data['value'] = 0.0
elif data['player'] == winner:
data['value'] = 1.0
else:
data['value'] = -1.0
return game_history
def generate_training_data(mcts, num_games=100):
"""Gera dados de treinamento"""
all_data = []
for i in range(num_games):
print(f"Partida de auto-jogo {i+1}/{num_games}")
game_data = self_play_game(mcts)
all_data.extend(game_data)
return all_data
Passo 6: Treinador
# training/trainer.py
import torch
import torch.nn.functional as F
import numpy as np
from torch.utils.data import DataLoader, TensorDataset
class Trainer:
"""Treinador"""
def __init__(self, network, learning_rate=0.001):
self.network = network
self.optimizer = torch.optim.Adam(network.parameters(), lr=learning_rate)
def train_step(self, batch):
"""Passo de treinamento"""
features, target_policy, target_value = batch
self.network.train()
self.optimizer.zero_grad()
# Forward pass
pred_policy, pred_value = self.network(features)
# Calcula perda
policy_loss = F.kl_div(pred_policy, target_policy, reduction='batchmean')
value_loss = F.mse_loss(pred_value.squeeze(), target_value)
total_loss = policy_loss + value_loss
# Backward pass
total_loss.backward()
self.optimizer.step()
return {
'total_loss': total_loss.item(),
'policy_loss': policy_loss.item(),
'value_loss': value_loss.item()
}
def train_epoch(self, data, batch_size=32):
"""Treina uma epoca"""
# Prepara dados
features = np.array([d['features'] for d in data])
policies = np.array([d['policy'] for d in data])
values = np.array([d['value'] for d in data])
features = torch.tensor(features, dtype=torch.float32)
policies = torch.tensor(policies, dtype=torch.float32)
values = torch.tensor(values, dtype=torch.float32)
dataset = TensorDataset(features, policies, values)
loader = DataLoader(dataset, batch_size=batch_size, shuffle=True)
total_losses = []
for batch in loader:
losses = self.train_step(batch)
total_losses.append(losses['total_loss'])
return np.mean(total_losses)
def save(self, path):
"""Salva modelo"""
torch.save(self.network.state_dict(), path)
def load(self, path):
"""Carrega modelo"""
self.network.load_state_dict(torch.load(path))
Passo 7: Programa Principal
# main.py
from model.network import create_network
from mcts.search import MCTS
from training.self_play import generate_training_data
from training.trainer import Trainer
def main():
# Cria rede
network = create_network(board_size=9)
mcts = MCTS(network, board_size=9, num_simulations=100)
trainer = Trainer(network)
# Loop de treinamento
num_iterations = 100
games_per_iteration = 50
epochs_per_iteration = 10
for iteration in range(num_iterations):
print(f"\n=== Iteracao {iteration + 1}/{num_iterations} ===")
# Auto-jogo
print("Gerando partidas de auto-jogo...")
training_data = generate_training_data(mcts, num_games=games_per_iteration)
# Treinamento
print("Treinando...")
for epoch in range(epochs_per_iteration):
loss = trainer.train_epoch(training_data)
print(f" Epoca {epoch + 1}: loss = {loss:.4f}")
# Salva
trainer.save(f"model_iter_{iteration + 1}.pt")
print("\nTreinamento completo!")
if __name__ == "__main__":
main()
Execucao e Teste
Instalar Dependencias
pip install torch numpy
Executar Treinamento
python main.py
Saida Esperada
=== Iteracao 1/100 ===
Gerando partidas de auto-jogo...
Partida de auto-jogo 1/50
Partida de auto-jogo 2/50
...
Treinando...
Epoca 1: loss = 2.3456
Epoca 2: loss = 1.8765
...
Sugestoes de Melhoria
Melhorias de Curto Prazo
| Item de Melhoria | Descricao |
|---|---|
| Aumentar blocos residuais | 4 → 8 → 16 blocos |
| Aumentar numero de canais | 64 → 128 → 256 |
| Aumentar numero de simulacoes | 100 → 400 → 800 |
| Conjunto de treinamento maior | 50 → 200 → 1000 jogos/iteracao |
Melhorias de Longo Prazo
- Suportar tabuleiro 19x19
- Adicionar objetivos de treinamento auxiliares (previsao de territorio)
- Implementar auto-jogo paralelo
- Adicionar aceleracao por GPU
Leitura Adicional
- Arquitetura de Rede Neural Detalhada — Design de rede mais aprofundado
- Detalhes de Implementacao do MCTS — Tecnicas de busca avancadas
- Analise do Mecanismo de Treinamento do KataGo — Sistema de treinamento de producao
- Guia de Artigos Importantes — Fundamentos teoricos