Python(PyTorch): TypeError: string indices must be integers

Question

我编写了以下代码来在我的数据集上训练 bert 模型，但是当我执行它时，我在实现 tqdm 的部分遇到了错误。我在下面编写了完整的训练代码，并对错误进行了完整描述。如何解决这个问题？

代码

型号

TRANSFORMERS = {
    "bert-multi-cased": (BertModel, BertTokenizer, "bert-base-uncased"),
}

class Transformer(nn.Module):
    def __init__(self, model, num_classes=1):
        """
        Constructor
    
    Arguments:
        model {string} -- Transformer to build the model on. Expects "camembert-base".
        num_classes {int} -- Number of classes (default: {1})
    """
    super().__init__()
    self.name = model

    model_class, tokenizer_class, pretrained_weights = TRANSFORMERS[model]

    bert_config = BertConfig.from_json_file(MODEL_PATHS[model] + 'bert_config.json')
    bert_config.output_hidden_states = True
    
    self.transformer = BertModel(bert_config)

    self.nb_features = self.transformer.pooler.dense.out_features

    self.pooler = nn.Sequential(
        nn.Linear(self.nb_features, self.nb_features), 
        nn.Tanh(),
    )

    self.logit = nn.Linear(self.nb_features, num_classes)

def forward(self, tokens):
    """
    Usual torch forward function
    
    Arguments:
        tokens {torch tensor} -- Sentence tokens
    
    Returns:
        torch tensor -- Class logits
    """
    _, _, hidden_states = self.transformer(
        tokens, attention_mask=(tokens > 0).long()
    )

    hidden_states = hidden_states[-1][:, 0] # Use the representation of the first token of the last layer

    ft = self.pooler(hidden_states)

    return self.logit(ft)

培训

def fit(model, train_dataset, val_dataset, epochs=1, batch_size=8, warmup_prop=0, lr=5e-4):
    
    train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)
    val_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False)

    optimizer = AdamW(model.parameters(), lr=lr)
    
    num_warmup_steps = int(warmup_prop * epochs * len(train_loader))
    num_training_steps = epochs * len(train_loader)
    
    scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps, num_training_steps)

    loss_fct = nn.BCEWithLogitsLoss(reduction='mean').cuda()
    
    for epoch in range(epochs):
        model.train()
        start_time = time.time()
        
        optimizer.zero_grad()
        avg_loss = 0
        
        for step, (x, y_batch) in tqdm(enumerate(train_loader), total=len(train_loader)):
            
            y_pred = model(x.to(device))
            
            loss = loss_fct(y_pred.view(-1).float(), y_batch.float().to(device))
            loss.backward()
            avg_loss += loss.item() / len(train_loader)

            xm.optimizer_step(optimizer, barrier=True)
            #optimizer.step()
            scheduler.step()
            model.zero_grad()
            optimizer.zero_grad()
                
        model.eval()
        preds = []
        truths = []
        avg_val_loss = 0.

        with torch.no_grad():
            for x, y_batch in tqdm(val_loader):                
                y_pred = model(x.to(device))
                loss = loss_fct(y_pred.detach().view(-1).float(), y_batch.float().to(device))
                avg_val_loss += loss.item() / len(val_loader)
                
                probs = torch.sigmoid(y_pred).detach().cpu().numpy()
                preds += list(probs.flatten())
                truths += list(y_batch.numpy().flatten())
            score = roc_auc_score(truths, preds)
            
        
        dt = time.time() - start_time
        lr = scheduler.get_last_lr()[0]
        print(f'Epoch {epoch + 1}/{epochs} \t lr={lr:.1e} \t t={dt:.0f}s \t loss={avg_loss:.4f} \t val_loss={avg_val_loss:.4f} \t val_auc={score:.4f}')

错误

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<timed eval> in <module>

<ipython-input-19-e47eae808597> in fit(model, train_dataset, val_dataset, epochs, batch_size, warmup_prop, lr)
     22         for step, (x, y_batch) in tqdm(enumerate(train_loader), total=len(train_loader)):
     23 
---> 24             y_pred = model(x.to(device))
     25 
     26             loss = loss_fct(y_pred.view(-1).float(), y_batch.float().to(device))

/opt/conda/lib/python3.6/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
    724             result = self._slow_forward(*input, **kwargs)
    725         else:
--> 726             result = self.forward(*input, **kwargs)
    727         for hook in itertools.chain(
    728                 _global_forward_hooks.values(),

<ipython-input-11-2002cc7ec843> in forward(self, tokens)
     41         )
     42 
---> 43         hidden_states = hidden_states[-1][:, 0] # Use the representation of the first token of the last layer
     44 
     45         ft = self.pooler(hidden_states)

TypeError: string indices must be integers

Answer 1

您的代码是为旧版本的变形金刚库设计的：

AttributeError: 'str' object has no attribute 'dim' in pytorch

因此，您需要降级到 3.0.0 版，或者调整代码以处理 bert 的新格式输出。

Python(PyTorch): TypeError: string indices must be integers

Python(PyTorch): TypeError: string indices must be integers

python

python-3.x

deep-learning

pytorch

tqdm

代码

型号

培训