Natural Language Processing with RNNs - Toxic Comment Classification

import os

os.environ['KAGGLE_CONFIG_DIR'] = '.'

!kaggle competitions download -c jigsaw-toxic-comment-classification-challenge

!unzip jigsaw-toxic-comment-classification-challenge.zip -d data

import pandas as pd

raw_df = pd.read_csv('data/train.csv.zip')
test_df = pd.read_csv('data/test.csv.zip')
sub_df = pd.read_csv('data/sample_submission.csv.zip')

raw_df.info()

raw_df.sample(10)

raw_df.comment_text.values[0]

target_cols = ['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']

for col in target_cols:
    print(raw_df[col].value_counts(normalize=True))

test_df

sub_df

from torchtext.data.utils import get_tokenizer

tokenizer = get_tokenizer('basic_english')

sample_comment = raw_df.comment_text.values[0]

sample_comment

sample_comment_tokens = tokenizer(sample_comment)
sample_comment_tokens[:10]

from torchtext.vocab import build_vocab_from_iterator

comment_tokens = raw_df.comment_text.map(tokenizer)

VOCAB_SIZE = 1500

unk_token = '<unk>'
pad_token = '<pad>'

vocab = build_vocab_from_iterator(comment_tokens, 
                                  specials=[unk_token, pad_token], 
                                  max_tokens=VOCAB_SIZE)

vocab[unk_token]

vocab.set_default_index(vocab[unk_token])

vocab['this']

vocab['harcore']

sample_indices = vocab.lookup_indices(sample_comment_tokens)
sample_indices[:10]

sample_comment_recovered = vocab.lookup_tokens(sample_indices)
sample_comment_recovered[:10]

raw_df.comment_text.sample(1000).map(tokenizer).map(len).plot(kind='hist')

MAX_LENGTH = 150

def pad_tokens(tokens):
    if (len(tokens) >= MAX_LENGTH):
        return tokens[:MAX_LENGTH]
    else:
        return tokens + [pad_token] * (MAX_LENGTH - len(tokens))

import torch

from torch.utils.data import Dataset

class JigsawDataset(Dataset):
    def __init__(self, df, is_test=False):
        self.df = df
        self.is_test = is_test

    def __getitem__(self, index):
        comment_text = self.df.comment_text.values[index]
        comment_tokens = pad_tokens(tokenizer(comment_text))
        input = torch.tensor(vocab.lookup_indices(comment_tokens))
        if self.is_test:
            target = torch.tensor([0,0,0,0,0,0]).float()
        else:
            target = torch.tensor(self.df[target_cols].values[index]).float()
        return input, target

    def __len__(self):
        return len(self.df)

raw_ds = JigsawDataset(raw_df)

raw_df.head(5)

raw_ds[0]

from torch.utils.data import random_split

VAL_FRAC = 0.25

train_ds, val_ds = random_split(raw_ds, [1-VAL_FRAC, VAL_FRAC])

len(raw_ds), len(train_ds), len(val_ds)

test_ds = JigsawDataset(test_df, is_test=True)

test_df.head(5)

test_ds[0]

from torch.utils.data import DataLoader

BATCH_SIZE = 256

train_dl = DataLoader(train_ds, batch_size=BATCH_SIZE, shuffle=True, num_workers=8, pin_memory=True)
val_dl = DataLoader(val_ds, batch_size=BATCH_SIZE*2, num_workers=8, pin_memory=True)
test_dl = DataLoader(test_ds, batch_size=BATCH_SIZE*2, num_workers=8, pin_memory=True)

for batch in train_dl:
    b_inputs, b_targets = batch
    print('b_input.shape', b_inputs.shape)
    print('b_targets.shape', b_targets.shape)

    break

import torch.nn as nn
import torch.functional as F

vocab[pad_token]

emb_layer = nn.Embedding(VOCAB_SIZE, 256, 1)

rnn_layer = nn.RNN(256, 128, 1, batch_first=True)

for batch in train_dl:
    b_inputs, b_targets = batch
    print('b_input.shape', b_inputs.shape)
    print('b_targets.shape', b_targets.shape)

    emb_out = emb_layer(b_inputs)
    print('emb_out.shape', emb_out.shape)

    rnn_out, hn = rnn_layer(emb_out)
    print('rnn_out.shape', rnn_out.shape)
    print('hn.shape', hn.shape)

    break

!pip install pytorch_lightning --quiet

import pytorch_lightning as pl

import torch.nn.functional as F

import numpy as np

class JigsawModel(pl.LightningModule):
    def __init__(self):
        super().__init__()
        self.emb = nn.Embedding(VOCAB_SIZE, 256, 1)
        self.lstm = nn.LSTM(256, 128, 1, batch_first=True)
        self.linear = nn.Linear(128, 6)
        self.learning_rate = 0.001

    def forward(self, x):
        out = self.emb(x)
        out, hn = self.lstm(out)
        out = F.relu(out[:,-1,:])
        out = self.linear(out)
        return out

    def training_step(self, batch, batch_idx):
        inputs, targets = batch
        outputs = self(inputs)
        probs = torch.sigmoid(outputs)
        loss = F.binary_cross_entropy(probs, targets)
        return loss

    def validation_step(self, batch, batch_idx):
        inputs, targets = batch
        outputs = self(inputs)
        probs = torch.sigmoid(outputs)
        loss = F.binary_cross_entropy(probs, targets)
        return loss.item()

    def validation_epoch_end(self, validation_step_outputs):
        loss = np.mean(validation_step_outputs)
        print("Epoch #{}; Loss: {:4f} ".format(self.current_epoch, loss)) 

    def predict_step(self, batch, batch_idx):
        inputs, targets = batch
        outputs = self(inputs)
        probs = torch.sigmoid(outputs)
        return probs       

    def configure_optimizers(self):
        return torch.optim.Adam(self.parameters(), lr=self.learning_rate)

model = JigsawModel()

for batch in train_dl:
    b_inputs, b_targets = batch
    print('b_input.shape', b_inputs.shape)
    print('b_targets.shape', b_targets.shape)

    outputs = model(b_inputs)
    print('outputs.shape', outputs.shape)

    probs = torch.sigmoid(outputs)
    loss = F.binary_cross_entropy(probs, b_targets)
    print('Loss', loss)
    break

trainer = pl.Trainer(max_epochs=3, accelerator='gpu', auto_lr_find=True)

trainer.tune(model, train_dl)

model.learning_rate

trainer.fit(model, train_dl, val_dl)

test_df.head(5)

for batch in test_dl:
    b_inputs, b_targets = batch
    print('b_inputs.shape', b_inputs.shape)
    print('b_targets.shape', b_targets.shape)
    break

test_preds = trainer.predict(model, test_dl)

test_preds = torch.cat(test_preds)

test_preds.shape

test_preds

test_probs = torch.sigmoid(test_preds)

sub_df[target_cols] = test_probs.detach().cpu().numpy()

sub_df

sub_df.to_csv('submission.csv', index=None)

!head submission.csv

sample_df = raw_df.sample(10)

sample_df

sample_ds = JigsawDataset(sample_df)

sample_dl = DataLoader(sample_ds, batch_size=10)

sample_preds = trainer.predict(model, sample_dl)

sample_probs = torch.sigmoid(sample_preds[0])

sample_probs

sample_df