Add development folder

development/hate-speech-classification.ipynb

@@ -0,0 +1,815 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "id": "c99a9e2c",
+ "metadata": {},
+ "source": [
+ "# Import the necessary libraries"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "bb19171c",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import os\n",
+ "import pickle\n",
+ "import re\n",
+ "import string\n",
+ "from collections.abc import Iterable\n",
+ "\n",
+ "import keras\n",
+ "import matplotlib.pyplot as plt\n",
+ "import nltk\n",
+ "import numpy as np\n",
+ "import pandas as pd\n",
+ "import seaborn as sns\n",
+ "from keras.callbacks import EarlyStopping, ModelCheckpoint\n",
+ "from keras.layers import (LSTM, Activation, Dense, Dropout, Embedding, Input,\n",
+ " SpatialDropout1D)\n",
+ "from keras.models import Model, Sequential\n",
+ "from keras.optimizers import RMSprop\n",
+ "from keras.preprocessing import sequence\n",
+ "from keras.preprocessing.text import Tokenizer\n",
+ "from keras.utils import pad_sequences, to_categorical\n",
+ "from nltk.corpus import stopwords\n",
+ "from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer\n",
+ "from sklearn.metrics import confusion_matrix\n",
+ "from sklearn.model_selection import train_test_split\n",
+ "\n",
+ "nltk.download('stopwords')\n",
+ "pd.set_option('display.max_rows', None)\n",
+ "pd.set_option('display.max_columns', None)\n",
+ "pd.set_option('display.max_colwidth', 255)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "77ee39a1",
+ "metadata": {},
+ "source": [
+ "# Dataset"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "2289c89e",
+ "metadata": {},
+ "source": [
+ "## Dataset 1"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "70bddc47",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "df1 = pd.read_csv(\"/kaggle/input/twitter-hate-speech/train_E6oV3lV.csv\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "e407435d",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "df1.head()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "4ea10f67",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "sns.countplot(x='label', data=df1)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "4bef62c7",
+ "metadata": {},
+ "source": [
+ "From the above plot we can see that classes are imbalanced, we will fix it later."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "252edcb4",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Checking the shape of the data\n",
+ "df1.shape"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "0e256090",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Cheking if null values are present in the dataset or not.\n",
+ "df1.isnull().sum()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "8d0cc255",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Drop unnecessary columns\n",
+ "df1.drop('id', axis=1, inplace=True)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "963f8229",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "df1.head()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "5767e166",
+ "metadata": {},
+ "source": [
+ "## Dataset 2"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "bd8dde1a",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "df2 = pd.read_csv(\n",
+ " \"/kaggle/input/hate-speech-and-offensive-language-dataset/labeled_data.csv\")\n",
+ "df2.head()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "a8a4a332",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "df2.shape"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "b66a6907",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "df2.isnull().sum()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "49db9d8d",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Drop the columns which are not required for us.\n",
+ "df2.drop(['Unnamed: 0', 'count', 'hate_speech',\n",
+ " 'offensive_language', 'neither'], axis=1, inplace=True)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "48981e64",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "df2.head()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "97b0500b",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# All the unique class labels\n",
+ "df2['class'].unique()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "71971d95",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Plotting the countplot for our new dataset\n",
+ "sns.countplot(x='class', data=df2)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "1ce30639",
+ "metadata": {},
+ "source": [
+ "- class 0 - hate speech; class 1 - offensive language; class 2 - neither"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "ce04999f",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Merge class 0 and 1 into 1. Class 1 now represents hate speech\n",
+ "df2[\"class\"].replace({0: 1}, inplace=True)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "499d5336",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "df2[\"class\"].unique()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "2cb91824",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "sns.countplot(x=\"class\", data=df2)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "9bf7ba3a",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Replace the value of 2 to 0.Class 0 is now \"No hate\"\n",
+ "df2[\"class\"].replace({2: 0}, inplace=True)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "16bc2c3e",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "sns.countplot(x='class', data=df2)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "d5834f0e",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Rename 'class' to label\n",
+ "df2.rename(columns={'class': 'label'}, inplace=True)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "0e6a6a19",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "df2.head()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "b76458f2",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "df2.iloc[0]['tweet']"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "42a65071",
+ "metadata": {},
+ "source": [
+ "## Merge df1 and df2"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "77c925a5",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "df = pd.concat([df1, df2])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "b81eef43",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "df.head()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "952ef123",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "sns.countplot(x='label', data=df)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "608c3277",
+ "metadata": {},
+ "source": [
+ "Now we can see that the problem of imbalace data has been solved."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "293d0d21",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "df.shape"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "4d8117e1",
+ "metadata": {},
+ "source": [
+ "## Data cleaning"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "e76a3db9",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Apply regex and do cleaning.\n",
+ "def clean_text(words: str) -> str:\n",
+ " words = str(words).lower()\n",
+ " words = re.sub('\\[.*?\\]', '', words)\n",
+ " words = re.sub('https?://\\S+|www\\.\\S+', '', words)\n",
+ " words = re.sub('<.*?>+', '', words)\n",
+ " words = re.sub(r'@\\w+', '', words)\n",
+ " words = re.sub('[%s]' % re.escape(string.punctuation), '', words)\n",
+ " words = re.sub('\\n', '', words)\n",
+ " words = re.sub('\\w*\\d\\w*', '', words)\n",
+ "\n",
+ " stopword = set(stopwords.words('english'))\n",
+ " words = ' '.join(\n",
+ " [word for word in words.split(' ') if word not in stopword])\n",
+ "\n",
+ " stemmer = nltk.SnowballStemmer(\"english\")\n",
+ " words = ' '.join([stemmer.stem(word) for word in words.split(' ')])\n",
+ "\n",
+ " return words"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "fd98ec5a",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Apply the data_cleaning on the data.\n",
+ "df_cleaned = df.copy()\n",
+ "df_cleaned['tweet'] = df['tweet'].apply(clean_text)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "b5c6a309",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "df_cleaned['tweet'][1]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "3df4b3e0",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "df_cleaned.head(10)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "39e9dff5",
+ "metadata": {},
+ "source": [
+ "## Train test split"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "060e1f76",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "x = df_cleaned['tweet']\n",
+ "y = df_cleaned['label']"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "5b39fbd9",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Split the data into train and test\n",
+ "x_train, x_test, y_train, y_test = train_test_split(x, y, random_state=42)\n",
+ "print(len(x_train), len(y_train))\n",
+ "print(len(x_test), len(y_test))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "29be47f4",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "type(x_test), type(y_test), type(x_train), type(y_train)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "402ecb50",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "len(x_test)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "0187c473",
+ "metadata": {},
+ "source": [
+ "## Tokenization and padding"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "cc49a7f7",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def tokenize_and_pad(text_list: Iterable[str], tokenizer: Tokenizer, max_len: int) -> np.ndarray[np.str_]:\n",
+ " sequences = tokenizer.texts_to_sequences(text_list)\n",
+ " sequences_matrix = pad_sequences(sequences, maxlen=max_len)\n",
+ " return sequences_matrix"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "e4329001",
+ "metadata": {
+ "lines_to_next_cell": 2
+ },
+ "outputs": [],
+ "source": [
+ "max_words = 50000\n",
+ "max_len = 300\n",
+ "\n",
+ "tokenizer = Tokenizer(num_words=max_words)\n",
+ "tokenizer.fit_on_texts(x_train)\n",
+ "\n",
+ "x_train_tokenized = tokenize_and_pad(x_train, tokenizer, max_len)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "21261eee",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "with open('tokenizer.pickle', 'wb') as handle:\n",
+ " pickle.dump(tokenizer, handle, protocol=pickle.HIGHEST_PROTOCOL)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "5833c859",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "x_train_tokenized"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "811f8996",
+ "metadata": {},
+ "source": [
+ "# Model"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "b42ceb66",
+ "metadata": {},
+ "source": [
+ "## Model architecture"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "15e9d814",
+ "metadata": {
+ "lines_to_next_cell": 2
+ },
+ "outputs": [],
+ "source": [
+ "# Creating model architecture.\n",
+ "model = Sequential()\n",
+ "model.add(Embedding(max_words, 100, input_length=max_len))\n",
+ "model.add(SpatialDropout1D(0.2))\n",
+ "model.add(LSTM(100, dropout=0.2, recurrent_dropout=0.2))\n",
+ "model.add(Dense(1, activation='sigmoid'))\n",
+ "\n",
+ "model.summary()\n",
+ "\n",
+ "model.compile(loss='binary_crossentropy',\n",
+ " optimizer=RMSprop(), metrics=['accuracy'])"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "ae55985d",
+ "metadata": {},
+ "source": [
+ "## Callbacks"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "9065382d",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "early_stopping_callback = EarlyStopping(\n",
+ " monitor='val_loss', # Metric to monitor (e.g., validation loss)\n",
+ " patience=3, # Number of epochs with no improvement to wait\n",
+ " restore_best_weights=True # Restore model weights to the best achieved during training\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "90fb2dbf",
+ "metadata": {},
+ "source": [
+ "## Training\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "fb3a5153",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# starting model training\n",
+ "history = model.fit(\n",
+ " x_train_tokenized, y_train,\n",
+ " batch_size=128,\n",
+ " epochs=20,\n",
+ " validation_split=0.2,\n",
+ " callbacks=[early_stopping_callback]\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "b509694a",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "model.save(\"model.h5\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "01484e53",
+ "metadata": {},
+ "source": [
+ "## Evaluation and testing"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "86a6cd51",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "test_sequences = tokenizer.texts_to_sequences(x_test)\n",
+ "test_sequences_matrix = pad_sequences(test_sequences, maxlen=max_len)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "7674863a",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Model evaluation\n",
+ "accr = model.evaluate(test_sequences_matrix, y_test)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "03f93f02",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "lstm_prediction = model.predict(test_sequences_matrix)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "2b04a6f5",
+ "metadata": {
+ "lines_to_next_cell": 2
+ },
+ "outputs": [],
+ "source": [
+ "res = []\n",
+ "for prediction in lstm_prediction:\n",
+ " if prediction[0] < 0.5:\n",
+ " res.append(0)\n",
+ " else:\n",
+ " res.append(1)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "20ec485c",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "print(confusion_matrix(y_test, res))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "0062900e",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "load_model = keras.models.load_model(\"model.h5\")\n",
+ "with open('tokenizer.pickle', 'rb') as handle:\n",
+ " load_tokenizer = pickle.load(handle)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "5612cac0",
+ "metadata": {
+ "lines_to_next_cell": 2
+ },
+ "outputs": [],
+ "source": [
+ "# Let's test our model on custom data.\n",
+ "test = 'humans are idiots'\n",
+ "\n",
+ "\n",
+ "def clean_text(text):\n",
+ " print(text)\n",
+ " text = str(text).lower()\n",
+ " text = re.sub('\\[.*?\\]', '', text)\n",
+ " text = re.sub('https?://\\S+|www\\.\\S+', '', text)\n",
+ " text = re.sub('<.*?>+', '', text)\n",
+ " text = re.sub('[%s]' % re.escape(string.punctuation), '', text)\n",
+ " text = re.sub('\\n', '', text)\n",
+ " text = re.sub('\\w*\\d\\w*', '', text)\n",
+ " print(text)\n",
+ " text = [word for word in text.split(' ') if word not in stopword]\n",
+ " text = \" \".join(text)\n",
+ " text = [stemmer.stem(word) for word in text.split(' ')]\n",
+ " text = \" \".join(text)\n",
+ " return text\n",
+ "\n",
+ "\n",
+ "test = [clean_text(test)]\n",
+ "print(test)\n",
+ "seq = load_tokenizer.texts_to_sequences(test)\n",
+ "padded = pad_sequences(seq, maxlen=300)\n",
+ "print(seq)\n",
+ "pred = load_model.predict(padded)\n",
+ "print(\"pred\", pred)\n",
+ "if pred < 0.5:\n",
+ " print(\"no hate\")\n",
+ "else:\n",
+ " print(\"hate and abusive\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "d90fb1eb",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "model.summary()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "e564ae3e",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "while True:\n",
+ " pass"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "41301aee",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# https://www.kaggle.com/soumyaprabhamaiti/hate-speech-classification/edit"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

development/requirements_dev.txt

@@ -0,0 +1,8 @@
+tensorflow
+numpy
+pandas
+seaborn
+matplotlib
+gradio
+nltk
+jupytext