在 ML 分类器中编码文本
Encoding text in ML classifier
我正在尝试构建 ML 模型。但是我很难理解在哪里应用编码。
请参阅下面的步骤和函数来复制我一直遵循的过程。
首先我将数据集分成训练和测试：
# Import the resampling package
from sklearn.naive_bayes import MultinomialNB
import string
from nltk.corpus import stopwords
import re
from sklearn.model_selection import train_test_split
from sklearn.feature_extraction.text import CountVectorizer
from nltk.tokenize import RegexpTokenizer
from sklearn.utils import resample
from sklearn.metrics import f1_score, precision_score, recall_score, accuracy_score
# Split into training and test sets
# Testing Count Vectorizer
X = df[['Text']]
y = df['Label']
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=40)
# Returning to one dataframe
training_set = pd.concat([X_train, y_train], axis=1)
现在我应用（欠）采样：
# Separating classes
spam = training_set[training_set.Label == 1]
not_spam = training_set[training_set.Label == 0]
# Undersampling the majority
undersample = resample(not_spam,
replace=True,
n_samples=len(spam), #set the number of samples to equal the number of the minority class
random_state=40)
# Returning to new training set
undersample_train = pd.concat([spam, undersample])
然后我应用了所选的算法：
full_result = pd.DataFrame(columns = ['Preprocessing', 'Model', 'Precision', 'Recall', 'F1-score', 'Accuracy'])
X, y = BOW(undersample_train)
full_result = full_result.append(training_naive(X_train, X_test, y_train, y_test, 'Count Vectorize'), ignore_index = True)
其中BOW定义如下
def BOW(data):
df_temp = data.copy(deep = True)
df_temp = basic_preprocessing(df_temp)
count_vectorizer = CountVectorizer(analyzer=fun)
count_vectorizer.fit(df_temp['Text'])
list_corpus = df_temp["Text"].tolist()
list_labels = df_temp["Label"].tolist()
X = count_vectorizer.transform(list_corpus)
return X, list_labels
basic_preprocessing定义如下：
def basic_preprocessing(df):
df_temp = df.copy(deep = True)
df_temp = df_temp.rename(index = str, columns = {'Clean_Titles_2': 'Text'})
df_temp.loc[:, 'Text'] = [text_prepare(x) for x in df_temp['Text'].values]
#le = LabelEncoder()
#le.fit(df_temp['medical_specialty'])
#df_temp.loc[:, 'class_label'] = le.transform(df_temp['medical_specialty'])
tokenizer = RegexpTokenizer(r'\w+')
df_temp["Tokens"] = df_temp["Text"].apply(tokenizer.tokenize)
return df_temp
其中 text_prepare 是：
def text_prepare(text):
REPLACE_BY_SPACE_RE = re.compile('[/(){}\[\]\|@,;]')
BAD_SYMBOLS_RE = re.compile('[^0-9a-z #+_]')
STOPWORDS = set(stopwords.words('english'))
text = text.lower()
text = REPLACE_BY_SPACE_RE.sub('', text) # replace REPLACE_BY_SPACE_RE symbols by space in text
text = BAD_SYMBOLS_RE.sub('', text) # delete symbols which are in BAD_SYMBOLS_RE from text
words = text.split()
i = 0
while i < len(words):
if words[i] in STOPWORDS:
words.pop(i)
else:
i += 1
text = ' '.join(map(str, words))# delete stopwords from text
return text
和
def training_naive(X_train_naive, X_test_naive, y_train_naive, y_test_naive, preproc):
clf = MultinomialNB() # Gaussian Naive Bayes
clf.fit(X_train_naive, y_train_naive)
res = pd.DataFrame(columns = ['Preprocessing', 'Model', 'Precision', 'Recall', 'F1-score', 'Accuracy'])
y_pred = clf.predict(X_test_naive)
f1 = f1_score(y_pred, y_test_naive, average = 'weighted')
pres = precision_score(y_pred, y_test_naive, average = 'weighted')
rec = recall_score(y_pred, y_test_naive, average = 'weighted')
acc = accuracy_score(y_pred, y_test_naive)
res = res.append({'Preprocessing': preproc, 'Model': 'Naive Bayes', 'Precision': pres,
'Recall': rec, 'F1-score': f1, 'Accuracy': acc}, ignore_index = True)
return res
如您所见，顺序是：
- 定义 text_prepare 用于文本清理；
- 定义basic_preprocessing;
- 定义弓；
- 将数据集拆分为训练和测试；
- 应用抽样；
- 应用算法。
我不明白的是如何正确编码文本以使算法正常工作。
我的数据集称为 df，列为：
Label Text Year
1 bla bla bla 2000
0 add some words 2012
1 this is just an example 1998
0 unfortunately the code does not work 2018
0 where should I apply the encoding? 2000
0 What am I missing here? 2005
我应用 BOW 时的顺序是错误的，因为我得到这个错误：ValueError: could not convert string to float: 'Expect a good results if ... '
我按照步骤（和代码=来自这个 link: kaggle.com/ruzarx/oversampling-smote-and-adasyn 。
然而，采样部分是错误的，因为它应该只对火车进行，所以在拆分之后。原则应该是：（1）拆分training/test； (2) 对训练集进行重采样，使模型得到均衡的数据训练； (3) 将模型应用于测试集并对其进行评估。
我很乐意提供更多信息、数据 and/or 代码，但我认为我已经提供了所有最相关的步骤。
非常感谢。
您需要一个测试 BOW 函数，该函数应重用在训练阶段构建的计数向量化器模型。
考虑使用管道来减少代码冗长。
from sklearn.naive_bayes import MultinomialNB
import string
from nltk.corpus import stopwords
import re
from sklearn.model_selection import train_test_split
from io import StringIO
from sklearn.feature_extraction.text import CountVectorizer
from nltk.tokenize import RegexpTokenizer
from sklearn.utils import resample
from sklearn.metrics import f1_score, precision_score, recall_score, accuracy_score
def fun(text):
remove_punc = [c for c in text if c not in string.punctuation]
remove_punc = ''.join(remove_punc)
cleaned = [w for w in remove_punc.split() if w.lower()
not in stopwords.words('english')]
return cleaned
# Testing Count Vectorizer
def BOW(data):
df_temp = data.copy(deep=True)
df_temp = basic_preprocessing(df_temp)
count_vectorizer = CountVectorizer(analyzer=fun)
count_vectorizer.fit(df_temp['Text'])
list_corpus = df_temp["Text"].tolist()
list_labels = df_temp["Label"].tolist()
X = count_vectorizer.transform(list_corpus)
return X, list_labels, count_vectorizer
def test_BOW(data, count_vectorizer):
df_temp = data.copy(deep=True)
df_temp = basic_preprocessing(df_temp)
list_corpus = df_temp["Text"].tolist()
list_labels = df_temp["Label"].tolist()
X = count_vectorizer.transform(list_corpus)
return X, list_labels
def basic_preprocessing(df):
df_temp = df.copy(deep=True)
df_temp = df_temp.rename(index=str, columns={'Clean_Titles_2': 'Text'})
df_temp.loc[:, 'Text'] = [text_prepare(x) for x in df_temp['Text'].values]
tokenizer = RegexpTokenizer(r'\w+')
df_temp["Tokens"] = df_temp["Text"].apply(tokenizer.tokenize)
return df_temp
def text_prepare(text):
REPLACE_BY_SPACE_RE = re.compile('[/(){}\[\]\|@,;]')
BAD_SYMBOLS_RE = re.compile('[^0-9a-z #+_]')
STOPWORDS = set(stopwords.words('english'))
text = text.lower()
# replace REPLACE_BY_SPACE_RE symbols by space in text
text = REPLACE_BY_SPACE_RE.sub('', text)
# delete symbols which are in BAD_SYMBOLS_RE from text
text = BAD_SYMBOLS_RE.sub('', text)
words = text.split()
i = 0
while i < len(words):
if words[i] in STOPWORDS:
words.pop(i)
else:
i += 1
text = ' '.join(map(str, words)) # delete stopwords from text
return text
s = """Label Text Year
1 bla bla bla 2000
0 add some words 2012
1 this is just an example 1998
0 unfortunately the code does not work 2018
0 where should I apply the encoding? 2000
0 What am I missing here? 2005"""
df = pd.read_csv(StringIO(s), sep='\s{2,}')
X = df[['Text']]
y = df['Label']
X_train, X_test, y_train, y_test = train_test_split(
X, y, test_size=0.2, random_state=40)
# Returning to one dataframe
training_set = pd.concat([X_train, y_train], axis=1)
# Separating classes
spam = training_set[training_set.Label == 1]
not_spam = training_set[training_set.Label == 0]
# Undersampling the majority
undersample = resample(not_spam,
replace=True,
# set the number of samples to equal the number of the minority class
n_samples=len(spam),
random_state=40)
# Returning to new training set
undersample_train = pd.concat([spam, undersample])
full_result = pd.DataFrame(columns=['Preprocessing', 'Model', 'Precision',
'Recall', 'F1-score', 'Accuracy'])
train_x, train_y, count_vectorizer = BOW(undersample_train)
testing_set = pd.concat([X_test, y_test], axis=1)
test_x, test_y = test_BOW(testing_set, count_vectorizer)
def training_naive(X_train_naive, X_test_naive, y_train_naive, y_test_naive, preproc):
clf = MultinomialNB() # Gaussian Naive Bayes
clf.fit(X_train_naive, y_train_naive)
res = pd.DataFrame(columns = ['Preprocessing', 'Model', 'Precision', 'Recall', 'F1-score', 'Accuracy'])
y_pred = clf.predict(X_test_naive)
f1 = f1_score(y_pred, y_test_naive, average = 'weighted')
pres = precision_score(y_pred, y_test_naive, average = 'weighted')
rec = recall_score(y_pred, y_test_naive, average = 'weighted')
acc = accuracy_score(y_pred, y_test_naive)
res = res.append({'Preprocessing': preproc, 'Model': 'Naive Bayes', 'Precision': pres,
'Recall': rec, 'F1-score': f1, 'Accuracy': acc}, ignore_index = True)
return res
full_result = full_result.append(training_naive(train_x, test_x, train_y, test_y, 'Count Vectorize'), ignore_index = True)
我正在尝试构建 ML 模型。但是我很难理解在哪里应用编码。 请参阅下面的步骤和函数来复制我一直遵循的过程。
首先我将数据集分成训练和测试：
# Import the resampling package
from sklearn.naive_bayes import MultinomialNB
import string
from nltk.corpus import stopwords
import re
from sklearn.model_selection import train_test_split
from sklearn.feature_extraction.text import CountVectorizer
from nltk.tokenize import RegexpTokenizer
from sklearn.utils import resample
from sklearn.metrics import f1_score, precision_score, recall_score, accuracy_score
# Split into training and test sets
# Testing Count Vectorizer
X = df[['Text']]
y = df['Label']
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=40)
# Returning to one dataframe
training_set = pd.concat([X_train, y_train], axis=1)
现在我应用（欠）采样：
# Separating classes
spam = training_set[training_set.Label == 1]
not_spam = training_set[training_set.Label == 0]
# Undersampling the majority
undersample = resample(not_spam,
replace=True,
n_samples=len(spam), #set the number of samples to equal the number of the minority class
random_state=40)
# Returning to new training set
undersample_train = pd.concat([spam, undersample])
然后我应用了所选的算法：
full_result = pd.DataFrame(columns = ['Preprocessing', 'Model', 'Precision', 'Recall', 'F1-score', 'Accuracy'])
X, y = BOW(undersample_train)
full_result = full_result.append(training_naive(X_train, X_test, y_train, y_test, 'Count Vectorize'), ignore_index = True)
其中BOW定义如下
def BOW(data):
df_temp = data.copy(deep = True)
df_temp = basic_preprocessing(df_temp)
count_vectorizer = CountVectorizer(analyzer=fun)
count_vectorizer.fit(df_temp['Text'])
list_corpus = df_temp["Text"].tolist()
list_labels = df_temp["Label"].tolist()
X = count_vectorizer.transform(list_corpus)
return X, list_labels
basic_preprocessing定义如下：
def basic_preprocessing(df):
df_temp = df.copy(deep = True)
df_temp = df_temp.rename(index = str, columns = {'Clean_Titles_2': 'Text'})
df_temp.loc[:, 'Text'] = [text_prepare(x) for x in df_temp['Text'].values]
#le = LabelEncoder()
#le.fit(df_temp['medical_specialty'])
#df_temp.loc[:, 'class_label'] = le.transform(df_temp['medical_specialty'])
tokenizer = RegexpTokenizer(r'\w+')
df_temp["Tokens"] = df_temp["Text"].apply(tokenizer.tokenize)
return df_temp
其中 text_prepare 是：
def text_prepare(text):
REPLACE_BY_SPACE_RE = re.compile('[/(){}\[\]\|@,;]')
BAD_SYMBOLS_RE = re.compile('[^0-9a-z #+_]')
STOPWORDS = set(stopwords.words('english'))
text = text.lower()
text = REPLACE_BY_SPACE_RE.sub('', text) # replace REPLACE_BY_SPACE_RE symbols by space in text
text = BAD_SYMBOLS_RE.sub('', text) # delete symbols which are in BAD_SYMBOLS_RE from text
words = text.split()
i = 0
while i < len(words):
if words[i] in STOPWORDS:
words.pop(i)
else:
i += 1
text = ' '.join(map(str, words))# delete stopwords from text
return text
和
def training_naive(X_train_naive, X_test_naive, y_train_naive, y_test_naive, preproc):
clf = MultinomialNB() # Gaussian Naive Bayes
clf.fit(X_train_naive, y_train_naive)
res = pd.DataFrame(columns = ['Preprocessing', 'Model', 'Precision', 'Recall', 'F1-score', 'Accuracy'])
y_pred = clf.predict(X_test_naive)
f1 = f1_score(y_pred, y_test_naive, average = 'weighted')
pres = precision_score(y_pred, y_test_naive, average = 'weighted')
rec = recall_score(y_pred, y_test_naive, average = 'weighted')
acc = accuracy_score(y_pred, y_test_naive)
res = res.append({'Preprocessing': preproc, 'Model': 'Naive Bayes', 'Precision': pres,
'Recall': rec, 'F1-score': f1, 'Accuracy': acc}, ignore_index = True)
return res
如您所见，顺序是：
- 定义 text_prepare 用于文本清理；
- 定义basic_preprocessing;
- 定义弓；
- 将数据集拆分为训练和测试；
- 应用抽样；
- 应用算法。
我不明白的是如何正确编码文本以使算法正常工作。 我的数据集称为 df，列为：
Label Text Year
1 bla bla bla 2000
0 add some words 2012
1 this is just an example 1998
0 unfortunately the code does not work 2018
0 where should I apply the encoding? 2000
0 What am I missing here? 2005
我应用 BOW 时的顺序是错误的，因为我得到这个错误：ValueError: could not convert string to float: 'Expect a good results if ... '
我按照步骤（和代码=来自这个 link: kaggle.com/ruzarx/oversampling-smote-and-adasyn 。 然而，采样部分是错误的，因为它应该只对火车进行，所以在拆分之后。原则应该是：（1）拆分training/test； (2) 对训练集进行重采样，使模型得到均衡的数据训练； (3) 将模型应用于测试集并对其进行评估。
我很乐意提供更多信息、数据 and/or 代码，但我认为我已经提供了所有最相关的步骤。
非常感谢。
您需要一个测试 BOW 函数，该函数应重用在训练阶段构建的计数向量化器模型。
考虑使用管道来减少代码冗长。
from sklearn.naive_bayes import MultinomialNB
import string
from nltk.corpus import stopwords
import re
from sklearn.model_selection import train_test_split
from io import StringIO
from sklearn.feature_extraction.text import CountVectorizer
from nltk.tokenize import RegexpTokenizer
from sklearn.utils import resample
from sklearn.metrics import f1_score, precision_score, recall_score, accuracy_score
def fun(text):
remove_punc = [c for c in text if c not in string.punctuation]
remove_punc = ''.join(remove_punc)
cleaned = [w for w in remove_punc.split() if w.lower()
not in stopwords.words('english')]
return cleaned
# Testing Count Vectorizer
def BOW(data):
df_temp = data.copy(deep=True)
df_temp = basic_preprocessing(df_temp)
count_vectorizer = CountVectorizer(analyzer=fun)
count_vectorizer.fit(df_temp['Text'])
list_corpus = df_temp["Text"].tolist()
list_labels = df_temp["Label"].tolist()
X = count_vectorizer.transform(list_corpus)
return X, list_labels, count_vectorizer
def test_BOW(data, count_vectorizer):
df_temp = data.copy(deep=True)
df_temp = basic_preprocessing(df_temp)
list_corpus = df_temp["Text"].tolist()
list_labels = df_temp["Label"].tolist()
X = count_vectorizer.transform(list_corpus)
return X, list_labels
def basic_preprocessing(df):
df_temp = df.copy(deep=True)
df_temp = df_temp.rename(index=str, columns={'Clean_Titles_2': 'Text'})
df_temp.loc[:, 'Text'] = [text_prepare(x) for x in df_temp['Text'].values]
tokenizer = RegexpTokenizer(r'\w+')
df_temp["Tokens"] = df_temp["Text"].apply(tokenizer.tokenize)
return df_temp
def text_prepare(text):
REPLACE_BY_SPACE_RE = re.compile('[/(){}\[\]\|@,;]')
BAD_SYMBOLS_RE = re.compile('[^0-9a-z #+_]')
STOPWORDS = set(stopwords.words('english'))
text = text.lower()
# replace REPLACE_BY_SPACE_RE symbols by space in text
text = REPLACE_BY_SPACE_RE.sub('', text)
# delete symbols which are in BAD_SYMBOLS_RE from text
text = BAD_SYMBOLS_RE.sub('', text)
words = text.split()
i = 0
while i < len(words):
if words[i] in STOPWORDS:
words.pop(i)
else:
i += 1
text = ' '.join(map(str, words)) # delete stopwords from text
return text
s = """Label Text Year
1 bla bla bla 2000
0 add some words 2012
1 this is just an example 1998
0 unfortunately the code does not work 2018
0 where should I apply the encoding? 2000
0 What am I missing here? 2005"""
df = pd.read_csv(StringIO(s), sep='\s{2,}')
X = df[['Text']]
y = df['Label']
X_train, X_test, y_train, y_test = train_test_split(
X, y, test_size=0.2, random_state=40)
# Returning to one dataframe
training_set = pd.concat([X_train, y_train], axis=1)
# Separating classes
spam = training_set[training_set.Label == 1]
not_spam = training_set[training_set.Label == 0]
# Undersampling the majority
undersample = resample(not_spam,
replace=True,
# set the number of samples to equal the number of the minority class
n_samples=len(spam),
random_state=40)
# Returning to new training set
undersample_train = pd.concat([spam, undersample])
full_result = pd.DataFrame(columns=['Preprocessing', 'Model', 'Precision',
'Recall', 'F1-score', 'Accuracy'])
train_x, train_y, count_vectorizer = BOW(undersample_train)
testing_set = pd.concat([X_test, y_test], axis=1)
test_x, test_y = test_BOW(testing_set, count_vectorizer)
def training_naive(X_train_naive, X_test_naive, y_train_naive, y_test_naive, preproc):
clf = MultinomialNB() # Gaussian Naive Bayes
clf.fit(X_train_naive, y_train_naive)
res = pd.DataFrame(columns = ['Preprocessing', 'Model', 'Precision', 'Recall', 'F1-score', 'Accuracy'])
y_pred = clf.predict(X_test_naive)
f1 = f1_score(y_pred, y_test_naive, average = 'weighted')
pres = precision_score(y_pred, y_test_naive, average = 'weighted')
rec = recall_score(y_pred, y_test_naive, average = 'weighted')
acc = accuracy_score(y_pred, y_test_naive)
res = res.append({'Preprocessing': preproc, 'Model': 'Naive Bayes', 'Precision': pres,
'Recall': rec, 'F1-score': f1, 'Accuracy': acc}, ignore_index = True)
return res
full_result = full_result.append(training_naive(train_x, test_x, train_y, test_y, 'Count Vectorize'), ignore_index = True)