如何使用 FeatureUnion 转换 PipeLine 中的多个特征?
How to transform multiple features in a PipeLine using FeatureUnion?
我有一个 pandas 数据框,其中包含有关用户发送的消息的信息。
对于我的模型,我有兴趣预测消息的丢失收件人,即给定消息的收件人 A、B、C 我想预测还有谁应该是收件人的一部分。
我正在使用 OneVsRestClassifier 和 LinearSVC 进行多标签分类。
对于功能,我想使用邮件的收件人。主题和正文。
由于收件人是一个用户列表,我想使用 MultiLabelBinarizer 转换该列。对于主题和正文,我想使用 TFIDF
我的输入 pickle 文件有如下数据:所有值都是字符串,除了 recipients 是一个 set()
[[message_id,sent_time,subject,body,set(recipients),message_type, is_sender]]
我在管道中使用带有自定义转换器的特征联合来实现如下。
from sklearn.preprocessing import MultiLabelBinarizer
from sklearn.base import TransformerMixin, BaseEstimator
from sklearn.pipeline import Pipeline, FeatureUnion
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.multiclass import OneVsRestClassifier
from sklearn.model_selection import GridSearchCV
from sklearn.multiclass import OneVsRestClassifier
from sklearn.linear_model import SGDClassifier
from sklearn.svm import SVC, LinearSVC
import pickle
import pandas as pd
import numpy as np
if __name__ == "__main__":
class ColumnSelector(BaseEstimator, TransformerMixin):
def __init__(self, column):
self.column = column
def fit(self, X, y=None, **fit_params):
return self
def transform(self, X, y=None, **fit_params):
return X[self.column]
class MultiLabelTransformer(BaseEstimator, TransformerMixin):
def __init__(self, column):
self.column = column
def fit(self, X, y=None):
return self
def transform(self, X):
mlb = MultiLabelBinarizer()
return mlb.fit_transform(X[self.column])
pipeline = Pipeline([
('features', FeatureUnion([
('subject_tfidf', Pipeline([
('selector', ColumnSelector(column='Subject')),
('tfidf', TfidfVectorizer(min_df=0.0025, ngram_range=(1, 4)))
])),
('body_tfidf', Pipeline([
('selector', ColumnSelector(column='Body')),
('tfidf', TfidfVectorizer(min_df=0.0025, ngram_range=(1, 4)))
])),
('recipients_binarizer', Pipeline([
('multi_label', MultiLabelTransformer(column='CoRecipients'))
])),
])),
('classifier', OneVsRestClassifier(LinearSVC(), n_jobs=-1))
])
top_recips = ['A', 'B', 'C, 'D]
corpus_data = pickle.load(
open("E:\Data\messages_items.pkl", "rb"))
df = pd.DataFrame(corpus_data, columns=[
'MessageId', 'SentTime', 'Subject', 'Body', 'Recipients', 'MessageType', 'IsSender'])
df = df.dropna()
# add co recipients and top recipients columns
df['CoRecipients'] = df['Recipients'].apply(
lambda r: [x for x in r if x not in top_recips])
df['TopRecipients'] = df['Recipients'].apply(
lambda r: [x for x in top_recips if x in r])
# drop rows where top recipients = 0
df = df.loc[df['TopRecipients'].str.len() > 0]
df_train = df.loc[df['SentTime'] <= '2017-10-15']
df_test = df.loc[(df['SentTime'] > '2017-10-15') & (df['MessageType'] == 'Meeting')]
mlb = MultiLabelBinarizer(classes=top_recips)
train_x = df_train[['Subject', 'Body', 'CoRecipients']]
train_y = mlb.fit_transform(df_train['TopRecipients'])
test_x = df_train[['Subject', 'Body', 'CoRecipients']]
test_y = mlb.fit_transform(df_train['TopRecipients'])
print "train"
pipeline.fit(train_x, train_y)
print "predict"
predictions = pipeline.predict(test_x)
print "done"
我不确定我是否正确地对 CoRecipients 列进行了特征化。由于结果看起来不正确。有什么线索吗?
UPDATE 1
修改了MLB transformer的代码如下:
class MultiLabelTransformer(BaseEstimator, TransformerMixin):
def __init__(self, column):
self.column = column
def fit(self, X, y=None):
self.mlb = MultiLabelBinarizer()
self.mlb.fit(X[self.column])
return self
def transform(self, X):
return self.mlb.transform(X[self.column])
并修复了测试集以使用 df_test
mlb = MultiLabelBinarizer(classes=top_recips)
train_x = df_train[['Subject', 'Body', 'CoRecipients']]
train_y = mlb.fit_transform(df_train['TopRecipients'])
test_x = df_test[['Subject', 'Body', 'CoRecipients']]
test_y = mlb.transform(df_test['TopRecipients'])
看到下面的KeyError
Traceback (most recent call last):
File "E:\Projects\NLP\FeatureUnion.py", line 99, in <module>
predictions = pipeline.predict(test_x)
File "C:\Python27\lib\site-packages\sklearn\utils\metaestimators.py", line 115, in <lambda>
out = lambda *args, **kwargs: self.fn(obj, *args, **kwargs)
File "C:\Python27\lib\site-packages\sklearn\pipeline.py", line 306, in predict
Xt = transform.transform(Xt)
File "C:\Python27\lib\site-packages\sklearn\pipeline.py", line 768, in transform
for name, trans, weight in self._iter())
File "C:\Python27\lib\site-packages\sklearn\externals\joblib\parallel.py", line 779, in __call__
while self.dispatch_one_batch(iterator):
File "C:\Python27\lib\site-packages\sklearn\externals\joblib\parallel.py", line 625, in dispatch_one_batch
self._dispatch(tasks)
File "C:\Python27\lib\site-packages\sklearn\externals\joblib\parallel.py", line 588, in _dispatch
job = self._backend.apply_async(batch, callback=cb)
File "C:\Python27\lib\site-packages\sklearn\externals\joblib\_parallel_backends.py", line 111, in apply_async
result = ImmediateResult(func)
File "C:\Python27\lib\site-packages\sklearn\externals\joblib\_parallel_backends.py", line 332, in __init__
self.results = batch()
File "C:\Python27\lib\site-packages\sklearn\externals\joblib\parallel.py", line 131, in __call__
return [func(*args, **kwargs) for func, args, kwargs in self.items]
File "C:\Python27\lib\site-packages\sklearn\pipeline.py", line 571, in _transform_one
res = transformer.transform(X)
File "C:\Python27\lib\site-packages\sklearn\pipeline.py", line 426, in _transform
Xt = transform.transform(Xt)
File "E:\Projects\NLP\FeatureUnion.py", line 37, in transform
return self.mlb.transform(X[self.column])
File "C:\Python27\lib\site-packages\sklearn\preprocessing\label.py", line 765, in transform
yt = self._transform(y, class_to_index)
File "C:\Python27\lib\site-packages\sklearn\preprocessing\label.py", line 789, in _transform
indices.extend(set(class_mapping[label] for label in labels))
File "C:\Python27\lib\site-packages\sklearn\preprocessing\label.py", line 789, in <genexpr>
indices.extend(set(class_mapping[label] for label in labels))
KeyError: u'cf3024@gmail.com'
> 更新 2
工作代码
class MultiLabelTransformer(BaseEstimator, TransformerMixin):
def __init__(self, column, classes):
self.column = column
self.classes = classes
def fit(self, X, y=None):
self.mlb = MultiLabelBinarizer(classes=self.classes)
self.mlb.fit(X[self.column])
return self
def transform(self, X):
return self.mlb.transform(X[self.column])
# drop rows where top recipients = 0
df = df.loc[df['TopRecipients'].str.len() > 0]
df_train = df.loc[df['SentTime'] <= '2017-10-15']
df_test = df.loc[(df['SentTime'] > '2017-10-15') &
(df['MessageType'] == 'Meeting')]
mlb = MultiLabelBinarizer(classes=top_recips)
train_x = df_train[['Subject', 'Body', 'CoRecipients']]
train_y = mlb.fit_transform(df_train['TopRecipients'])
test_x = df_test[['Subject', 'Body', 'CoRecipients']]
test_y = mlb.transform(df_test['TopRecipients'])
# get all unique co-recipients
co_recips = list(set([a for b in df.CoRecipients.tolist() for a in b]))
# create pipeline
pipeline = Pipeline([
('features', FeatureUnion(
# list of features
transformer_list=[
('subject_tfidf', Pipeline([
('selector', ColumnSelector(column='Subject')),
('tfidf', TfidfVectorizer(min_df=0.0025, ngram_range=(1, 4)))
])),
('body_tfidf', Pipeline([
('selector', ColumnSelector(column='Body')),
('tfidf', TfidfVectorizer(min_df=0.0025, ngram_range=(1, 4)))
])),
('recipients_binarizer', Pipeline([
('multi_label', MultiLabelTransformer(column='CoRecipients', classes=co_recips))
]))
],
# weight components in FeatureUnion
transformer_weights={
'subject_tfidf': 3.0,
'body_tfidf': 1.0,
'recipients_binarizer': 1.0,
}
)),
('classifier', OneVsRestClassifier(LinearSVC(), n_jobs=-1))
])
print "train"
pipeline.fit(train_x, train_y)
print "predict"
predictions = pipeline.predict(test_x)
您对 MultiLabelBinarizer 的转换有误。您适合训练和测试数据。那是不正确的方式。
您应该始终只适合训练数据并在测试数据上使用变换。
这个错误你犯了两次:
- 在 MultiLabelTransformer 中,您可以在其中转换 'Co-recipients'
- 在转换 test_y 期间,您转换 'TopRecipients'
问题是当测试数据在 'Co-recipients' 或 'TopRecipients' 中有不同的(或新的)值时,返回的数组将具有与训练期间不同的形状。这将导致错误的结果。
像这样更改您的代码:
class MultiLabelTransformer(BaseEstimator, TransformerMixin):
#Updated
def __init__(self, column, classes):
self.column = column
self.classes = classes
def fit(self, X, y=None):
# Updated
self.mlb = MultiLabelBinarizer(classes = self.classes)
self.mlb.fit(X[self.column])
return self
def transform(self, X):
return self.mlb.transform(X[self.column])
和
test_y = mlb.transform(df_train['TopRecipients'])
管道内部:
....
....
('multi_label', MultiLabelTransformer(column='CoRecipients',
classes=set([a for b in df.CoRecipients.tolist() for a in b]))
....
....
尽管 test_y
中的最后一次更改不会影响返回的数组,因为您在 mlb = MultiLabelBinarizer(classes=top_recips)
期间使用 top_recips
指定了 类,但最好还是这样做仅对测试数据进行转换(永远不会适合或 fit_transform)。
我有一个 pandas 数据框,其中包含有关用户发送的消息的信息。 对于我的模型,我有兴趣预测消息的丢失收件人,即给定消息的收件人 A、B、C 我想预测还有谁应该是收件人的一部分。
我正在使用 OneVsRestClassifier 和 LinearSVC 进行多标签分类。 对于功能,我想使用邮件的收件人。主题和正文。
由于收件人是一个用户列表,我想使用 MultiLabelBinarizer 转换该列。对于主题和正文,我想使用 TFIDF
我的输入 pickle 文件有如下数据:所有值都是字符串,除了 recipients 是一个 set()
[[message_id,sent_time,subject,body,set(recipients),message_type, is_sender]]
我在管道中使用带有自定义转换器的特征联合来实现如下。
from sklearn.preprocessing import MultiLabelBinarizer
from sklearn.base import TransformerMixin, BaseEstimator
from sklearn.pipeline import Pipeline, FeatureUnion
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.multiclass import OneVsRestClassifier
from sklearn.model_selection import GridSearchCV
from sklearn.multiclass import OneVsRestClassifier
from sklearn.linear_model import SGDClassifier
from sklearn.svm import SVC, LinearSVC
import pickle
import pandas as pd
import numpy as np
if __name__ == "__main__":
class ColumnSelector(BaseEstimator, TransformerMixin):
def __init__(self, column):
self.column = column
def fit(self, X, y=None, **fit_params):
return self
def transform(self, X, y=None, **fit_params):
return X[self.column]
class MultiLabelTransformer(BaseEstimator, TransformerMixin):
def __init__(self, column):
self.column = column
def fit(self, X, y=None):
return self
def transform(self, X):
mlb = MultiLabelBinarizer()
return mlb.fit_transform(X[self.column])
pipeline = Pipeline([
('features', FeatureUnion([
('subject_tfidf', Pipeline([
('selector', ColumnSelector(column='Subject')),
('tfidf', TfidfVectorizer(min_df=0.0025, ngram_range=(1, 4)))
])),
('body_tfidf', Pipeline([
('selector', ColumnSelector(column='Body')),
('tfidf', TfidfVectorizer(min_df=0.0025, ngram_range=(1, 4)))
])),
('recipients_binarizer', Pipeline([
('multi_label', MultiLabelTransformer(column='CoRecipients'))
])),
])),
('classifier', OneVsRestClassifier(LinearSVC(), n_jobs=-1))
])
top_recips = ['A', 'B', 'C, 'D]
corpus_data = pickle.load(
open("E:\Data\messages_items.pkl", "rb"))
df = pd.DataFrame(corpus_data, columns=[
'MessageId', 'SentTime', 'Subject', 'Body', 'Recipients', 'MessageType', 'IsSender'])
df = df.dropna()
# add co recipients and top recipients columns
df['CoRecipients'] = df['Recipients'].apply(
lambda r: [x for x in r if x not in top_recips])
df['TopRecipients'] = df['Recipients'].apply(
lambda r: [x for x in top_recips if x in r])
# drop rows where top recipients = 0
df = df.loc[df['TopRecipients'].str.len() > 0]
df_train = df.loc[df['SentTime'] <= '2017-10-15']
df_test = df.loc[(df['SentTime'] > '2017-10-15') & (df['MessageType'] == 'Meeting')]
mlb = MultiLabelBinarizer(classes=top_recips)
train_x = df_train[['Subject', 'Body', 'CoRecipients']]
train_y = mlb.fit_transform(df_train['TopRecipients'])
test_x = df_train[['Subject', 'Body', 'CoRecipients']]
test_y = mlb.fit_transform(df_train['TopRecipients'])
print "train"
pipeline.fit(train_x, train_y)
print "predict"
predictions = pipeline.predict(test_x)
print "done"
我不确定我是否正确地对 CoRecipients 列进行了特征化。由于结果看起来不正确。有什么线索吗?
UPDATE 1
修改了MLB transformer的代码如下:
class MultiLabelTransformer(BaseEstimator, TransformerMixin):
def __init__(self, column):
self.column = column
def fit(self, X, y=None):
self.mlb = MultiLabelBinarizer()
self.mlb.fit(X[self.column])
return self
def transform(self, X):
return self.mlb.transform(X[self.column])
并修复了测试集以使用 df_test
mlb = MultiLabelBinarizer(classes=top_recips)
train_x = df_train[['Subject', 'Body', 'CoRecipients']]
train_y = mlb.fit_transform(df_train['TopRecipients'])
test_x = df_test[['Subject', 'Body', 'CoRecipients']]
test_y = mlb.transform(df_test['TopRecipients'])
看到下面的KeyError
Traceback (most recent call last):
File "E:\Projects\NLP\FeatureUnion.py", line 99, in <module>
predictions = pipeline.predict(test_x)
File "C:\Python27\lib\site-packages\sklearn\utils\metaestimators.py", line 115, in <lambda>
out = lambda *args, **kwargs: self.fn(obj, *args, **kwargs)
File "C:\Python27\lib\site-packages\sklearn\pipeline.py", line 306, in predict
Xt = transform.transform(Xt)
File "C:\Python27\lib\site-packages\sklearn\pipeline.py", line 768, in transform
for name, trans, weight in self._iter())
File "C:\Python27\lib\site-packages\sklearn\externals\joblib\parallel.py", line 779, in __call__
while self.dispatch_one_batch(iterator):
File "C:\Python27\lib\site-packages\sklearn\externals\joblib\parallel.py", line 625, in dispatch_one_batch
self._dispatch(tasks)
File "C:\Python27\lib\site-packages\sklearn\externals\joblib\parallel.py", line 588, in _dispatch
job = self._backend.apply_async(batch, callback=cb)
File "C:\Python27\lib\site-packages\sklearn\externals\joblib\_parallel_backends.py", line 111, in apply_async
result = ImmediateResult(func)
File "C:\Python27\lib\site-packages\sklearn\externals\joblib\_parallel_backends.py", line 332, in __init__
self.results = batch()
File "C:\Python27\lib\site-packages\sklearn\externals\joblib\parallel.py", line 131, in __call__
return [func(*args, **kwargs) for func, args, kwargs in self.items]
File "C:\Python27\lib\site-packages\sklearn\pipeline.py", line 571, in _transform_one
res = transformer.transform(X)
File "C:\Python27\lib\site-packages\sklearn\pipeline.py", line 426, in _transform
Xt = transform.transform(Xt)
File "E:\Projects\NLP\FeatureUnion.py", line 37, in transform
return self.mlb.transform(X[self.column])
File "C:\Python27\lib\site-packages\sklearn\preprocessing\label.py", line 765, in transform
yt = self._transform(y, class_to_index)
File "C:\Python27\lib\site-packages\sklearn\preprocessing\label.py", line 789, in _transform
indices.extend(set(class_mapping[label] for label in labels))
File "C:\Python27\lib\site-packages\sklearn\preprocessing\label.py", line 789, in <genexpr>
indices.extend(set(class_mapping[label] for label in labels))
KeyError: u'cf3024@gmail.com'
> 更新 2
工作代码
class MultiLabelTransformer(BaseEstimator, TransformerMixin):
def __init__(self, column, classes):
self.column = column
self.classes = classes
def fit(self, X, y=None):
self.mlb = MultiLabelBinarizer(classes=self.classes)
self.mlb.fit(X[self.column])
return self
def transform(self, X):
return self.mlb.transform(X[self.column])
# drop rows where top recipients = 0
df = df.loc[df['TopRecipients'].str.len() > 0]
df_train = df.loc[df['SentTime'] <= '2017-10-15']
df_test = df.loc[(df['SentTime'] > '2017-10-15') &
(df['MessageType'] == 'Meeting')]
mlb = MultiLabelBinarizer(classes=top_recips)
train_x = df_train[['Subject', 'Body', 'CoRecipients']]
train_y = mlb.fit_transform(df_train['TopRecipients'])
test_x = df_test[['Subject', 'Body', 'CoRecipients']]
test_y = mlb.transform(df_test['TopRecipients'])
# get all unique co-recipients
co_recips = list(set([a for b in df.CoRecipients.tolist() for a in b]))
# create pipeline
pipeline = Pipeline([
('features', FeatureUnion(
# list of features
transformer_list=[
('subject_tfidf', Pipeline([
('selector', ColumnSelector(column='Subject')),
('tfidf', TfidfVectorizer(min_df=0.0025, ngram_range=(1, 4)))
])),
('body_tfidf', Pipeline([
('selector', ColumnSelector(column='Body')),
('tfidf', TfidfVectorizer(min_df=0.0025, ngram_range=(1, 4)))
])),
('recipients_binarizer', Pipeline([
('multi_label', MultiLabelTransformer(column='CoRecipients', classes=co_recips))
]))
],
# weight components in FeatureUnion
transformer_weights={
'subject_tfidf': 3.0,
'body_tfidf': 1.0,
'recipients_binarizer': 1.0,
}
)),
('classifier', OneVsRestClassifier(LinearSVC(), n_jobs=-1))
])
print "train"
pipeline.fit(train_x, train_y)
print "predict"
predictions = pipeline.predict(test_x)
您对 MultiLabelBinarizer 的转换有误。您适合训练和测试数据。那是不正确的方式。
您应该始终只适合训练数据并在测试数据上使用变换。
这个错误你犯了两次:
- 在 MultiLabelTransformer 中,您可以在其中转换 'Co-recipients'
- 在转换 test_y 期间,您转换 'TopRecipients'
问题是当测试数据在 'Co-recipients' 或 'TopRecipients' 中有不同的(或新的)值时,返回的数组将具有与训练期间不同的形状。这将导致错误的结果。
像这样更改您的代码:
class MultiLabelTransformer(BaseEstimator, TransformerMixin):
#Updated
def __init__(self, column, classes):
self.column = column
self.classes = classes
def fit(self, X, y=None):
# Updated
self.mlb = MultiLabelBinarizer(classes = self.classes)
self.mlb.fit(X[self.column])
return self
def transform(self, X):
return self.mlb.transform(X[self.column])
和
test_y = mlb.transform(df_train['TopRecipients'])
管道内部:
....
....
('multi_label', MultiLabelTransformer(column='CoRecipients',
classes=set([a for b in df.CoRecipients.tolist() for a in b]))
....
....
尽管 test_y
中的最后一次更改不会影响返回的数组,因为您在 mlb = MultiLabelBinarizer(classes=top_recips)
期间使用 top_recips
指定了 类,但最好还是这样做仅对测试数据进行转换(永远不会适合或 fit_transform)。