python如何提高熵权法的算法效率

Question

下面是代码，但是，处理大数据时速度很慢。（5,000,000 行，6 列 数据框可能需要 >1 天。

只是想知道我该如何优化它？非常感谢

def ewm(df):
    df = df.apply(lambda x: ((x - np.min(x)) / (np.max(x) - np.min(x))))
    rows, cols = df.shape 
    k = 1.0 / math.log(rows)
 
    lnf = [[None] * cols for i in range(rows)]
    for i in range(0, rows):         
        for j in range(0, cols):
            if df.iloc[i][j] == 0:
                lnfij = 0.0
            else:
                p = df.iloc[i][j] / df.iloc[:,j].sum()
                lnfij = math.log(p) * p * (-k)
            lnf[i][j] = lnfij
    lnf = pd.DataFrame(lnf)
 
    d = 1 - lnf.sum(axis=0)
    w = [[None] * 1 for i in range(cols)]
    for j in range(0, cols):
        wj = d[j] / sum(d)
        w[j] = wj
    
    w = pd.DataFrame(w)
    w = w.round(5)    #.applymap(lambda x:format(x,'.5f'))
    w.index = df.columns
    w.columns =['weight']
    return w

Answer 1

获取特定值时使用 iat 而不是 iloc 如果你做同样的 iloc 两次，将它保存在 tmp

import pandas as pd
import time
import numpy as np
import math

#original method
def ewm(df):
    df = df.apply(lambda x: ((x - np.min(x)) / (np.max(x) - np.min(x))))
    rows, cols = df.shape 
    k = 1.0 / math.log(rows)
 
    lnf = [[None] * cols for i in range(rows)]
    for i in range(0, rows):         
        for j in range(0, cols):
            if df.iloc[i][j] == 0:
                lnfij = 0.0
            else:
                p = df.iloc[i][j] / df.iloc[:,j].sum()
                lnfij = math.log(p) * p * (-k)
            lnf[i][j] = lnfij
    lnf = pd.DataFrame(lnf)
 
    d = 1 - lnf.sum(axis=0)
    w = [[None] * 1 for i in range(cols)]
    for j in range(0, cols):
        wj = d[j] / sum(d)
        w[j] = wj
    
    w = pd.DataFrame(w)
    w = w.round(5)    #.applymap(lambda x:format(x,'.5f'))
    w.index = df.columns
    w.columns =['weight']
    return w


#modified method
def ewm1(df):
    df = df.apply(lambda x: ((x - np.min(x)) / (np.max(x) - np.min(x))))
    rows, cols = df.shape 
    k = 1.0 / math.log(rows)
 
    lnf = [[None] * cols for i in range(rows)]
    for i in range(0, rows):         
        for j in range(0, cols):
            tmp = df.iat[i,j]  #********************************* modified section
            if tmp == 0:
                lnfij = 0.0
            else:
                p = tmp / df.iloc[:,j].sum()  #************************ end of modified
                lnfij = math.log(p) * p * (-k)
            lnf[i][j] = lnfij
    lnf = pd.DataFrame(lnf)
 
    d = 1 - lnf.sum(axis=0)
    w = [[None] * 1 for i in range(cols)]
    for j in range(0, cols):
        wj = d[j] / sum(d)
        w[j] = wj
    
    w = pd.DataFrame(w)
    w = w.round(5)    #.applymap(lambda x:format(x,'.5f'))
    w.index = df.columns
    w.columns =['weight']
    return w



df = pd.DataFrame(np.random.rand(1000,6))
start = time.time()
ewm(df)
print(time.time()-start)

start1 = time.time()
ewm1(df)
print(time.time()-start1)

第一个函数的时间是 1.9747240543365479

第二个是 0.820796012878418

我不确定该方法的作用但是如果你可以将它分解成几个具有数字 return 值的函数你可以散列它们并进一步改进它

Answer 2

让 numpy 执行循环应该会加快很多速度

import numpy as np
import pandas as pd

def ewm(df):
    df = df.apply(lambda x: ((x - np.min(x)) / (np.max(x) - np.min(x))))
    rows, cols = df.shape 
    k = 1.0 / math.log(rows)
    
    p = df / df.sum(axis=0)
    lnf = -np.log(p , where = df!=0 )*p*k
    
    d = 1 - lnf.sum(axis=0)
    w = d / d.sum()
    
    w = pd.DataFrame(w)
    w = w.round(5)
    w.index = df.columns
    w.columns =['weight']
    return w

python如何提高熵权法的算法效率

how to improve algorithm efficiency of entropy weight method in python

python

algorithm

numpy

entropy

time-complexity