如何去除分段线性或非线性波形的趋势？

Question

我正在尝试删除波形中的趋势，如下所示：

为此，我使用 scipy.signal.detrend() 如下：

autocorr = scipy.signal.detrend(autocorr)

但我没有看到任何明显的趋平趋势。我得到以下信息：

我的objective是要从波形中完全消除趋势。而且我还需要对其进行概括，以便它可以去除任何类型的波形 - 无论是线性波形、分段线性波形、多项式波形等

你能推荐一种方法吗？

注意：为了复制上面的波形，您可以简单地运行下面我用来生成它的代码：

#Loading Libraries
import warnings
warnings.filterwarnings("ignore")

import json
import sys, os
import numpy as np
import pandas as pd
import glob
import pickle

from statsmodels.tsa.stattools import adfuller, acf, pacf
from scipy.signal import find_peaks, square
from statsmodels.graphics.tsaplots import plot_acf, plot_pacf
import matplotlib.pyplot as plt

#Generating a function with Dual Seasonality:
def white_noise(mu, sigma, num_pts):
    """ Function to generate Gaussian Normal Noise
    Args:
        sigma: std value
        num_pts: no of points
        mu: mean value

    Returns:
        generated Gaussian Normal Noise
    """
    
    noise = np.random.normal(mu, sigma, num_pts)
    return noise

def signal_line_plot(input_signal: pd.Series, title: str = "", y_label: str = "Signal"):
    """ Function to plot a time series signal
    Args:
        input_signal: time series signal that you want to plot
        title: title on plot
        y_label: label of the signal being plotted
        
    Returns:
        signal plot
    """

    plt.plot(input_signal)
    plt.title(title)
    plt.ylabel(y_label)
    plt.show()

# Square with two periodicities of daily and weekly. With @15min sampling frequency it means 4*24=96 samples and 4*24*7=672 

t_week = np.linspace(1,480, 480)
t_weekend=np.linspace(1,192,192)
T=96 #Time Period
x_weekday = 10*square(2*np.pi*t_week/T, duty=0.7)+10 + white_noise(0, 1,480)
x_weekend = 2*square(2*np.pi*t_weekend/T, duty=0.7)+2 + white_noise(0,1,192)
x_daily_weekly = np.concatenate((x_weekday, x_weekend)) 
x_daily_weekly_long = np.concatenate((x_daily_weekly,x_daily_weekly,x_daily_weekly,x_daily_weekly,x_daily_weekly,x_daily_weekly,x_daily_weekly,x_daily_weekly,x_daily_weekly,x_daily_weekly))
signal_line_plot(x_daily_weekly_long)
signal_line_plot(x_daily_weekly_long[0:1000])

#Finding Autocorrelation & Lags for the signal [WHICH  THE FINAL PARAMETERS WHICH ARE TO BE PLOTTED]:

#Determining Autocorrelation & Lag values
import scipy.signal as signal
autocorr = signal.correlate(x_daily_weekly_long, x_daily_weekly_long, mode="same")

#Normalize the autocorr values (such that the hightest peak value is at 1)
autocorr = (autocorr-min(autocorr))/(max(autocorr)-min(autocorr))

lags = signal.correlation_lags(len(x_daily_weekly_long), len(x_daily_weekly_long), mode = "same")

#Visualization
f = plt.figure()
f.set_figwidth(40)
f.set_figheight(10)
plt.plot(lags, autocorr)

#DETRENDING:
autocorr = scipy.signal.detrend(autocorr)

#Visualization
f = plt.figure()
f.set_figwidth(40)
f.set_figheight(10)
plt.plot(lags, autocorr)

Answer 1

因为它是 auto-correlation，它总是偶数；因此，在 lag=0 处使用断点去趋势化应该让你到达那里的一部分。

另一种消除趋势的方法是使用 high-pass 过滤器；你可以通过两种方式做到这一点。棘手的是决定 cut-off 频率应该是多少。

这里有一个可行的方法：

#Loading Libraries
import numpy as np
from scipy import signal
import matplotlib.pyplot as plt

#Generating a function with Dual Seasonality:
def white_noise(mu, sigma, num_pts):
    """ Function to generate Gaussian Normal Noise
    Args:
        sigma: std value
        num_pts: no of points
        mu: mean value

    Returns:
        generated Gaussian Normal Noise
    """
    
    noise = np.random.normal(mu, sigma, num_pts)
    return noise

# High-pass filter via discrete Fourier transform
# Drop all components from 0th to dropcomponent-th
def dft_highpass(x, dropcomponent):
    fx = np.fft.rfft(x)
    fx[:dropcomponent] = 0
    return np.fft.irfft(fx)


# Square with two periodicities of daily and weekly. With @15min sampling frequency it means 4*24=96 samples and 4*24*7=672 

t_week = np.linspace(1,480, 480)
t_weekend=np.linspace(1,192,192)
T=96 #Time Period
x_weekday = 10*signal.square(2*np.pi*t_week/T, duty=0.7)+10 + white_noise(0, 1,480)
x_weekend = 2*signal.square(2*np.pi*t_weekend/T, duty=0.7)+2 + white_noise(0,1,192)
x_daily_weekly = np.concatenate((x_weekday, x_weekend)) 
x_daily_weekly_long = np.concatenate((x_daily_weekly,x_daily_weekly,x_daily_weekly,x_daily_weekly,x_daily_weekly,x_daily_weekly,x_daily_weekly,x_daily_weekly,x_daily_weekly,x_daily_weekly))

#Finding Autocorrelation & Lags for the signal [WHICH  THE FINAL PARAMETERS WHICH ARE TO BE PLOTTED]:

#Determining Autocorrelation & Lag values
autocorr = signal.correlate(x_daily_weekly_long, x_daily_weekly_long, mode="same")

#Normalize the autocorr values (such that the hightest peak value is at 1)
autocorr = (autocorr-min(autocorr))/(max(autocorr)-min(autocorr))

lags = signal.correlation_lags(len(x_daily_weekly_long), len(x_daily_weekly_long), mode = "same")


# detrend w/ breakpoints
dautocorr = signal.detrend(autocorr, bp=len(lags)//2)

# detrend w/ high-pass filter
# use `filtfilt` to get zero-phase
b, a = signal.butter(1, 1e-3, 'high')
fautocorr = signal.filtfilt(b, a, autocorr)

# detrend with DFT HPF
rautocorr = dft_highpass(autocorr, len(autocorr) // 1000)

#Visualization
fig, ax = plt.subplots(3)

for i in range(3):
    ax[i].plot(lags, autocorr, label='orig')

ax[0].plot(lags, dautocorr, label='detrend w/ bp')
ax[1].plot(lags, fautocorr, label='HPF')
ax[2].plot(lags, rautocorr, label='DFT')

for i in range(3):
    ax[i].legend()
    ax[i].set_ylabel('autocorr')

ax[-1].set_xlabel('lags')

给予

如何去除分段线性或非线性波形的趋势？

How to De-Trend a waveform which is piece-wise linear or non-linear?

python

numpy

time-series

scipy

trend