如何使用 rsyslog 将使用通配符文件和文件夹找到的文件保存到中央服务器上的正确文件名？

Question

我正在尝试使用 rainerscript 和 rsyslog v8.38 通过通配符和通配符文件夹中的通配符文件从我的服务器上获取日志，然后将它们保存在远端的相同文件夹结构中。由于文章 here，我可以使用通配符，但我正在尝试扩展此概念以使其也适用于通配符文件夹。

目前我正确地从文件夹中收集了文件，但是一旦它被保存，所有文件中的行就会被保存到每个文件夹的一个文件中，例如，如果我这样做的话在我的服务器上：

echo "TEST1" >> /srv/log/test-new/test.log
echo "TEST1" >> /srv/log/test-new/test-new.log

我最终在我的中央服务器上得到了这个：

# cat /srv/rsyslog/2018/HOSTNAME/10/26/test-new

<133>2018-10-26T15:32:37.975449+00:00 HOSTNAME  test-new/test.log nested-srv-logs TEST1
<133>2018-10-26T15:32:51.042633+00:00 HOSTNAME  test-new/test-new.log nested-srv-logs TEST1

我希望我可以将文件保存在中央服务器上，文件夹结构与它们在发送机器上的文件夹结构相同，这可能吗？

我的配置如下，发送机有：

module(load="omrelp")
module(load="omfwd")

template(name="CustomForwardFormat" type="list") {
    constant(value="<")
    property(name="pri")
    constant(value=">")
    property(name="timestamp" dateFormat="rfc3339")
    constant(value=" ")
    property(name="hostname")
    constant(value=" ")
    property(name=".suffix")
    constant(value=" ")
    property(name="syslogtag" position.from="1" position.to="32")
    property(name="msg" spifno1stsp="on" )
    property(name="msg")
    constant(value="\n")
    }

ruleset(name="sendToLogserver") {
    action(type="omrelp" Target="rsyslog" Port="25014" template="CustomForwardFormat"
    queue.type="LinkedList" queue.size="10000" queue.filename="q_sendToLogserver" queue.highwatermark="9000"
    queue.lowwatermark="50" queue.maxdiskspace="500m" queue.saveonshutdown="on" action.resumeRetryCount="-1"
    action.reportSuspension="on" action.reportSuspensionContinuation="on" action.resumeInterval="10")
}

ruleset(name="sendToJsonLogserver") {
    action(type="omfwd" protocol="tcp" Target="logstash" Port="5114" template="RSYSLOG_SyslogProtocol23Format"
    queue.type="LinkedList" queue.size="10000" queue.filename="q_sendToJsonLogserver" queue.highwatermark="9000"
    queue.lowwatermark="50" queue.maxdiskspace="500m" queue.saveonshutdown="on" action.resumeRetryCount="-1"
    action.reportSuspension="on" action.reportSuspensionContinuation="on" action.resumeInterval="10")
}

input(type="imfile"
    File="/srv/log/*.log"
        Tag="srv-logs"
        Ruleset="srv_logs"
        addMetadata="on")

input(type="imfile"
    File="/srv/log/*/*.log"
        Tag="nested-srv-logs"
        Ruleset="srv_logs"
        addMetadata="on")

ruleset(name="srv_logs") {
    # http://www.rsyslog.com/doc/v8-stable/rainerscript/functions.html
          # re_extract(expr, re, match, submatch, no-found)
    set $.suffix=re_extract($!metadata!filename, "(.*)/([^/]*)", 0, 2, "unknown.log");
    if ( $programname == "nested-srv-logs" ) then {
      set $.sub-suffix=re_extract($!metadata!filename, "(.*)/([^/]*)/(.*)", 0, 2, "unknown.log");
        set $.suffix=$.sub-suffix & "/" & $.suffix;
    }
    if( $!metadata!filename contains 'json' ) then {
            call sendToJsonLogserver
    } else {
      call sendToLogserver
    }
    stop
}

中央服务器有：

module(load="imrelp")
input(type="imrelp" port="25014" ruleset="RemoteLogProcess")

module(load="builtin:omfile" FileOwner="syslog" FileGroup="syslog" dirOwner="syslog" dirGroup="syslog" FileCreateMode="0644" DirCreateMode="0755")

template(name="CustomForwardFormat" type="list") {
    constant(value="<")
    property(name="pri")
    constant(value=">")
    property(name="timestamp" dateFormat="rfc3339")
    constant(value=" ")
    property(name="hostname")
    constant(value=" ")
    property(name=".suffix")
    constant(value=" ")
    property(name="syslogtag" position.from="1" position.to="32")
    property(name="msg" spifno1stsp="on" )
    property(name="msg")
    constant(value="\n")
    }

$EscapeControlCharactersOnReceive off

template(name="FloowLogSavePath" type="list") {
    constant(value="/srv/rsyslog/")
    property(name="timegenerated" dateFormat="year")
    constant(value="/")
    property(name="hostname")
    constant(value="/")
    property(name="timegenerated" dateFormat="month")
    constant(value="/")
    property(name="timegenerated" dateFormat="day")
    constant(value="/")
    property(name="$.logpath" )
}

ruleset(name="RemoteLogProcess") {
    # For facilities local0-7 set log filename from $programname field: replace __ with /
    if ( $syslogfacility >= 16 ) then
    {
        set $.logpath = replace($programname, "__", "/");
        action(type="omfile" dynaFileCacheSize="1024" dynaFile="FloowLogSavePath" template="CustomForwardFormat"
        flushOnTXEnd="off" asyncWriting="on" flushInterval="1" ioBufferSize="64k")
    } else {
        if (($syslogfacility == 0)) then {
            set $.logpath = "kern.log";
        } else if (($syslogfacility == 1)) then {
            set $.logpath = "user";
        } else if (($syslogfacility == 2)) then {
            set $.logpath = "mail";
        } else if (($syslogfacility == 3)) then {
            set $.logpath = "daemon";
        } else if (($syslogfacility == 4) or ($syslogfacility == 10)) then {
            set $.logpath = "auth.log";
        } else if (($syslogfacility == 9) or ($syslogfacility == 15)) then {
            set $.logpath = "cron";
        } else {
            set $.logpath ="syslog";
        }
        # Built-in template RSYSLOG_FileFormat: High-precision timestamps and timezone information
        action(type="omfile" dynaFileCacheSize="1024" dynaFile="FloowLogSavePath" template="CustomForwardFormat"
        flushOnTXEnd="off" asyncWriting="on" flushInterval="1" ioBufferSize="64k")
    }
}

Answer 1

我看到你在此处插入了 space：

property(name=".suffix")
constant(value=" ")
property(name="syslogtag" position.from="1" position.to="32")

空格是分隔符，因此可能会扰乱您的设置。

此外，您以错误的顺序连接它们：$.suffix 是文件和目录名称，我想它应该放在最后。而且您只使用了 syslogtag 中的 1-32 个符号，为什么？

Answer 2

检查我的配置后，我之前的回答不正确，为了让通配符文件夹正常工作，我做了以下操作：

发件人：

input(type="imfile"
    File="/srv/log/*.log"
    Tag="srv-logs"
    Ruleset="send_sorted"
    addMetadata="on")

input(type="imfile"
    File="/srv/log/*/*.log"
    Tag="nested-srv-logs"
    Ruleset="send_sorted"
    addMetadata="on")

module(load="omrelp")

ruleset(name="send_sorted") {
    set $.suffix=substring($!metadata!filename, 9, 150);
    if( $!metadata!filename contains 'json' ) then {
      call sendToJsonLogserver
    } else {
      call sendToLogserver
    }
    stop
}

template(name="CustomForwardFormat" type="list") {
    constant(value="<")
    property(name="pri")
    constant(value=">")
    property(name="timestamp" dateFormat="rfc3339")
    constant(value=" ")
    property(name="hostname")
    constant(value=" ")
    property(name=".suffix")
    constant(value=" ")
    property(name="syslogtag")
    property(name="msg" spifno1stsp="on" )
    property(name="msg")
    constant(value="\n")
    }
ruleset(name="sendToLogserver") {
    action(type="omrelp"
           target="rsyslog"
           port="25014"
           template="CustomForwardFormat"
           queue.type="LinkedList" 
           queue.size="10000"
           queue.filename="q_sendToLogserver"
           queue.highwatermark="9000"
           queue.lowwatermark="50"
           queue.maxdiskspace="500m"
           queue.saveonshutdown="on" 
           action.resumeRetryCount="-1"
           action.reportSuspension="on"
           action.reportSuspensionContinuation="on"
           action.resumeInterval="10")
}

ruleset(name="sendToJsonLogserver") {
    action(type="omfwd" 
           target="logstash" 
           protocol="tcp" 
           port="5114" 
           template="RSYSLOG_SyslogProtocol23Format"
           queue.type="LinkedList"
           queue.size="10000"
           queue.filename="q_sendToJsonLogserver"
           queue.highwatermark="9000"
           queue.lowwatermark="50"
           queue.maxdiskspace="500m" 
           queue.saveonshutdown="on"
           action.resumeRetryCount="-1"
           action.reportSuspension="on"
           action.reportSuspensionContinuation="on"
           action.resumeInterval="10")
}

在我的中央服务器上：

module(load="imrelp")
input(type="imrelp" port="25014" ruleset="RemoteLogProcess")

module(load="builtin:omfile" FileOwner="syslog" FileGroup="syslog" dirOwner="syslog" dirGroup="syslog" FileCreateMode="0644" DirCreateMode="0755")

template(name="CustomForwardFormat" type="list") {
    constant(value="<")
    property(name="pri")
    constant(value=">")
    property(name="timestamp" dateFormat="rfc3339")
    constant(value=" ")
    property(name="hostname")
    constant(value=" ")
    property(name=".suffix")
    constant(value=" ")
    property(name="syslogtag")
    property(name="msg" spifno1stsp="on" )
    property(name="msg")
    constant(value="\n")
    }

template(name="FloowLogSavePath" type="list") {
    constant(value="/srv/rsyslog/")
    property(name="timegenerated" dateFormat="year")
    constant(value="/")
    property(name="hostname")
    constant(value="/")
    property(name="timegenerated" dateFormat="month")
    constant(value="/")
    property(name="timegenerated" dateFormat="day")
    constant(value="/")
    property(name=".logpath")
}

template(name="extract" type="string" string="%syslogtag%")

ruleset(name="RemoteLogProcess") {
    if ( $syslogfacility >= 16 ) then
    {
        set $.logpath = exec_template("extract");
        action(type="omfile" 
               dynaFileCacheSize="1024" 
               dynaFile="FloowLogSavePath" 
               template="CustomForwardFormat"
               flushOnTXEnd="off"
               asyncWriting="on"
               flushInterval="1" 
               ioBufferSize="64k")
    } else {
        if (($syslogfacility == 0)) then {
          set $.logpath = "kern.log";
        } else if (($syslogfacility == 1)) then {
            set $.logpath = "user";
        } else if (($syslogfacility == 2)) then {
            set $.logpath = "mail";
        } else if (($syslogfacility == 3)) then {
            set $.logpath = "daemon";
        } else if (($syslogfacility == 4) or ($syslogfacility == 10)) then {
            set $.logpath = "auth.log";
        } else if (($syslogfacility == 9) or ($syslogfacility == 15)) then {
            set $.logpath = "cron";
        } else {
            set $.logpath ="syslog";
        }
        action(type="omfile"
               dynaFileCacheSize="1024"
               dynaFile="FloowLogSavePath"
               template="CustomForwardFormat"
               flushOnTXEnd="off"
               asyncWriting="on"
               flushInterval="1"
               ioBufferSize="64k")
    }
}

如何使用 rsyslog 将使用通配符文件和文件夹找到的文件保存到中央服务器上的正确文件名？

How to save files found with wildcard file & folder to the right file name on the central server with rsyslog?

rsyslog

rainerscript