Início / server / Perguntas / 1162871
Accepted
Umair
Asked: 2024-07-22 18:35:11 +0800 CST

Fluent-bit Match não funciona, exceto *

  • 772

Match está funcionando apenas para * e não vejo por que isso está acontecendo. Aqui está minha configuração.

Lua

function process_record(tag, timestamp, record)
    print("Input tag: ", tag)
    print("Record: ", record["tag"])
    local s3_log = {}
    local cw_log = {}

    for k, v in pairs(record) do
        cw_log[k] = v
        cw_log["tag"] = "cw"

        s3_log[k] = v
        s3_log["tag"] = "s3"
    end
    print("S3 tag: ", s3_log["tag"])
    print("CW tag: ", cw_log["tag"])
    print("S3 Log: ", s3_log["log"])
    print("CW Log: ", cw_log["log"])
    return 2, timestamp, {s3_log, cw_log}
end

configuração

[SERVICE]
    flush 1
    daemon off
    log_level debug

[FILTER]
    Name          lua
    Match         *
    script        /append_tag.lua
    call          process_record

[FILTER]
    Name          lua
    Match         $tag cw
    script        /print.lua
    call          process_record


[FILTER]
    Name          rewrite_tag
    Match         cw
    Rule          $tag .* cw true
    Emitter_Name  cw_emitted
[OUTPUT]
    Name                cloudwatch_logs
    Match               *cw*
#    Match               *
#    Match_regex         ^(cw)$
    log_key             log
    region              af-south-1
    log_group_name      /ecs/log
    log_stream_prefix   fluent-bit-
    auto_create_group   On
    log_format          json/emf

Informações adicionais:

Versão Fluent Bit: AWS_FOR_FLUENT_BIT_VERSION 2.32.2.20240627

Configuração do AWS Fargate

Usando contêineres log_router

amazon-ecs

1 respostas

  1. Best Answer

    Depois de trabalhar nisso, finalmente encontrei a solução. Aqui está minha configuração de trabalho que envia dados para cw e s3 e remove alguns logs para cw

    logDestinos.conf

    [FILTER]
    Name          lua
    Match         *
    script        /script.lua
    call          process_record

[FILTER]
    Name          rewrite_tag
    Match_regex   .*demo.*
    Rule          $tag "^(cw)$" cw.$container_id true
    Emitter_Name  re_emitted_cw

[FILTER]
    Name    grep
    Match_regex   .*cw.*
    Exclude tag s3

[FILTER]
    Name    grep
    Match_regex   .*cw.*
    Exclude log /.*message\":\"remove.*/

[FILTER]
    Name    grep
    Match_regex   .*demo.*
    Exclude tag cw

[OUTPUT]
    Name                cloudwatch_logs
    Match_regex         .*cw.*
    log_key             log
    region              af-south-1
    log_group_name      /ecs/demo
    log_stream_prefix   fluent-bit-
    auto_create_group   On
    log_format          json/emf


[OUTPUT]                   
    bucket     s3_bucket
    total_file_size    200M
    log_key    log
    Match_regex   .*demo.*
    s3_key_format    /demo/%Y-%m-%d/$TAG/
    use_put_object    on
    upload_timeout    1M
    region    af-south-1
    Name    s3

    script.lua

    function process_record(tag, timestamp, record)
    
    local s3_log = {}
    local cw_log = {}

    if record["log"] == nil then
        record["log"] = record["message"] or "default_log_message"
    end

    for k, v in pairs(record) do
        cw_log[k] = v
        cw_log["tag"] = "cw"

        s3_log[k] = v
        s3_log["tag"] = "s3"
    end
    
    return 2, timestamp, {s3_log, cw_log}
end

    Dockerfile

    FROM cr.fluentbit.io/fluent/fluent-bit
ADD logDestinations.conf /logDestinations.conf
ADD script.lua /script.lua
    • 0

relate perguntas