OTC for K8S configuration

config.yaml

receivers:
  filelog:
    include: [ local/pods/*/*/*.log ]
    start_at: beginning
    include_file_path: true
    include_file_name: false
    operators:
      # FInd out which format is used by kubernetes
      - type: regex_parser
        id: get-format
        regex: '^((?P<docker_format>\{)|(?P<crio_format>[^ Z]+) |(?P<containerd_format>[^ ^Z]+Z) )'
        preserve_to: original
      # Parse CRI-O format
      - type: regex_parser
        id: parser-crio
        if: '$$record.crio_format != ""'
        parse_from: original
        regex: '^(?P<time>[^ Z]+) (?P<stream>stdout|stderr) (?P<logtag>[^ ]*) (?P<log>.*)$'
        timestamp:
          parse_from: time
          layout_type: gotime
          layout: '2006-01-02T15:04:05.000000000-07:00'
      # Parse CRI-Containerd format
      - type: regex_parser
        id: parser-containerd
        if: '$$record.containerd_format != ""'
        parse_from: original
        regex: '^(?P<time>[^ ^Z]+Z) (?P<stream>stdout|stderr) (?P<logtag>[^ ]*) (?P<log>.*)$'
        timestamp:
          parse_from: time
          layout: '%Y-%m-%dT%H:%M:%S.%LZ'
      # Parse Docker format
      - type: json_parser
        id: parser-docker
        if: '$$record.docker_format != ""'
        parse_from: original
        timestamp:
          parse_from: time
          layout: '%Y-%m-%dT%H:%M:%S.%LZ'
      # Clean up format detection
      - type: restructure
        id: clean-up-format-detection
        ops:
          - remove: original
          - remove: docker_format
          - remove: crio_format
          - remove: containerd_format
      # Extract metadata from file path
      - type: regex_parser
        id: extract_metadata_from_filepath
        regex: '^.*\/(?P<namespace>[^_]+)_(?P<pod_name>[^_]+)_(?P<uid>[a-f0-9\-]{36})\/(?P<container_name>[^\._]+)\/(?P<run_id>\d+)\.log$'
        parse_from: $$labels.file_path
      # Move out attributes to Attributes
      - type: metadata
        labels:
          stream: 'EXPR($.stream)'
          k8s.container.name: 'EXPR($.container_name)'
          k8s.namespace.name: 'EXPR($.namespace)'
          k8s.pod.name: 'EXPR($.pod_name)'
          run_id: 'EXPR($.run_id)'
          k8s.pod.uid: 'EXPR($.uid)'
      # Clean up log record
      - type: restructure
        id: clean-up-log-record
        ops:
          - remove: logtag
          - remove: stream
          - remove: container_name
          - remove: namespace
          - remove: pod_name
          - remove: run_id
          - remove: uid
  journald:
    operators:
      # Filter in only related units
      - type: filter
        id: filter
        expr: >-
          ($$record._SYSTEMD_UNIT != "addon-config.service") &&
          ($$record._SYSTEMD_UNIT != "addon-run.service") &&
          ($$record._SYSTEMD_UNIT != "cfn-etcd-environment.service") &&
          ($$record._SYSTEMD_UNIT != "cfn-signal.service") &&
          ($$record._SYSTEMD_UNIT != "clean-ca-certificates.service") &&
          ($$record._SYSTEMD_UNIT != "containerd.service") &&
          ($$record._SYSTEMD_UNIT != "coreos-metadata.service") &&
          ($$record._SYSTEMD_UNIT != "coreos-setup-environment.service") &&
          ($$record._SYSTEMD_UNIT != "coreos-tmpfiles.service") &&
          ($$record._SYSTEMD_UNIT != "dbus.service") &&
          ($$record._SYSTEMD_UNIT != "docker.service") &&
          ($$record._SYSTEMD_UNIT != "efs.service") &&
          ($$record._SYSTEMD_UNIT != "etcd-member.service") &&
          ($$record._SYSTEMD_UNIT != "etcd.service") &&
          ($$record._SYSTEMD_UNIT != "etcd2.service") &&
          ($$record._SYSTEMD_UNIT != "etcd3.service") &&
          ($$record._SYSTEMD_UNIT != "etcdadm-check.service") &&
          ($$record._SYSTEMD_UNIT != "etcdadm-reconfigure.service") &&
          ($$record._SYSTEMD_UNIT != "etcdadm-save.service") &&
          ($$record._SYSTEMD_UNIT != "etcdadm-update-status.service") &&
          ($$record._SYSTEMD_UNIT != "flanneld.service") &&
          ($$record._SYSTEMD_UNIT != "format-etcd2-volume.service") &&
          ($$record._SYSTEMD_UNIT != "kube-node-taint-and-uncordon.service") &&
          ($$record._SYSTEMD_UNIT != "kubelet.service") &&
          ($$record._SYSTEMD_UNIT != "ldconfig.service") &&
          ($$record._SYSTEMD_UNIT != "locksmithd.service") &&
          ($$record._SYSTEMD_UNIT != "logrotate.service") &&
          ($$record._SYSTEMD_UNIT != "lvm2-monitor.service") &&
          ($$record._SYSTEMD_UNIT != "mdmon.service") &&
          ($$record._SYSTEMD_UNIT != "nfs-idmapd.service") &&
          ($$record._SYSTEMD_UNIT != "nfs-mountd.service") &&
          ($$record._SYSTEMD_UNIT != "nfs-server.service") &&
          ($$record._SYSTEMD_UNIT != "nfs-utils.service") &&
          ($$record._SYSTEMD_UNIT != "node-problem-detector.service") &&
          ($$record._SYSTEMD_UNIT != "ntp.service") &&
          ($$record._SYSTEMD_UNIT != "oem-cloudinit.service") &&
          ($$record._SYSTEMD_UNIT != "rkt-gc.service") &&
          ($$record._SYSTEMD_UNIT != "rkt-metadata.service") &&
          ($$record._SYSTEMD_UNIT != "rpc-idmapd.service") &&
          ($$record._SYSTEMD_UNIT != "rpc-mountd.service") &&
          ($$record._SYSTEMD_UNIT != "rpc-statd.service") &&
          ($$record._SYSTEMD_UNIT != "rpcbind.service") &&
          ($$record._SYSTEMD_UNIT != "set-aws-environment.service") &&
          ($$record._SYSTEMD_UNIT != "system-cloudinit.service") &&
          ($$record._SYSTEMD_UNIT != "systemd-timesyncd.service") &&
          ($$record._SYSTEMD_UNIT != "update-ca-certificates.service") &&
          ($$record._SYSTEMD_UNIT != "user-cloudinit.service") &&
          ($$record._SYSTEMD_UNIT != "var-lib-etcd2.service")
exporters:
  logging:
    loglevel: debug
service:
  pipelines:
    logs:
      receivers: [filelog, journald]
      exporters: [logging]