management-fluentd.yml

# Fluentd log handling
# + Elasticsearch + Kibana in Docker
# https://www.fluentd.org/guides/recipes/elasticsearch-and-s3

# This assumes docker has already been installed
# (by management-prometheus.yml)
- hosts: "{{ idr_environment | default('idr') }}-management-hosts"

  tasks:

  - name: Create fluentd server configuration directory
    become: yes
    file:
      path: /etc/fluentd
      recurse: yes
      state: directory

  - name: Create data top level directory
    become: yes
    file:
      path: /data
      state: directory
      owner: root
      group: root

  - name: Create fluentd aggregated logs directory
    become: yes
    file:
      path: /data/fluentd
      state: directory
      owner: "{{ fluentd_uid }}"
      group: "{{ fluentd_uid }}"

  - name: Copy fluentd server configuration files
    become: yes
    template:
      src: files/fluentd-server/fluent.conf.j2
      dest: /etc/fluentd/fluent.conf
    register: fluent_conf_status

  - name: Create elasticsearch directory
    become: yes
    file:
      path: /data/elasticsearch
      state: directory
      # User id in elasticsearch Docker image
      owner: 1000
      group: 1000

  - name: Create docker network
    become: yes
    docker_network:
      name: fluent-es-kb
      state: present

  - name: Run docker elasticsearch
    become: yes
    docker_container:
      image: "{{ elasticsearch_docker_image }}"
      name: elasticsearch
      cleanup: True
      env:
        discovery.type: single-node
        ES_JAVA_OPTS: "-Xmx4096m"
      networks:
      - name: fluent-es-kb
      published_ports:
      - "9200:9200"
      #- "9300:9300"
      state: started
      restart_policy: always
      volumes:
      - /data/elasticsearch:/usr/share/elasticsearch/data

  - name: Run docker fluentd
    become: yes
    docker_container:
      image: "{{ fluentd_docker_image }}"
      name: fluentd
      cleanup: True
      env:
        FLUENT_UID: "{{ fluentd_uid }}"
      networks:
      - name: fluent-es-kb
      published_ports:
      - "24224:24224/udp"
      - "24224:24224"
      restart: "{{ fluent_conf_status is changed }}"
      state: started
      restart_policy: always
      volumes:
      - /etc/fluentd/fluent.conf:/fluentd/etc/fluent.conf:ro
      - /data/fluentd:/data/fluentd

  - name: Run docker kibana
    become: yes
    docker_container:
      image: "{{ kibana_docker_image }}"
      name: kibana
      cleanup: True
      networks:
      - name: fluent-es-kb
      published_ports:
      - "5601:5601"
      state: started
      restart_policy: always

  - name: Run elasticsearch curator
    become: yes
    docker_container:
      image: "{{ elasticsearch_curator_docker_image }}"
      name: elasticsearch-curator
      cleanup: True
      env:
        OLDER_THAN_IN_DAYS: "{{ elasticsearch_expire_logs_days }}"
        INTERVAL_IN_HOURS: "24"
      networks:
      - name: fluent-es-kb
      state: started
      restart_policy: always

  vars:
    fluentd_shared_key: "{{ idr_secret_fluentd_shared_key | default('fluentd') }}"
    fluentd_slack_token: "{{ idr_secret_management_slack_token | default(None) }}"
    fluentd_slack_channel: "{{ idr_logs_slack_channel | default('idr-logs') }}"
    fluentd_elasticsearch_host: elasticsearch
    fluentd_uid: "1010"
    elasticsearch_docker_image: "docker.elastic.co/elasticsearch/elasticsearch-oss:6.1.1"
    elasticsearch_curator_docker_image: "openmicroscopy/elasticsearch-curator:5.4.1"
    elasticsearch_expire_logs_days: "14"
    fluentd_docker_image: "openmicroscopy/fluentd:0.1.0"
    kibana_docker_image: "docker.elastic.co/kibana/kibana-oss:6.1.1"


# Load hostvars for management server
- hosts: >-
    {{ idr_environment | default('idr') }}-management-hosts
    {{ idr_parent_environment | default('idr') }}-management-hosts


- hosts: >
    {{ idr_environment | default('idr') }}-proxy-hosts
    {{ idr_environment | default('idr') }}-omero-hosts

  # TODO: Ideally we'd use the `validate:` option to check these config
  # files, but it's not possible to validate only a fragment.
  # Instead we need to create these files before applying the role
  # However, we can only trigger the restart handler if the agent is
  # already installed.
  pre_tasks:

  - name: Check if td-agent already installed
    stat:
      path: /etc/init.d/td-agent
    register: _td_agent_st

  - name: Create fluentd conf.d
    become: yes
    file:
      path: /etc/td-agent/conf.d
      recurse: yes
      state: directory

  - name: Configure fluentd forwarding
    become: yes
    template:
      src: files/fluentd/forward-conf.j2
      dest: /etc/td-agent/conf.d/forward.conf
    notify:
    - restart fluentd if installed

  - name: Copy fluentd configuration
    become: yes
    copy:
      src: "{{ item }}"
      dest: "/etc/td-agent/conf.d/{{ item | basename }}"
    with_items:
    # These are specifc to each host-group
    - "{{ fluentd_source_configs }}"
    notify:
    - restart fluentd if installed

  handlers:
  - name: restart fluentd if installed
    become: yes
    systemd:
      daemon_reload: yes
      name: td-agent
      state: restarted
    when: _td_agent_st.stat.exists

  roles:
  - role: ome.fluentd

  vars:
    _monitoring_idr_environment: "{{ idr_parent_environment | default(idr_environment | default('idr')) + '-management-hosts' }}"
    fluentd_server_address: "{{ hostvars[groups[_monitoring_idr_environment][0]]['ansible_' + (idr_net_iface | default('eth0'))]['ipv4']['address'] }}"
    fluentd_shared_key: "{{ idr_secret_fluentd_shared_key | default('fluentd') }}"