Logstash Shipper and Indexer

http://logstash.net/docs/1.2.2/tutorials/getting-started-centralized-overview-diagram.png

logstash shipper configuration  

[root@CentOS6 init.d]# pwd
/etc/init.d
[root@CentOS6 init.d]# cat logstash-agent
#!/bin/bash
# From The Logstash Book
# The original of this file can be found at: http://logstashbook.com/code/index.html
#
#
# Logstash Start/Stop logstash
#
# chkconfig: 345 99 99
# description: Logstash
# processname: logstash

name="logstash-agent"
logstash_bin="/opt/logstash/bin/logstash"
logstash_conf="/etc/logstash/shipper.conf"
logstash_log="/var/log/logstash/shipper.log"

find_logstash_process () {
    PIDTEMP=`ps ux | grep logstash | grep java | awk '{ print $2 }'`
    # Pid not found
    if [ "x$PIDTEMP" = "x" ]; then
        PID=-1
    else
        PID=$PIDTEMP
    fi
}

start () {
    LOG_DIR=`dirname ${logstash_log}`
    if [ ! -d $LOG_DIR ]; then
      echo "Log dir ${LOG_DIR} doesn't exist. Creating"
      mkdir $LOG_DIR
    fi
    nohup ${logstash_bin} agent --verbose -f ${logstash_conf} --log ${logstash_log} > /dev/null 2>&1 &
}

stop () {
    find_logstash_process
    if [ $PID -ne -1 ]; then
        kill $PID
    fi
}

case $1 in
start)
        start
        ;;
stop)
        stop
        exit 0
        ;;
reload)
        stop
        sleep 2
        start
        ;;
restart)
        stop
        sleep 2
        start
        ;;
status)
        find_logstash_process
        if [ $PID -gt 0 ]; then
          echo "logstash running: $PID"
          exit 0
        else
          echo "logstash not running"
          exit 1
        fi
        ;;
*)
        echo $"Usage: $0 {start|stop|restart|reload|status}"
        RETVAL=1
esac
exit 0
[root@CentOS6 init.d]# cat /etc/logstash/shipper.conf 
input {
file {
        type => "syslog"
path => ["/syslog/apache/mod_jk*.log","/syslog/tomcat/*.log"]
        tags => "tomcat"
}
}


output {
       redis {
       host => "192.168.1.145"
       data_type => "list"
       key => "logstash"
       }
}
[root@CentOS6 init.d]#

Logstash Indexer configuration 


[root@Test-Logstash conf.d]# pwd
/etc/logstash/conf.d
[root@Test--Logstash conf.d]# cat syslog.conf 
input {
  tcp {
    type => "syslog"
    port => 514
  }
  udp {
    type => "syslog"
    port => 514
  }
  redis {
host => "127.0.0.1"
type => "redis-input"
data_type => "list"
key => "logstash"
# codec => "json"
 }
}


filter {
  if [type] == "syslog" {
    grok {
      match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" }
      add_field => [ "received_at", "%{@timestamp}" ]
      add_field => [ "received_from", "%{host}" ]
    }
    syslog_pri { }
    date {
      match => [ "syslog_timestamp", "MMM  d HH:mm:ss", "MMM dd HH:mm:ss" ]
    }
  }
  if [type] == "apache" {
    grok {
      # See the following URL for a complete list of named patterns
      # logstash/grok ships with by default:
      # https://github.com/logstash/logstash/tree/master/patterns
      #
      # The grok filter will use the below pattern and on successful match use
      # any captured values as new fields in the event.
      match => { "message" => "%{COMBINEDAPACHELOG}" }
    }
    date {
      # Try to pull the timestamp from the 'timestamp' field (parsed above with
      # grok). The apache time format looks like: "18/Aug/2011:05:44:34 -0700"
      match => [ "timestamp", "dd/MMM/yyyy:HH:mm:ss Z" ]
    }
  }
}


output {
  elasticsearch {
    embedded => true
  }
}

[root@Test-Logstash conf.d]#