First release to Github. Long used V2.01

cron/job

@@ -0,0 +1,8 @@
+
+#Netlog components
+#* * * * *       root    /bin/bash /usr/share/syslog-ng/sbin/logscavenger.sh 1>/dev/null 2>&1
+#* * * * *       root    /usr/bin/php /usr/share/syslog-ng/php/lograte.php 1>/dev/null 2>&1
+#0 1 * * *       root    /usr/bin/php /usr/share/syslog-ng/php/logarchiver.php 1>/dev/null 2>&1
+#0 1 * * *       root    /bin/bash /usr/share/syslog-ng/sbin/prunelograte.sh 1>/dev/null 2>&1
+##0 5 11 * *      root    /usr/bin/php /usr/share/syslog-ng/php/prunelog.php 1>/dev/null 2>&1
+*/5 * * * *     root    [[ ! `systemctl is-active logparser` == "active" ]] && systemctl restart logparser

db/database.sql

@@ -0,0 +1,82 @@
+CREATE DATABASE `syslog`;
+CREATE DATABASE `netlogconfig`;
+
+GRANT ALL ON syslog.* TO 'syslog'@'localhost' IDENTIFIED BY 'WonFaznu$(s#3nCi';
+GRANT ALL ON netlogconfig.* TO 'syslog'@'localhost' IDENTIFIED BY 'WonFaznu$(s#3nCi';
+
+USE syslog;
+CREATE TABLE `template` (
+  `id` int(15) unsigned NOT NULL AUTO_INCREMENT,
+  `HOST` varchar(39) NOT NULL,
+  `FAC` varchar(255) NOT NULL,
+  `PRIO` varchar(255) NOT NULL,
+  `LVL` varchar(255) NOT NULL,
+  `TAG` varchar(255) NOT NULL,
+  `DAY` varchar(10) NOT NULL,
+  `TIME` varchar(8) NOT NULL,
+  `PROG` varchar(255) NOT NULL,
+  `MSG` text NOT NULL,
+  PRIMARY KEY (`id`),
+  KEY `HOST` (`HOST`),
+  KEY `DAY` (`DAY`),
+  KEY `TIME` (`TIME`)
+) ENGINE=MyISAM DEFAULT CHARSET=latin1;
+
+USE netlogconfig;
+CREATE TABLE `hostnames` (
+  `id` int(10) unsigned NOT NULL auto_increment,
+  `hostip` text NOT NULL,
+  `hostname` text NOT NULL,
+  `hosttype` int(10) unsigned NOT NULL,
+  PRIMARY KEY  (`id`)
+) ENGINE=MyISAM DEFAULT CHARSET=latin1;
+
+CREATE TABLE `hosttype` (
+  `id` int(10) unsigned NOT NULL auto_increment,
+  `name` text NOT NULL,
+  PRIMARY KEY  (`id`)
+) ENGINE=MyISAM;
+
+CREATE TABLE `lograte` (
+  `id` int(10) NOT NULL auto_increment,
+  `hostnameid` int(10) NOT NULL,
+  `timestamp` timestamp NOT NULL default CURRENT_TIMESTAMP on update CURRENT_TIMESTAMP,
+  `1min` float default NULL,
+  `5min` float default NULL,
+  `10min` float default NULL,
+  PRIMARY KEY  (`id`)
+) ENGINE=MyISAM;
+
+CREATE TABLE `lograteconf` (
+  `hostnameid` int(10) NOT NULL,
+  `samplerate` int(10) default NULL
+) ENGINE=MyISAM;
+
+CREATE TABLE `logscavenger` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `keyword` varchar(100) DEFAULT NULL,
+  `dateadded` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+  `active` int(1) NOT NULL,
+  `datedeleted` timestamp NULL DEFAULT NULL,
+  `emailrcpt` varchar(255) DEFAULT NULL,
+  `emailgroup` int(2) DEFAULT NULL,
+  KEY `id` (`id`)
+) ENGINE=MyISAM DEFAULT CHARSET=latin1;
+
+CREATE TABLE `emailgroups` (
+  `id` int(10) unsigned NOT NULL AUTO_INCREMENT,
+  `groupname` varchar(40) NOT NULL,
+  `recepients` text NOT NULL,
+  `active` int(1) DEFAULT '1',
+  PRIMARY KEY (`id`)
+) ENGINE=MyISAM DEFAULT CHARSET=latin1;
+
+CREATE TABLE `logcache` (
+  `id` int(10) unsigned NOT NULL AUTO_INCREMENT,
+  `timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  `host` varchar(39) NOT NULL,
+  `msg` text NOT NULL,
+  PRIMARY KEY (`id`),
+  KEY `host` (`host`),
+  KEY `timestamp` (`timestamp`)
+) ENGINE=MyISAM DEFAULT CHARSET=latin1;

install.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+
+if [ ! -d /usr/share/syslog-ng ]; then
+	printf "No Syslog-NG directory found at designated location, please correct manually\n";
+	exit 1
+elif [ -d /usr/share/syslog-ng ]; then
+	printf "copying...\n";
+	cp -R usrsharesyslog-ng/etc /usr/share/syslog-ng;
+	cp -R usrsharesyslog-ng/php /usr/share/syslog-ng;
+	cp -R usrsharesyslog-ng/sbin /usr/share/syslog-ng;
+fi
+
+if [ ! -d /var/www/html ]; then
+	printf "No HTML directory found at designated location, please correct manually\n";
+	exit 1
+elif [ -d /var/www/html ]; then
+	printf "copying...\n";
+        cp -R varwww/html /var/www/;
+fi
+
+if [ ! -f /etc/crontab ]; then
+	printf "No crontab file found, something is wrong with your system\n";
+	exit 1
+elif [ -f /etc/crontab ]; then
+	printf "setting cronjobs, but not active in /etc/cron.d/netlog\n";
+	cp cron/job >> /etc/cron.d/netlog;
+fi
+
+if [ -d /usr/share/fonts ]; then
+	tar xf extsrc/truetype.tgz -C /usr/share/fonts;
+fi
+if [ -d /usr/share/php ]; then
+	tar xf extsrc/jpgraph-4.0.2.tar.gz -C extsrc;
+	mv extsrc/jpgraph-4.0.2/src /usr/share/php/jpgraph;
+	restorecon -R /usr/share/php/jpgraph
+fi
+
+if [ -f /root/.my.cnf ]; then
+	mysql < db/database.sql
+fi
+
+printf "done!\n\nPlease make sure the webserver with php is working and the MySQL db is in order, correct the passwords in /usr/share/etc/logparser.conf and /var/www/html/config/config.php\n";
+printf "then, uncomment the jobs in /etc/crontab\n";
+
+exit 0

usrsharesyslog-ng/etc/logparser.conf

@@ -0,0 +1,27 @@
+<?php
+
+// Versioning
+
+	$VERSION = "v1.1";
+	$NAME = "Syslog-ng to MySQL parser";
+
+// Mysql DB Information
+
+	$db_NAME = "syslog";
+	$db_archNAME = "syslogarchive";
+	$db_USER = "syslog";
+	$db_PASS = "WonFaznu$(s#3nCi";
+	$db_HOST = "127.0.0.1";
+	$db_logHOST = "netlog";
+	$db_archHOST = "netlogarchive";
+
+// Fifo socket
+
+	$log_fifo = "/var/log/syslog.fifo";
+
+// Mail
+
+$mail_from = "no-reply@domain.tld";
+$mail_rcpt = "john.doe@domain.tld";
+
+?>

usrsharesyslog-ng/php/logarchiver.php

@@ -0,0 +1,62 @@
+<?php
+
+// This script will be called on a daily bassis to aggergate logging
+// In this setup logging is divided as per following rules:
+// - A table per day per host for the first 14 days of logging
+// - A table per host per month for the current and last months of logging
+// - Anything beyond last month will be archived
+// This way the most active tables will remain reasonable in size, thus queryable :P
+
+// Including logparses variables
+
+include("/usr/share/syslog-ng/etc/logparser.conf");
+
+// Create and check database link
+
+$db_link = mysqli_connect($db_HOST, $db_USER, $db_PASS, $db_NAME);
+if (!$db_link) {
+	die('Could not connect to MySQL server: ' . mysqli_error());
+}
+
+if (!mysqli_select_db($db_link, $db_NAME)) {
+	die('Unable to select DB: ' . mysqli_error());
+}
+
+
+$today = date("Y_m_d");
+$archinterval = date("Y_m_d",strtotime("-14 days"));
+
+$query = "SELECT TABLE_NAME FROM information_schema.TABLES WHERE TABLE_SCHEMA='syslog' AND TABLE_NAME NOT IN ('template') AND CREATE_TIME <= '$archinterval' ORDER BY CREATE_TIME";
+
+$result = mysqli_query($db_link, $query);
+
+while ( $tables = mysqli_fetch_assoc($result)) {
+	$table_name = $tables['TABLE_NAME'];
+	$table = explode('_DATE_',$table_name);
+  if(!isset($table['1'])) { continue; }
+
+	if ( preg_match('/\d{4}_\d{2}_\d{2}/',$table['1'])) {
+	$host = $table['0'];
+
+	$daysplitup = explode('_',$table['1']);
+	$year = $daysplitup['0'];
+	$month = $daysplitup['1'];
+	$monthname = date('M', mktime(0,0,0, $month));
+
+	$dsttable = $host . "_DATE_" . $year . "_" . $monthname;
+
+	$dstquery = "CREATE TABLE IF NOT EXISTS $dsttable LIKE template";
+	$dstresult = mysqli_query($db_link, $dstquery);
+	unset($dstquery, $dstresult);
+
+	$archquery = "INSERT INTO $dsttable (HOST, FAC, PRIO, LVL, TAG, DAY, TIME, PROG, MSG) SELECT HOST, FAC, PRIO, LVL, TAG, DAY, TIME, PROG, MSG FROM $table_name";
+	$archresult = mysqli_query($db_link, $archquery);
+	unset($archquery, $archresult);
+
+	$dropquery = "DROP TABLE $table_name";
+	$dropresult = mysqli_query($db_link, $dropquery);
+	unset($dropquery, $dropresult);
+	}
+
+}
+?>

usrsharesyslog-ng/php/logparser.php

@@ -0,0 +1,115 @@
+<?php
+
+// This script will work together with syslog-ng to parse logging information to a MySQL database
+// We will read the logging from a fifo pipe in which syslog-ng with write the syslog info
+// Standard syslog-ng - to - MySQL solutions would write all logging to one single table.
+// This script will parse and divide logging into:
+// In this setup logging is divided as per following rules:
+// - A table per day per host for the first 14 days of logging
+// - A table per host per month for the current and last months of logging
+// - Anything beyond last month will be archived
+// This way the most active tables will remain reasonable in size, thus queryable :P
+
+// Including logparses variables
+include("/usr/share/syslog-ng/etc/logparser.conf");
+
+$DAY = "";
+$HOST = "";
+
+// Create and check database link
+
+$db_link = mysqli_connect($db_HOST, $db_USER, $db_PASS, $db_NAME);
+if (!$db_link) {
+	die('Could not connect to MySQL server: ' . mysqli_error());
+}
+
+if (!mysqli_select_db($db_link, $db_NAME)) {
+	die('Unable to select DB: ' . mysqli_error());
+}
+
+// Check if fifo socket exists
+
+if ( file_exists($log_fifo)) {
+	read_fifo();
+} else {
+	exec("mkfifo $log_fifo", $output, $ret);
+	if ( $ret == "0" ) {
+		read_fifo();
+	} else {
+		die("Unable to create fifo socket: $log_fifo");
+	}
+}
+
+function read_fifo() {
+	global $log_fifo;
+
+	while($fifo = fopen("$log_fifo",'r') ) {
+		$buffer = fgets($fifo);
+		$logitems = explode(' _,_ ', $buffer);
+		parse_log($logitems);
+	}
+}
+
+function parse_log($logitems) {
+	global $DAY;
+	global $HOST;
+	global $db_link;
+
+	// Parse items on line
+	$fields = '';
+	$values = '';
+
+	foreach($logitems as $linepart) {
+		$item = explode('_:_',$linepart);
+		if(!isset($item['1'])) { continue; }
+		${$item['0']} = trim($item['1']);
+
+		$fields .= $item['0'] . ", ";
+		$values .= "'" . trim($item['1']) . "',";
+	}
+
+	if(isset($fields) && isset($values)) {
+		$trimmedfields = trim($fields,', ');
+		$trimmedvalues = trim($values,',');
+
+		// Should hostname be empty, we will default to Unidentified Host Object table ;)
+		if($HOST == '') {
+			$HOST = 'UHO';
+		}
+		//echo "INSERT INTO $HOST ($trimmedfields) VALUES ($trimmedvalues)" . "\n";
+
+		unset($fields, $values);
+
+		// NetLog Scavenger to NetAlert for visability
+		if (false != preg_match('/%LOGSCAVENGER%/',$trimmedvalues)) {
+			 $HOST = "127.0.0.2";
+		}
+
+		// Create tablename
+		$HOST_us = str_replace('.','_',$HOST);
+		$DAY_us = str_replace('-','_',$DAY);
+		$tablename = 'HST_' . $HOST_us . '_DATE_' . $DAY_us;
+
+		$query = "INSERT INTO $tablename ($trimmedfields) VALUES ($trimmedvalues)";
+		$result = mysqli_query($db_link, $query);
+
+		if(!$result) {
+			create_table($tablename);
+			$query = "INSERT INTO $tablename ($trimmedfields) VALUES ($trimmedvalues)";
+			$result = mysqli_query($db_link, $query);
+		}
+	}
+}
+
+function create_table($tablename) {
+	global $db_link;
+
+	$query = "CREATE TABLE IF NOT EXISTS $tablename LIKE template";
+	$result = mysqli_query($db_link, $query);
+	if(!$result) {
+		die("Failed to create table $tablename");
+	}
+}
+
+?>
+