ENH: keep spitting out logging to the screen in LogCaptureTestCases if HEAVYDEBUG

Conflicts:
	fail2ban/tests/clientreadertestcase.py -- removed not needed
        time in imports

ChangeLog

@@ -48,12 +48,16 @@ ver. 0.9.1 (2014/xx/xx) - better, faster, stronger
    * cyrus-imap -- also catch also failed logins via secured (imaps/pop3s).
      Regression was introduced while strengthening failregex in 0.8.11 (bd175f)
      Debian bug #755173
-   * postfix-sasl -- added journalmatch.  Thanks Luc Maisonobe
+   * postfix-sasl - added journalmatch.  Thanks Luc Maisonobe
+   * postfix* - match with a new daemon string (postfix/submission/smtpd).
+     Closes gh-804 .  Thanks Paul Traina
+   * apache - added filter for AH01630 client denied by server configuration.
 
 - New features:
    - New filters:
      - monit  Thanks Jason H Martin
      - directadmin  Thanks niorg
+     - apache-shellshock  Thanks Eugene Hopkinson (SlowRiot)
    - New actions:
      - symbiosis-blacklist-allports  for Bytemark symbiosis firewall
    - fail2ban-client can fetch the running server version

THANKS

@@ -34,6 +34,7 @@ David Nutter
 Derek Atkins
 Eric Gerbier
 Enrico Labedzki
+Eugene Hopkinson (SlowRiot)
 ftoppi
 François Boulogne
 Frédéric
@@ -81,6 +82,7 @@ Mika (mkl)
 Nick Munger
 onorua
 Paul Marrapese
+Paul Traina
 Noel Butler
 Patrick Börjesson
 Pressy

config/filter.d/apache-auth.conf

@@ -10,7 +10,7 @@ before = apache-common.conf
 [Definition]
 
 
-failregex = ^%(_apache_error_client)s (AH01797: )?client denied by server configuration: (uri )?\S*(, referer: \S+)?\s*$
+failregex = ^%(_apache_error_client)s (AH(01797|01630): )?client denied by server configuration: (uri )?\S*(, referer: \S+)?\s*$
             ^%(_apache_error_client)s (AH01617: )?user .*? authentication failure for "\S*": Password Mismatch(, referer: \S+)?$
             ^%(_apache_error_client)s (AH01618: )?user .*? not found(: )?\S*(, referer: \S+)?\s*$
             ^%(_apache_error_client)s (AH01614: )?client used wrong authentication scheme: \S*(, referer: \S+)?\s*$

config/filter.d/apache-shellshock.conf

@@ -0,0 +1,26 @@
+# Fail2Ban filter to block web requests containing custom headers attempting to exploit the shellshock bug
+#
+#
+
+[INCLUDES]
+
+# overwrite with apache-common.local if _apache_error_client is incorrect.
+before = apache-common.conf
+
+[Definition]
+
+failregex = ^%(_apache_error_client)s (AH01215: )?/bin/(ba)?sh: warning: HTTP_.*?: ignoring function definition attempt(, referer: \S+)?\s*$
+            ^%(_apache_error_client)s (AH01215: )?/bin/(ba)?sh: error importing function definition for `HTTP_.*?'(, referer: \S+)?\s*$
+
+ignoreregex = 
+
+
+# DEV Notes:
+#
+# https://wiki.apache.org/httpd/ListOfErrors for apache error IDs
+#
+# example log lines: 
+# [Thu Sep 25 09:27:18.813902 2014] [cgi:error] [pid 16860] [client 89.207.132.76:59635] AH01215: /bin/bash: warning: HTTP_TEST: ignoring function definition attempt
+# [Thu Sep 25 09:29:56.141832 2014] [cgi:error] [pid 16864] [client 162.247.73.206:41273] AH01215: /bin/bash: error importing function definition for `HTTP_TEST'
+#
+# Author: Eugene Hopkinson (riot@riot.so)

config/filter.d/postfix-sasl.conf

@@ -7,10 +7,12 @@ before = common.conf
 
 [Definition]
 
-_daemon = postfix/smtpd
+_daemon = postfix/(submission/)?smtp(d|s)
 
 failregex = ^%(__prefix_line)swarning: [-._\w]+\[<HOST>\]: SASL (?:LOGIN|PLAIN|(?:CRAM|DIGEST)-MD5) authentication failed(: [ A-Za-z0-9+/]*={0,2})?\s*$
 
+ignoreregex = 
+
 [Init]
 
 journalmatch = _SYSTEMD_UNIT=postfix.service

config/filter.d/postfix.conf

@@ -10,7 +10,7 @@ before = common.conf
 
 [Definition]
 
-_daemon = postfix/smtpd
+_daemon = postfix/(submission/)?smtp(d|s)
 
 failregex = ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[<HOST>\]: 554 5\.7\.1 .*$
             ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[<HOST>\]: 450 4\.7\.1 : Helo command rejected: Host not found; from=<> to=<> proto=ESMTP helo= *$

config/jail.conf

@@ -283,6 +283,11 @@ port     = http,https
 logpath  = %(apache_error_log)s
 maxretry = 2
 
+[apache-shellshock]
+
+port    = http,https
+logpath = $(apache_error_log)s
+maxretry = 1
 
 [nginx-http-auth]
 

fail2ban/tests/action_d/test_badips.py

@@ -22,11 +22,7 @@ import unittest
 import sys
 
 from ..dummyjail import DummyJail
-
-if os.path.exists('config/fail2ban.conf'):
-	CONFIG_DIR = "config"
-else:
-	CONFIG_DIR='/etc/fail2ban'
+from ..utils import CONFIG_DIR
 
 if sys.version_info >= (2,7):
 	class BadIPsActionTest(unittest.TestCase):

fail2ban/tests/action_d/test_smtp.py

@@ -30,10 +30,7 @@ else:
 
 from ..dummyjail import DummyJail
 
-if os.path.exists('config/fail2ban.conf'):
-	CONFIG_DIR = "config"
-else:
-	CONFIG_DIR='/etc/fail2ban'
+from ..utils import CONFIG_DIR
 
 class TestSMTPServer(smtpd.SMTPServer):
 

fail2ban/tests/clientreadertestcase.py

@@ -21,7 +21,7 @@ __author__ = "Cyril Jaquier, Yaroslav Halchenko"
 __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2011-2013 Yaroslav Halchenko"
 __license__ = "GPL"
 
-import os, glob, shutil, tempfile, unittest
+import os, glob, shutil, tempfile, unittest, re
 
 from ..client.configreader import ConfigReader
 from ..client.jailreader import JailReader
@@ -32,8 +32,10 @@ from ..client.configurator import Configurator
 from .utils import LogCaptureTestCase
 
 TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "files")
+
+from .utils import CONFIG_DIR
+
 STOCK = os.path.exists(os.path.join('config','fail2ban.conf'))
-CONFIG_DIR='config' if STOCK else '/etc/fail2ban'
 
 IMPERFECT_CONFIG = os.path.join(os.path.dirname(__file__), 'config')
 
@@ -333,6 +335,36 @@ class FilterReaderTest(unittest.TestCase):
 		self.assertRaises(ValueError, FilterReader.convert, filterReader)
 
 
+class JailsReaderTestCache(LogCaptureTestCase):
+
+	def testTestJailConfCache(self):
+		basedir = tempfile.mkdtemp("fail2ban_conf")
+		try:
+			shutil.rmtree(basedir)
+			shutil.copytree(CONFIG_DIR, basedir)
+			shutil.copy(CONFIG_DIR + '/jail.conf', basedir + '/jail.local')
+			shutil.copy(CONFIG_DIR + '/fail2ban.conf', basedir + '/fail2ban.local')
+
+			# read whole configuration like a file2ban-client ...
+			configurator = Configurator()
+			configurator.setBaseDir(basedir)
+			configurator.readEarly()
+			configurator.getEarlyOptions()
+			configurator.readAll()
+			# from here we test a cache :
+			self.assertTrue(configurator.getOptions(None))
+			cnt = 0
+			for s in self.getLog().rsplit('\n'):
+				if re.match(r"^Reading files: .*jail.local", s):
+					cnt += 1
+			# if cnt > 2:
+			# 	self.printLog()
+			self.assertFalse(cnt > 2, "Too many times reading of config files, cnt = %s" % cnt)
+			self.assertFalse(cnt <= 0)
+		finally:
+			shutil.rmtree(basedir)
+
+
 class JailsReaderTest(LogCaptureTestCase):
 
 	def testProvidingBadBasedir(self):

fail2ban/tests/files/logs/apache-auth

@@ -19,6 +19,9 @@
 # failJSON: { "time": "2013-07-20T21:34:49", "match": true , "host": "127.0.0.1" } 
 [Sat Jul 20 21:34:49.453232 2013] [access_compat:error] [pid 17512:tid 140123104306944] [client 127.0.0.1:51380] AH01797: client denied by server configuration: /var/www/html/noentry/cant_get_me.html
 
+# failJSON: { "time": "2014-09-14T21:44:43", "match": true , "host": "192.3.9.178" } 
+[Sun Sep 14 21:44:43.008606 2014] [authz_core:error] [pid 10691] [client 192.3.9.178:44271] AH01630: client denied by server configuration: /var/www/html/noentry/cant_get_me.html
+
 # wget --http-user='' --http-password='' http://localhost/basic/file/cant_get_me.html -O /dev/null
 # failJSON: { "time": "2013-07-17T23:14:37", "match": true , "host": "127.0.0.1" }
 [Wed Jul 17 23:14:37 2013] [error] [client 127.0.0.1] user  not found: /basic/anon/cant_get_me.html

fail2ban/tests/files/logs/apache-shellshock

@@ -0,0 +1,4 @@
+# failJSON: { "time": "2014-09-25T09:27:18", "match": true , "host": "89.207.132.76" }
+[Thu Sep 25 09:27:18.813902 2014] [cgi:error] [pid 16860] [client 89.207.132.76:59635] AH01215: /bin/bash: warning: HTTP_TEST: ignoring function definition attempt
+# failJSON: { "time": "2014-09-25T09:29:56", "match": true , "host": "162.247.73.206" }
+[Thu Sep 25 09:29:56.141832 2014] [cgi:error] [pid 16864] [client 162.247.73.206:41273] AH01215: /bin/bash: error importing function definition for `HTTP_TEST'

fail2ban/tests/files/logs/postfix-sasl

@@ -5,3 +5,6 @@ Dec  2 22:24:22 hel postfix/smtpd[7676]: warning: 114-44-142-233.dynamic.hinet.n
 # failJSON: { "time": "2005-03-10T13:33:30", "match": true , "host": "1.1.1.1" }
 Mar 10 13:33:30 gandalf postfix/smtpd[3937]: warning: HOSTNAME[1.1.1.1]: SASL LOGIN authentication failed: authentication failure
 
+#3 Example from postfix post-debian changes to rename to add "submission" to syslog name
+# failJSON: { "time": "2004-09-06T00:44:56", "match": true , "host": "82.221.106.233" }
+Sep  6 00:44:56 trianon postfix/submission/smtpd[11538]: warning: unknown[82.221.106.233]: SASL LOGIN authentication failed: UGFzc3dvcmQ6

fail2ban/tests/samplestestcase.py

@@ -32,13 +32,9 @@ else:
 
 from ..server.filter import Filter
 from ..client.filterreader import FilterReader
-from .utils import setUpMyTime, tearDownMyTime
+from .utils import setUpMyTime, tearDownMyTime, CONFIG_DIR
 
 TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "files")
-if os.path.exists('config/fail2ban.conf'):
-	CONFIG_DIR = "config"
-else:
-	CONFIG_DIR='/etc/fail2ban'
 
 class FilterSamplesRegex(unittest.TestCase):
 

fail2ban/tests/utils.py

@@ -25,6 +25,7 @@ __license__ = "GPL"
 import logging
 import os
 import re
+import sys
 import time
 import unittest
 from StringIO import StringIO
@@ -34,6 +35,15 @@ from ..helpers import getLogger
 
 logSys = getLogger(__name__)
 
+CONFIG_DIR = os.environ.get('FAIL2BAN_CONFIG_DIR', None)
+
+if not CONFIG_DIR:
+# Use heuristic to figure out where configuration files are
+	if os.path.exists(os.path.join('config','fail2ban.conf')):
+		CONFIG_DIR = 'config'
+	else:
+		CONFIG_DIR = '/etc/fail2ban'
+
 def mtimesleep():
 	# no sleep now should be necessary since polling tracks now not only
 	# mtime but also ino and size
@@ -102,6 +112,7 @@ def gatherTests(regexps=None, no_network=False):
 	tests.addTest(unittest.makeSuite(clientreadertestcase.JailReaderTest))
 	tests.addTest(unittest.makeSuite(clientreadertestcase.FilterReaderTest))
 	tests.addTest(unittest.makeSuite(clientreadertestcase.JailsReaderTest))
+	tests.addTest(unittest.makeSuite(clientreadertestcase.JailsReaderTestCache))
 	# CSocket and AsyncServer
 	tests.addTest(unittest.makeSuite(sockettestcase.Socket))
 	# Misc helpers
@@ -195,6 +206,8 @@ class LogCaptureTestCase(unittest.TestCase):
 		# Let's log everything into a string
 		self._log = StringIO()
 		logSys.handlers = [logging.StreamHandler(self._log)]
+		if self._old_level < logging.DEBUG: # so if HEAVYDEBUG etc -- show them!
+			logSys.handlers += self._old_handlers
 		logSys.setLevel(getattr(logging, 'DEBUG'))
 
 	def tearDown(self):
@@ -207,5 +220,8 @@ class LogCaptureTestCase(unittest.TestCase):
 	def _is_logged(self, s):
 		return s in self._log.getvalue()
 
+	def getLog(self):
+		return self._log.getvalue()
+
 	def printLog(self):
 		print(self._log.getvalue())