fixed possible RCE vulnerability, unset escape variable (default tilde) stops consider "~" char after new-line as composing escape sequence

closes GHSA-m985-3f3v-cwmm for 0.9
jail counter-strike: removed link to site with redirect to malicious page (gh-2868)
2021-07-07 11:46:28 +02:00 · 2020-11-09 12:43:34 +01:00 · 2020-03-18 15:49:18 +01:00 · 2019-05-24 15:53:16 +02:00 · 2019-04-03 23:58:23 +02:00 · 2019-03-14 11:45:46 +01:00
139 changed files with 3050 additions and 1405 deletions
@@ -10,10 +10,13 @@ python:
  # - 3.2
  - 3.3
  - 3.4
-  - pypy3
+  # disabled since setuptools dropped support for Python 3.0 - 3.2
+  # - pypy3
+  - pypy3.3-5.2-alpha1
 before_install:
-  - if [[ $TRAVIS_PYTHON_VERSION == 2* || $TRAVIS_PYTHON_VERSION == 'pypy' ]]; then export F2B_PY_2=true && echo "Set F2B_PY_2"; fi
-  - if [[ $TRAVIS_PYTHON_VERSION == 3* || $TRAVIS_PYTHON_VERSION == 'pypy3' ]]; then export F2B_PY_3=true && echo "Set F2B_PY_3"; fi
+  - echo "running under $TRAVIS_PYTHON_VERSION"
+  - if [[ $TRAVIS_PYTHON_VERSION == 2* || $TRAVIS_PYTHON_VERSION == pypy* && $TRAVIS_PYTHON_VERSION != pypy3* ]]; then export F2B_PY_2=true && echo "Set F2B_PY_2"; fi
+  - if [[ $TRAVIS_PYTHON_VERSION == 3* || $TRAVIS_PYTHON_VERSION == pypy3* ]]; then export F2B_PY_3=true && echo "Set F2B_PY_3"; fi
  - travis_retry sudo apt-get update -qq
  # Set this so sudo executes the correct python binary
  #   Anything not using sudo will already have the correct environment
@@ -41,6 +44,8 @@ script:
  - if [[ "$F2B_PY_3" ]]; then coverage run bin/fail2ban-testcases; fi
  # Use $VENV_BIN (not python) or else sudo will always run the system's python (2.7)
  - sudo $VENV_BIN/pip install .
+  # Doc files should get installed on Travis under Linux
+  - test -e /usr/share/doc/fail2ban/FILTERS
 after_success:
  - coveralls
  - codecov
@@ -227,7 +227,7 @@ Regular expressions (failregex, ignoreregex) assume that the date/time has been
 removed from the log line (this is just how fail2ban works internally ATM).

 If the format is like '<date...> error 1.2.3.4 is evil' then you need to match
-the < at the start so regex should be similar to '^<> <HOST> is evil$' using
+the <> at the start so regex should be similar to '^<> error <HOST> is evil$' using
 <HOST> where the IP/domain name appears in the log line.

 The following general rules apply to regular expressions:
@@ -16,6 +16,8 @@ config/action.d/firewallcmd-allports.conf
 config/action.d/firewallcmd-ipset.conf
 config/action.d/firewallcmd-multiport.conf
 config/action.d/firewallcmd-new.conf
+config/action.d/firewallcmd-rich-logging.conf
+config/action.d/firewallcmd-rich-rules.conf
 config/action.d/hostsdeny.conf
 config/action.d/ipfilter.conf
 config/action.d/ipfw.conf
@@ -31,12 +33,15 @@ config/action.d/iptables-new.conf
 config/action.d/iptables-xt_recent-echo.conf
 config/action.d/mail-buffered.conf
 config/action.d/mail.conf
+config/action.d/mail-whois-common.conf
 config/action.d/mail-whois.conf
 config/action.d/mail-whois-lines.conf
 config/action.d/mynetwatchman.conf
+config/action.d/netscaler.conf
 config/action.d/nftables-allports.conf
 config/action.d/nftables-common.conf
 config/action.d/nftables-multiport.conf
+config/action.d/npf.conf
 config/action.d/nsupdate.conf
 config/action.d/osx-afctl.conf
 config/action.d/osx-ipfw.conf
@@ -52,6 +57,7 @@ config/action.d/sendmail-whois-ipmatches.conf
 config/action.d/sendmail-whois-lines.conf
 config/action.d/sendmail-whois-matches.conf
 config/action.d/shorewall.conf
+config/action.d/shorewall-ipset-proto6.conf
 config/action.d/smtp.py
 config/action.d/symbiosis-blacklist-allports.conf
 config/action.d/ufw.conf
@@ -67,6 +73,7 @@ config/filter.d/apache-modsecurity.conf
 config/filter.d/apache-nohome.conf
 config/filter.d/apache-noscript.conf
 config/filter.d/apache-overflows.conf
+config/filter.d/apache-pass.conf
 config/filter.d/apache-shellshock.conf
 config/filter.d/assp.conf
 config/filter.d/asterisk.conf
@@ -77,13 +84,16 @@ config/filter.d/courier-auth.conf
 config/filter.d/courier-smtp.conf
 config/filter.d/cyrus-imap.conf
 config/filter.d/directadmin.conf
+config/filter.d/domino-smtp.conf
 config/filter.d/dovecot.conf
 config/filter.d/dropbear.conf
+config/filter.d/drupal-auth.conf
 config/filter.d/ejabberd-auth.conf
 config/filter.d/exim-common.conf
 config/filter.d/exim.conf
 config/filter.d/exim-spam.conf
 config/filter.d/freeswitch.conf
+config/filter.d/froxlor-auth.conf
 config/filter.d/groupoffice.conf
 config/filter.d/gssftpd.conf
 config/filter.d/guacamole.conf
@@ -93,6 +103,7 @@ config/filter.d/ignorecommands
 config/filter.d/ignorecommands/apache-fakegooglebot
 config/filter.d/kerio.conf
 config/filter.d/lighttpd-auth.conf
+config/filter.d/mongodb-auth.conf
 config/filter.d/monit.conf
 config/filter.d/murmur.conf
 config/filter.d/mysqld-auth.conf
@@ -124,10 +135,12 @@ config/filter.d/sendmail-auth.conf
 config/filter.d/sendmail-reject.conf
 config/filter.d/sendmail-spam.conf
 config/filter.d/sieve.conf
+config/filter.d/slapd.conf
 config/filter.d/sogo-auth.conf
 config/filter.d/solid-pop3d.conf
 config/filter.d/squid.conf
 config/filter.d/squirrelmail.conf
+config/filter.d/sshd-aggressive.conf
 config/filter.d/sshd.conf
 config/filter.d/sshd-ddos.conf
 config/filter.d/stunnel.conf
@@ -147,6 +160,7 @@ config/paths-opensuse.conf
 config/paths-osx.conf
 CONTRIBUTING.md
 COPYING
+.coveragerc
 DEVELOP
 doc/run-rootless.txt
 fail2ban-2to3
@@ -191,6 +205,7 @@ fail2ban/server/server.py
 fail2ban/server/strptime.py
 fail2ban/server/ticket.py
 fail2ban/server/transmitter.py
+fail2ban/setup.py
 fail2ban-testcases-all
 fail2ban-testcases-all-python3
 fail2ban/tests/action_d/__init__.py
@@ -202,9 +217,11 @@ fail2ban/tests/banmanagertestcase.py
 fail2ban/tests/clientreadertestcase.py
 fail2ban/tests/config/action.d/brokenaction.conf
 fail2ban/tests/config/fail2ban.conf
+fail2ban/tests/config/filter.d/common.conf
 fail2ban/tests/config/filter.d/simple.conf
 fail2ban/tests/config/filter.d/test.conf
 fail2ban/tests/config/filter.d/test.local
+fail2ban/tests/config/filter.d/zzz-generic-example.conf
 fail2ban/tests/config/jail.conf
 fail2ban/tests/config/paths-common.conf
 fail2ban/tests/config/paths-debian.conf
@@ -252,6 +269,7 @@ fail2ban/tests/files/logs/apache-modsecurity
 fail2ban/tests/files/logs/apache-nohome
 fail2ban/tests/files/logs/apache-noscript
 fail2ban/tests/files/logs/apache-overflows
+fail2ban/tests/files/logs/apache-pass
 fail2ban/tests/files/logs/apache-shellshock
 fail2ban/tests/files/logs/assp
 fail2ban/tests/files/logs/asterisk
@@ -263,12 +281,15 @@ fail2ban/tests/files/logs/courier-auth
 fail2ban/tests/files/logs/courier-smtp
 fail2ban/tests/files/logs/cyrus-imap
 fail2ban/tests/files/logs/directadmin
+fail2ban/tests/files/logs/domino-smtp
 fail2ban/tests/files/logs/dovecot
 fail2ban/tests/files/logs/dropbear
+fail2ban/tests/files/logs/drupal-auth
 fail2ban/tests/files/logs/ejabberd-auth
 fail2ban/tests/files/logs/exim
 fail2ban/tests/files/logs/exim-spam
 fail2ban/tests/files/logs/freeswitch
+fail2ban/tests/files/logs/froxlor-auth
 fail2ban/tests/files/logs/groupoffice
 fail2ban/tests/files/logs/gssftpd
 fail2ban/tests/files/logs/guacamole
@@ -276,6 +297,7 @@ fail2ban/tests/files/logs/haproxy-http-auth
 fail2ban/tests/files/logs/horde
 fail2ban/tests/files/logs/kerio
 fail2ban/tests/files/logs/lighttpd-auth
+fail2ban/tests/files/logs/mongodb-auth
 fail2ban/tests/files/logs/monit
 fail2ban/tests/files/logs/murmur
 fail2ban/tests/files/logs/mysqld-auth
@@ -306,11 +328,13 @@ fail2ban/tests/files/logs/sendmail-auth
 fail2ban/tests/files/logs/sendmail-reject
 fail2ban/tests/files/logs/sendmail-spam
 fail2ban/tests/files/logs/sieve
+fail2ban/tests/files/logs/slapd
 fail2ban/tests/files/logs/sogo-auth
 fail2ban/tests/files/logs/solid-pop3d
 fail2ban/tests/files/logs/squid
 fail2ban/tests/files/logs/squirrelmail
 fail2ban/tests/files/logs/sshd
+fail2ban/tests/files/logs/sshd-aggressive
 fail2ban/tests/files/logs/sshd-ddos
 fail2ban/tests/files/logs/stunnel
 fail2ban/tests/files/logs/suhosin
@@ -320,6 +344,7 @@ fail2ban/tests/files/logs/vsftpd
 fail2ban/tests/files/logs/webmin-auth
 fail2ban/tests/files/logs/wuftpd
 fail2ban/tests/files/logs/xinetd-fail
+fail2ban/tests/files/logs/zzz-generic-example
 fail2ban/tests/files/testcase01.log
 fail2ban/tests/files/testcase02.log
 fail2ban/tests/files/testcase03.log
@@ -350,6 +375,8 @@ files/gentoo-confd
 files/gentoo-initd
 files/ipmasq-ZZZzzz_fail2ban.rul
 files/logwatch/fail2ban
+files/logwatch/fail2ban-0.8.log
+files/logwatch/fail2ban-0.9.log
 files/macosx-initd
 files/monit/fail2ban
 files/nagios/check_fail2ban
@@ -367,8 +394,11 @@ man/fail2ban-regex.1
 man/fail2ban-regex.h2m
 man/fail2ban-server.1
 man/fail2ban-server.h2m
+man/fail2ban-testcases.1
+man/fail2ban-testcases.h2m
 man/generate-man
 man/jail.conf.5
+.pylintrc
 README.md
 README.Solaris
 RELEASE
@@ -2,49 +2,51 @@
                        / _|__ _(_) |_  ) |__  __ _ _ _  
                       |  _/ _` | | |/ /| '_ \/ _` | ' \ 
                       |_| \__,_|_|_/___|_.__/\__,_|_||_|
-                       v0.9.4.dev0             2016/??/??
+                       v0.9.7.dev0             2017/??/??

 ## Fail2Ban: ban hosts that cause multiple authentication errors

-Fail2Ban scans log files like `/var/log/auth.log` and bans IP addresses having
+Fail2Ban scans log files like `/var/log/auth.log` and bans IP addresses conducting
 too many failed login attempts. It does this by updating system firewall rules
 to reject new connections from those IP addresses, for a configurable amount
 of time. Fail2Ban comes out-of-the-box ready to read many standard log files,
-such as those for sshd and Apache, and is easy to configure to read any log
-file you choose, for any error you choose.
+such as those for sshd and Apache, and is easily configured to read any log
+file of your choosing, for any error you wish.

-Though Fail2Ban is able to reduce the rate of incorrect authentications
-attempts, it cannot eliminate the risk that weak authentication presents.
-Configure services to use only two factor or public/private authentication
+Though Fail2Ban is able to reduce the rate of incorrect authentication
+attempts, it cannot eliminate the risk presented by weak authentication.
+Set up services to use only two factor, or public/private authentication
 mechanisms if you really want to protect services.

-This README is a quick introduction to Fail2ban. More documentation, FAQ, HOWTOs
-are available in fail2ban(1) manpage and on the website http://www.fail2ban.org
+This README is a quick introduction to Fail2Ban. More documentation, FAQ, and HOWTOs
+to be found on fail2ban(1) manpage and the website: https://www.fail2ban.org

 Installation:
 -------------

-**It is possible that Fail2ban is already packaged for your distribution.  In
-this case, you should use it instead.**
+**It is possible that Fail2Ban is already packaged for your distribution.  In
+this case, you should use that instead.**

 Required:
- [Python2 >= 2.6 or Python >= 3.2](http://www.python.org) or [PyPy](http://pypy.org)
+- [Python2 >= 2.6 or Python >= 3.2](https://www.python.org) or [PyPy](https://pypy.org)

 Optional:
- [pyinotify >= 0.8.3](https://github.com/seb-m/pyinotify)
-  - Linux >= 2.6.13
+- [pyinotify >= 0.8.3](https://github.com/seb-m/pyinotify), may require:
+  * Linux >= 2.6.13
 - [gamin >= 0.0.21](http://www.gnome.org/~veillard/gamin)
- [systemd >= 204](http://www.freedesktop.org/wiki/Software/systemd)
+- [systemd >= 204](http://www.freedesktop.org/wiki/Software/systemd) and python bindings:
+  * [python-systemd package](https://www.freedesktop.org/software/systemd/python-systemd/index.html)
 - [dnspython](http://www.dnspython.org/)

-To install, just do:

-    tar xvfj fail2ban-0.9.4.tar.bz2
-    cd fail2ban-0.9.4
+To install:
+
+    tar xvfj fail2ban-0.9.7.tar.bz2
+    cd fail2ban-0.9.7
    python setup.py install

 This will install Fail2Ban into the python library directory. The executable
-scripts are placed into `/usr/bin`, and configuration under `/etc/fail2ban`.
+scripts are placed into `/usr/bin`, and configuration in `/etc/fail2ban`.

 Fail2Ban should be correctly installed now. Just type:

@@ -86,7 +88,7 @@ Contact:
 See [CONTRIBUTING.md](https://github.com/fail2ban/fail2ban/blob/master/CONTRIBUTING.md)

 ### You just appreciate this program:
-send kudos to the original author ([Cyril Jaquier](mailto: Cyril Jaquier <cyril.jaquier@fail2ban.org>))
+Send kudos to the original author ([Cyril Jaquier](mailto: Cyril Jaquier <cyril.jaquier@fail2ban.org>))
 or *better* to the [mailing list](https://lists.sourceforge.net/lists/listinfo/fail2ban-users)
 since Fail2Ban is "community-driven" for years now.

@@ -53,7 +53,7 @@ Preparation

  or an alternative for comparison with previous release

-     git diff 0.9.4 | grep -B2 'index 0000000..' | grep -B1 'new file mode' | sed -n -e '/^diff /s,.* b/,,gp' >> MANIFEST
+     git diff 0.9.7 | grep -B2 'index 0000000..' | grep -B1 'new file mode' | sed -n -e '/^diff /s,.* b/,,gp' >> MANIFEST
     sort MANIFEST | uniq | sponge MANIFEST

 * Run::
@@ -66,24 +66,24 @@ Preparation

  * Which indicates that testcases/files/logs/mysqld.log has been moved or is a directory::

-      tar -C /tmp -jxf dist/fail2ban-0.9.4.tar.bz2
+      tar -C /tmp -jxf dist/fail2ban-0.9.7.tar.bz2

-* clean up current direcory::
+* clean up current directory::

-    diff -rul --exclude \*.pyc . /tmp/fail2ban-0.9.4/
+    diff -rul --exclude \*.pyc . /tmp/fail2ban-0.9.7/

  * Only differences should be files that you don't want distributed.


 * Ensure the tests work from the tarball::

-    cd /tmp/fail2ban-0.9.4/ && bin/fail2ban-testcases
+    cd /tmp/fail2ban-0.9.7/ && bin/fail2ban-testcases

 * Add/finalize the corresponding entry in the ChangeLog

  * To generate a list of committers use e.g.::

-      git shortlog -sn 0.9.4.. | sed -e 's,^[ 0-9\t]*,,g' | tr '\n' '\|' | sed -e 's:|:, :g'
+      git shortlog -sn 0.9.7.. | sed -e 's,^[ 0-9\t]*,,g' | tr '\n' '\|' | sed -e 's:|:, :g'

  * Ensure the top of the ChangeLog has the right version and current date.
  * Ensure the top entry of the ChangeLog has the right version and current date.
@@ -106,7 +106,7 @@ Preparation
 * Tag the release by using a signed (and annotated) tag.  Cut/paste
  release ChangeLog entry as tag annotation::

-    git tag -s 0.9.4
+    git tag -s 0.9.8

 Pre Release
 ===========
@@ -190,14 +190,14 @@ Post Release

 Add the following to the top of the ChangeLog::

-  ver. 0.9.6 (2016/XX/XXX) - wanna-be-released
+  ver. 0.9.9 (2016/XX/XXX) - wanna-be-released
  -----------
  
-  - Fixes:
+  ### Fixes
  
-  - New Features:
+  ### New Features
  
-  - Enhancements:
+  ### Enhancements

 Alter the git shortlog command in the previous section to refer to the just
 released version.
@@ -119,6 +119,7 @@ Thomas Mayer
 Tom Pike
 Tom Hendrikx
 Tomas Pihl
+Thomas Skierlo (phaleas)
 Tony Lawrence
 Tomasz Ciolek
 Tyler
@@ -176,7 +176,7 @@ class Fail2banClient:
 					if showRet:
 						self.__logSocketError()
 					return False
-				except Exception, e:
+				except Exception as e:
 					if showRet:
 						logSys.error(e)
 					return False
@@ -429,7 +429,7 @@ class Fail2banClient:
 						elif not cmd == "":
 							try:
 								self.__processCommand(shlex.split(cmd))
-							except Exception, e:
+							except Exception as e:
 								logSys.error(e)
 			except (EOFError, KeyboardInterrupt):
 				print
@@ -451,7 +451,7 @@ class Fail2banClient:
 			ret = self.__configurator.getOptions(jail)
 			self.__configurator.convertToProtocol()
 			self.__stream = self.__configurator.getConfigStream()
-		except Exception, e:
+		except Exception as e:
 			logSys.error("Failed during configuration: %s" % e)
 			ret = False
 		return ret
@@ -127,7 +127,7 @@ class Fail2banServer:
 								self.__conf["pidfile"],
 								self.__conf["force"])
 			return True
-		except Exception, e:
+		except Exception as e:
 			logSys.exception(e)
 			self.__server.quit()
 			return False
@@ -39,10 +39,18 @@ from fail2ban.version import version

 from fail2ban.tests.utils import gatherTests
 from fail2ban.helpers import FormatterWithTraceBack, getLogger
+from fail2ban.setup import updatePyExec
 from fail2ban.server.mytime import MyTime

 from optparse import OptionParser, Option

+# Update fail2ban-python env to current python version (where f2b-modules located/installed)
+bindir = os.path.dirname(
+	# __file__ seems to be overwritten sometimes on some python versions (e.g. bug of 2.6 by running under cProfile, etc.):
+	sys.argv[0] if os.path.basename(sys.argv[0]) == 'fail2ban-testcases' else __file__
+)
+updatePyExec(bindir)
+
 def get_opt_parser():
 	# use module docstring for help output
 	p = OptionParser(
@@ -1,6 +1,6 @@
 # Fail2ban reporting to badips.com
 #
-# Note: This reports and IP only and does not actually ban traffic. Use 
+# Note: This reports an IP only and does not actually ban traffic. Use
 # another action in the same jail if you want bans to occur.
 #
 # Set the category to the appropriate value before use.
@@ -80,14 +80,17 @@ class BadIPsAction(ActionBase):
 		If invalid `category`, `score`, `banaction` or `updateperiod`.
 	"""

+	TIMEOUT = 10
 	_badips = "http://www.badips.com"
 	def _Request(self, url, **argv):
 		return Request(url, headers={'User-Agent': self.agent}, **argv)

 	def __init__(self, jail, name, category, score=3, age="24h", key=None,
-		banaction=None, bancategory=None, bankey=None, updateperiod=900, agent="Fail2Ban"):
+		banaction=None, bancategory=None, bankey=None, updateperiod=900, agent="Fail2Ban", 
+		timeout=TIMEOUT):
 		super(BadIPsAction, self).__init__(jail, name)

+		self.timeout = timeout
 		self.agent = agent
 		self.category = category
 		self.score = score
@@ -119,7 +122,7 @@ class BadIPsAction(ActionBase):
 		"""
 		try:
 			response = urlopen(
-				self._Request("/".join([self._badips, "get", "categories"])), None, 3)
+				self._Request("/".join([self._badips, "get", "categories"])), timeout=self.timeout)
 		except HTTPError as response:
 			messages = json.loads(response.read().decode('utf-8'))
 			self._logSys.error(
@@ -173,7 +176,7 @@ class BadIPsAction(ActionBase):
 				urlencode({'age': age})])
 			if key:
 				url = "&".join([url, urlencode({'key': key})])
-			response = urlopen(self._Request(url))
+			response = urlopen(self._Request(url), timeout=self.timeout)
 		except HTTPError as response:
 			messages = json.loads(response.read().decode('utf-8'))
 			self._logSys.error(
@@ -358,7 +361,7 @@ class BadIPsAction(ActionBase):
 			url = "/".join([self._badips, "add", self.category, aInfo['ip']])
 			if self.key:
 				url = "?".join([url, urlencode({'key': self.key})])
-			response = urlopen(self._Request(url))
+			response = urlopen(self._Request(url), timeout=self.timeout)
 		except HTTPError as response:
 			messages = json.loads(response.read().decode('utf-8'))
 			self._logSys.error(
@@ -67,7 +67,7 @@ actionunban =
 [Init]

 # Option:  email
-# Notes    server email address, as per blocklise.de account
+# Notes    server email address, as per blocklist.de account
 # Values:  STRING  Default: None
 #
 #email =
@@ -14,7 +14,7 @@
 # Notes.:  command executed once at the start of Fail2Ban.
 # Values:  CMD
 #
-actionstart = ipfw show | fgrep -q 'table(<table>)' || ( ipfw show | awk 'BEGIN { b = 1 } { if ($1 <= b) { b = $1 + 1 } else { e = b } } END { if (e) exit e <br> else exit b }'; num=$?; ipfw -q add $num <blocktype> <block> from table\(<table>\) to me <port>; echo $num > "<startstatefile>" )
+actionstart = ipfw show | fgrep -c -m 1 -s 'table(<table>)' > /dev/null 2>&1 || ( ipfw show | awk 'BEGIN { b = <lowest_rule_num> } { if ($1 < b) {} else if ($1 == b) { b = $1 + 1 } else { e = b } } END { if (e) exit e <br> else exit b }'; num=$?; ipfw -q add $num <blocktype> <block> from table\(<table>\) to me <port>; echo $num > "<startstatefile>" )


 # Option:  actionstop
@@ -81,3 +81,11 @@ block = ip
 # Values:  STRING
 #
 blocktype = unreach port
+
+# Option:  lowest_rule_num
+# Notes:   When fail2ban starts with action and there is no rule for the given table yet
+#          then fail2ban will start looking for an empty slot starting with this rule number.
+# Values:  NUM
+lowest_rule_num = 111
+
+
@@ -40,7 +40,12 @@ actioncheck =
 #          <time>  unix timestamp of the ban time
 # Values:  CMD
 #
-actionban = curl -s -o /dev/null https://www.cloudflare.com/api_json.html -d 'a=ban' -d 'tkn=<cftoken>' -d 'email=<cfuser>' -d 'key=<ip>'
+# API v1
+#actionban = curl -s -o /dev/null https://www.cloudflare.com/api_json.html -d 'a=ban' -d 'tkn=<cftoken>' -d 'email=<cfuser>' -d 'key=<ip>'
+# API v4
+actionban = curl -s -o /dev/null -X POST -H 'X-Auth-Email: <cfuser>' -H 'X-Auth-Key: <cftoken>' \
+            -H 'Content-Type: application/json' -d '{ "mode": "block", "configuration": { "target": "ip", "value": "<ip>" } }' \
+            https://api.cloudflare.com/client/v4/user/firewall/access_rules/rules

 # Option:  actionunban
 # Notes.:  command executed when unbanning an IP. Take care that the
@@ -50,7 +55,12 @@ actionban = curl -s -o /dev/null https://www.cloudflare.com/api_json.html -d 'a=
 #          <time>  unix timestamp of the ban time
 # Values:  CMD
 #
-actionunban = curl -s -o /dev/null https://www.cloudflare.com/api_json.html -d 'a=nul' -d 'tkn=<cftoken>' -d 'email=<cfuser>' -d 'key=<ip>'
+# API v1
+#actionunban = curl -s -o /dev/null https://www.cloudflare.com/api_json.html -d 'a=nul' -d 'tkn=<cftoken>' -d 'email=<cfuser>' -d 'key=<ip>'
+# API v4
+actionunban = curl -s -o /dev/null -X DELETE -H 'X-Auth-Email: <cfuser>' -H 'X-Auth-Key: <cftoken>' \
+              https://api.cloudflare.com/client/v4/user/firewall/access_rules/rules/$(curl -s -X GET -H 'X-Auth-Email: <cfuser>' -H 'X-Auth-Key: <cftoken>' \
+              'https://api.cloudflare.com/client/v4/user/firewall/access_rules/rules?mode=block&configuration_target=ip&configuration_value=<ip>&page=1&per_page=1' | cut -d'"' -f6)

 [Init]

@@ -79,7 +79,7 @@ logpath = /dev/null
 # Notes.:  Your system mail command. Is passed 2 args: subject and recipient
 # Values:  CMD
 #
-mailcmd = mail -s
+mailcmd = mail -E 'set escape' -s

 # Option:  mailargs
 # Notes.:  Additional arguments to mail command. e.g. for standard Unix mail:
@@ -176,7 +176,7 @@ tcpflags =
 # Notes.:  Your system mail command. Is passed 2 args: subject and recipient
 # Values:  CMD
 #
-mailcmd = mail -s
+mailcmd = mail -E 'set escape' -s

 # Option:  mailargs
 # Notes.:  Additional arguments to mail command. e.g. for standard Unix mail:
@@ -19,9 +19,9 @@ before = iptables-common.conf
 [Definition]

 actionstart = ipset create fail2ban-<name> hash:ip timeout <bantime>
-              firewall-cmd --direct --add-rule ipv4 filter <chain> 0 -p <protocol> -m multiport --dports <port> -m set --match-set fail2ban-<name> src -j <blocktype>
+              firewall-cmd --direct --add-rule ipv4 filter <chain> 0 <actiontype> -m set --match-set fail2ban-<name> src -j <blocktype>

-actionstop = firewall-cmd --direct --remove-rule ipv4 filter <chain> 0 -p <protocol> -m multiport --dports <port> -m set --match-set fail2ban-<name> src -j <blocktype>
+actionstop = firewall-cmd --direct --remove-rule ipv4 filter <chain> 0 <actiontype> -m set --match-set fail2ban-<name> src -j <blocktype>
             ipset flush fail2ban-<name>
             ipset destroy fail2ban-<name>

@@ -44,6 +44,23 @@ chain = INPUT_direct

 bantime = 600

+# Option: actiontype
+# Notes.: defines additions to the blocking rule
+# Values: leave empty to block all attempts from the host
+# Default: Value of the multiport
+actiontype = <multiport>
+
+# Option: allports
+# Notes.: default addition to block all ports
+# Usage.: use in jail config:  banaction = firewallcmd-ipset[actiontype=<allports>]
+#         for all protocols:   banaction = firewallcmd-ipset[actiontype=""]
+allports = -p <protocol>
+
+# Option: multiport
+# Notes.: addition to block access only to specific ports
+# Usage.: use in jail config:  banaction = firewallcmd-ipset[actiontype=<multiport>]
+multiport = -p <protocol> -m multiport --dports <port>
+

 # DEV NOTES:
 #
@@ -29,7 +29,7 @@ actioncheck =
 # 	firewall-cmd --zone=<zone> --add-rich-rule="rule family='ipv4' source address='<ip>' port port='<port>' protocol='<protocol>' log prefix='f2b-<name>' level='<level>' limit value='<rate>/m' <blocktype>"
 # service name example:
 #	firewall-cmd --zone=<zone> --add-rich-rule="rule family='ipv4' source address='<ip>' service name='<service>' log prefix='f2b-<name>' level='<level>' limit value='<rate>/m' <blocktype>"
-# Because rich rules can only handle single or a range of ports we must split ports and execute the command for each port. Ports can be single and ranges seperated by a comma or space for an example: http, https, 22-60, 18 smtp 
+# Because rich rules can only handle single or a range of ports we must split ports and execute the command for each port. Ports can be single and ranges separated by a comma or space for an example: http, https, 22-60, 18 smtp 

 actionban = ports="<port>"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --add-rich-rule="rule family='ipv4' source address='<ip>' port port='$p' protocol='<protocol>' log prefix='f2b-<name>' level='<level>' limit value='<rate>/m' <blocktype>"; done
 	   
@@ -27,7 +27,7 @@ actioncheck =
 # 	firewall-cmd --zone=<zone> --add-rich-rule="rule family='ipv4' source address='<ip>' port port='<port>' protocol='<protocol>' <blocktype>"
 # service name example:
 #	firewall-cmd --zone=<zone> --add-rich-rule="rule family='ipv4' source address='<ip>' service name='<service>' <blocktype>"
-# Because rich rules can only handle single or a range of ports we must split ports and execute the command for each port. Ports can be single and ranges seperated by a comma or space for an example: http, https, 22-60, 18 smtp 
+# Because rich rules can only handle single or a range of ports we must split ports and execute the command for each port. Ports can be single and ranges separated by a comma or space for an example: http, https, 22-60, 18 smtp 

 actionban = ports="<port>"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --add-rich-rule="rule family='ipv4' source address='<ip>' port port='$p' protocol='<protocol>' <blocktype>"; done
 	   
@@ -31,8 +31,7 @@ actioncheck =
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = IP=<ip> &&
-            printf %%b "<daemon_list>: $IP\n" >> <file>
+actionban = IP=<ip> && printf %%b "<daemon_list>: $IP\n" >> <file>

 # Option:  actionunban
 # Notes.:  command executed when unbanning an IP. Take care that the
@@ -40,7 +39,7 @@ actionban = IP=<ip> &&
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionunban = echo "/^<daemon_list>: <ip>$/<br>d<br>w<br>q" | ed <file>
+actionunban = IP=$(echo <ip> | sed 's/\./\\./g') && sed -i "/^<daemon_list>: $IP$/d" <file>

 [Init]

@@ -14,7 +14,7 @@ actionstart = printf %%b "Hi,\n
              The jail <name> has been started successfully.\n
              Output will be buffered until <lines> lines are available.\n
              Regards,\n
-              Fail2Ban"|mail -s "[Fail2Ban] <name>: started on `uname -n`" <dest>
+              Fail2Ban"|mail -E 'set escape' -s "[Fail2Ban] <name>: started on `uname -n`" <dest>

 # Option:  actionstop
 # Notes.:  command executed once at the end of Fail2Ban
@@ -25,13 +25,13 @@ actionstop = if [ -f <tmpfile> ]; then
                 These hosts have been banned by Fail2Ban.\n
                 `cat <tmpfile>`
                 Regards,\n
-                 Fail2Ban"|mail -s "[Fail2Ban] <name>: Summary from `uname -n`" <dest>
+                 Fail2Ban"|mail -E 'set escape' -s "[Fail2Ban] <name>: Summary from `uname -n`" <dest>
                 rm <tmpfile>
             fi
             printf %%b "Hi,\n
             The jail <name> has been stopped.\n
             Regards,\n
-             Fail2Ban"|mail -s "[Fail2Ban] <name>: stopped on `uname -n`" <dest>
+             Fail2Ban"|mail -E 'set escape' -s "[Fail2Ban] <name>: stopped on `uname -n`" <dest>

 # Option:  actioncheck
 # Notes.:  command executed once before each actionban command
@@ -52,7 +52,7 @@ actionban = printf %%b "`date`: <ip> (<failures> failures)\n" >> <tmpfile>
                These hosts have been banned by Fail2Ban.\n
                `cat <tmpfile>`
                \nRegards,\n
-                Fail2Ban"|mail -s "[Fail2Ban] <name>: Summary" <dest>
+                Fail2Ban"|mail -E 'set escape' -s "[Fail2Ban] <name>: Summary" <dest>
                rm <tmpfile>
            fi

@@ -17,7 +17,7 @@ before = mail-whois-common.conf
 actionstart = printf %%b "Hi,\n
              The jail <name> has been started successfully.\n
              Regards,\n
-              Fail2Ban"|mail -s "[Fail2Ban] <name>: started on `uname -n`" <dest>
+              Fail2Ban" | <mailcmd> "[Fail2Ban] <name>: started on `uname -n`" <dest>

 # Option:  actionstop
 # Notes.:  command executed once at the end of Fail2Ban
@@ -26,7 +26,7 @@ actionstart = printf %%b "Hi,\n
 actionstop = printf %%b "Hi,\n
             The jail <name> has been stopped.\n
             Regards,\n
-             Fail2Ban"|mail -s "[Fail2Ban] <name>: stopped on `uname -n`" <dest>
+             Fail2Ban" | <mailcmd> "[Fail2Ban] <name>: stopped on `uname -n`" <dest>

 # Option:  actioncheck
 # Notes.:  command executed once before each actionban command
@@ -48,7 +48,7 @@ actionban = printf %%b "Hi,\n
            Lines containing IP:<ip> in <logpath>\n
            `grep -E <grepopts> '(^|[^0-9])<ip>([^0-9]|$)' <logpath>`\n\n
            Regards,\n
-            Fail2Ban"|mail -s "[Fail2Ban] <name>: banned <ip> from  `uname -n`" <dest>
+            Fail2Ban" | <mailcmd> "[Fail2Ban] <name>: banned <ip> from  `uname -n`" <dest>

 # Option:  actionunban
 # Notes.:  command executed when unbanning an IP. Take care that the
@@ -60,6 +60,12 @@ actionunban =

 [Init]

+# Option:  mailcmd
+# Notes.:  Your system mail command. Is passed 2 args: subject and recipient
+# Values:  CMD
+#
+mailcmd = mail -E 'set escape' -s
+
 # Default name of the chain
 #
 name = default
@@ -17,7 +17,7 @@ before = mail-whois-common.conf
 actionstart = printf %%b "Hi,\n
              The jail <name> has been started successfully.\n
              Regards,\n
-              Fail2Ban"|mail -s "[Fail2Ban] <name>: started on `uname -n`" <dest>
+              Fail2Ban"|mail -E 'set escape' -s "[Fail2Ban] <name>: started on `uname -n`" <dest>

 # Option:  actionstop
 # Notes.:  command executed once at the end of Fail2Ban
@@ -26,7 +26,7 @@ actionstart = printf %%b "Hi,\n
 actionstop = printf %%b "Hi,\n
             The jail <name> has been stopped.\n
             Regards,\n
-             Fail2Ban"|mail -s "[Fail2Ban] <name>: stopped on `uname -n`" <dest>
+             Fail2Ban"|mail -E 'set escape' -s "[Fail2Ban] <name>: stopped on `uname -n`" <dest>

 # Option:  actioncheck
 # Notes.:  command executed once before each actionban command
@@ -46,7 +46,7 @@ actionban = printf %%b "Hi,\n
            Here is more information about <ip> :\n
            `%(_whois_command)s`\n
            Regards,\n
-            Fail2Ban"|mail -s "[Fail2Ban] <name>: banned <ip> from `uname -n`" <dest>
+            Fail2Ban"|mail -E 'set escape' -s "[Fail2Ban] <name>: banned <ip> from `uname -n`" <dest>

 # Option:  actionunban
 # Notes.:  command executed when unbanning an IP. Take care that the
@@ -13,7 +13,7 @@
 actionstart = printf %%b "Hi,\n
              The jail <name> has been started successfully.\n
              Regards,\n
-              Fail2Ban"|mail -s "[Fail2Ban] <name>: started  on `uname -n`" <dest>
+              Fail2Ban"|mail -E 'set escape' -s "[Fail2Ban] <name>: started  on `uname -n`" <dest>

 # Option:  actionstop
 # Notes.:  command executed once at the end of Fail2Ban
@@ -22,7 +22,7 @@ actionstart = printf %%b "Hi,\n
 actionstop = printf %%b "Hi,\n
             The jail <name> has been stopped.\n
             Regards,\n
-             Fail2Ban"|mail -s "[Fail2Ban] <name>: stopped on `uname -n`" <dest>
+             Fail2Ban"|mail -E 'set escape' -s "[Fail2Ban] <name>: stopped on `uname -n`" <dest>

 # Option:  actioncheck
 # Notes.:  command executed once before each actionban command
@@ -40,7 +40,7 @@ actionban = printf %%b "Hi,\n
            The IP <ip> has just been banned by Fail2Ban after
            <failures> attempts against <name>.\n
            Regards,\n
-            Fail2Ban"|mail -s "[Fail2Ban] <name>: banned <ip> from `uname -n`" <dest>
+            Fail2Ban"|mail -E 'set escape' -s "[Fail2Ban] <name>: banned <ip> from `uname -n`" <dest>

 # Option:  actionunban
 # Notes.:  command executed when unbanning an IP. Take care that the
@@ -0,0 +1,33 @@
+# Fail2ban Citrix Netscaler Action
+# by Juliano Jeziorny
+# juliano@jeziorny.eu
+#
+# The script will add offender IPs to a dataset on netscaler, the dataset can then be used to block the IPs at a cs/vserver or global level
+# This dataset is then used to block IPs using responder policies on the netscaler.
+# 
+# The script assumes using HTTPS with unsecure certificate to access the netscaler, 
+# if you have a valid certificate installed remove the -k from the curl lines, or if you want http change it accordingly (and remove the -k)
+# 
+# This action depends on curl
+#
+# You need to populate the 3 options inside Init
+#
+# ns_host: IP or hostname of netslcaer appliance
+# ns_auth: username:password, suggest base64 encoded for a little added security (echo -n "username:password" | base64)
+# ns_dataset:  Name of the netscaler dataset holding the IPs to be blocked.
+# 
+# For further details on how to use it please check http://blog.ckzone.eu/2017/01/fail2ban-action-for-citrix-netscaler.html
+
+[Init]
+ns_host = 
+ns_auth = 
+ns_dataset = 
+
+[Definition]
+actionstart = curl -kH 'Authorization: Basic <ns_auth>' https://<ns_host>/nitro/v1/config
+
+actioncheck = 
+
+actionban = curl -k -H 'Authorization: Basic <ns_auth>' -X PUT -d '{"policydataset_value_binding":{"name":"<ns_dataset>","value":"<ip>"}}' https://<ns_host>/nitro/v1/config/
+
+actionunban = curl -H 'Authorization: Basic <ns_auth>' -X DELETE -k "https://<ns_host>/nitro/v1/config/policydataset_value_binding/<ns_dataset>?args=value:<ip>"
@@ -0,0 +1,61 @@
+# Fail2Ban configuration file
+#
+# NetBSD npf ban/unban
+#
+# Author: Nils Ratusznik <nils@NetBSD.org>
+# Based on pf.conf action file
+#
+
+[Definition]
+
+# Option:  actionstart
+# Notes.:  command executed once at the start of Fail2Ban.
+# Values:  CMD
+#
+# we don't enable NPF automatically, as it will be enabled elsewhere
+actionstart = 
+
+
+# Option:  actionstop
+# Notes.:  command executed once at the end of Fail2Ban
+# Values:  CMD
+#
+# we don't disable NPF automatically either
+actionstop = 
+
+
+# Option:  actioncheck
+# Notes.:  command executed once before each actionban command
+# Values:  CMD
+#
+actioncheck = 
+
+
+# Option:  actionban
+# Notes.:  command executed when banning an IP. Take care that the
+#          command is executed with Fail2Ban user rights.
+# Tags:    <ip>  IP address
+#          <failures>  number of failures
+#          <time>  unix timestamp of the ban time
+# Values:  CMD
+#
+actionban = /sbin/npfctl table <tablename> add <ip>
+
+
+# Option:  actionunban
+# Notes.:  command executed when unbanning an IP. Take care that the
+#          command is executed with Fail2Ban user rights.
+# Tags:    <ip>  IP address
+#          <failures>  number of failures
+#          <time>  unix timestamp of the ban time
+# Values:  CMD
+#
+# note -r option used to remove matching rule
+actionunban = /sbin/npfctl table <tablename> rem <ip>
+
+[Init]
+# Option:  tablename
+# Notes.:  The pf table name.
+# Values:  [ STRING ]
+#
+tablename = fail2ban
@@ -9,7 +9,7 @@
 # connections. So if the attempter goes on trying using the same connection
 # he could even log in. In order to get the same behavior of the iptable
 # action (so that the ban is immediate) the /etc/shorewall/shorewall.conf
-# file should me modified with "BLACKLISTNEWONLY=No". Note that as of
+# file should be modified with "BLACKLISTNEWONLY=No". Note that as of
 # Shorewall 4.5.13 BLACKLISTNEWONLY is deprecated; however the equivalent
 # of BLACKLISTNEWONLY=No can now be achieved by setting BLACKLIST="ALL".
 # 
@@ -48,9 +48,9 @@ actionban = oifs=${IFS}; IFS=.;SEP_IP=( <ip> ); set -- ${SEP_IP}; ADDRESSES=$(di
            PORT=<port>
            DATE=`LC_ALL=C date --date=@<time> +"%%a, %%d %%h %%Y %%T %%z"`
            if [ ! -z "$ADDRESSES" ]; then
-                (printf -- %%b "<header>\n<message>\n<report>\n";
+                (printf -- %%b "<header>\n<message>\n<report>\n\n";
                 date '+Note: Local timezone is %%z (%%Z)';
-                 printf -- %%b "<ipmatches>\n\n<footer>") | <mailcmd> <mailargs> ${ADDRESSES//,/\" \"}
+                 printf -- %%b "\n<ipmatches>\n\n<footer>") | <mailcmd> <mailargs> ${ADDRESSES//,/\" \"}
            fi

 actionunban =
@@ -7,7 +7,7 @@

 [Definition]

-badbotscustom = EmailCollector|WebEMailExtrac|TrackBack/1\.02|sogou music spider
+badbotscustom = EmailCollector|WebEMailExtrac|TrackBack/1\.02|sogou music spider|(?:Mozilla/\d+\.\d+ )?Jorgee
 badbots = Atomic_Email_Hunter/4\.0|atSpider/1\.0|autoemailspider|bwh3_user_agent|China Local Browse 2\.6|ContactBot/0\.2|ContentSmartz|DataCha0s/2\.0|DBrowse 1\.4b|DBrowse 1\.4d|Demo Bot DOT 16b|Demo Bot Z 16b|DSurf15a 01|DSurf15a 71|DSurf15a 81|DSurf15a VA|EBrowse 1\.4b|Educate Search VxB|EmailSiphon|EmailSpider|EmailWolf 1\.00|ESurf15a 15|ExtractorPro|Franklin Locator 1\.8|FSurf15a 01|Full Web Bot 0416B|Full Web Bot 0516B|Full Web Bot 2816B|Guestbook Auto Submitter|Industry Program 1\.0\.x|ISC Systems iRc Search 2\.1|IUPUI Research Bot v 1\.9a|LARBIN-EXPERIMENTAL \(efp@gmx\.net\)|LetsCrawl\.com/1\.0 \+http\://letscrawl\.com/|Lincoln State Web Browser|LMQueueBot/0\.2|LWP\:\:Simple/5\.803|Mac Finder 1\.0\.xx|MFC Foundation Class Library 4\.0|Microsoft URL Control - 6\.00\.8xxx|Missauga Locate 1\.0\.0|Missigua Locator 1\.9|Missouri College Browse|Mizzu Labs 2\.2|Mo College 1\.9|MVAClient|Mozilla/2\.0 \(compatible; NEWT ActiveX; Win32\)|Mozilla/3\.0 \(compatible; Indy Library\)|Mozilla/3\.0 \(compatible; scan4mail \(advanced version\) http\://www\.peterspages\.net/?scan4mail\)|Mozilla/4\.0 \(compatible; Advanced Email Extractor v2\.xx\)|Mozilla/4\.0 \(compatible; Iplexx Spider/1\.0 http\://www\.iplexx\.at\)|Mozilla/4\.0 \(compatible; MSIE 5\.0; Windows NT; DigExt; DTS Agent|Mozilla/4\.0 efp@gmx\.net|Mozilla/5\.0 \(Version\: xxxx Type\:xx\)|NameOfAgent \(CMS Spider\)|NASA Search 1\.0|Nsauditor/1\.x|PBrowse 1\.4b|PEval 1\.4b|Poirot|Port Huron Labs|Production Bot 0116B|Production Bot 2016B|Production Bot DOT 3016B|Program Shareware 1\.0\.2|PSurf15a 11|PSurf15a 51|PSurf15a VA|psycheclone|RSurf15a 41|RSurf15a 51|RSurf15a 81|searchbot admin@google\.com|ShablastBot 1\.0|snap\.com beta crawler v0|Snapbot/1\.0|Snapbot/1\.0 \(Snap Shots&#44; \+http\://www\.snap\.com\)|sogou develop spider|Sogou Orion spider/3\.0\(\+http\://www\.sogou\.com/docs/help/webmasters\.htm#07\)|sogou spider|Sogou web spider/3\.0\(\+http\://www\.sogou\.com/docs/help/webmasters\.htm#07\)|sohu agent|SSurf15a 11 |TSurf15a 11|Under the Rainbow 2\.2|User-Agent\: Mozilla/4\.0 \(compatible; MSIE 6\.0; Windows NT 5\.1\)|VadixBot|WebVulnCrawl\.unknown/1\.0 libwww-perl/5\.803|Wells Search II|WEP Search 00

 failregex = ^<HOST> -.*"(GET|POST|HEAD).*HTTP.*"(?:%(badbots)s|%(badbotscustom)s)"$
@@ -3,7 +3,7 @@
 # This filter is aimed at blocking specific URLs that don't exist. This
 # could be a set of URLs places in a Disallow: directive in robots.txt or
 # just some web services that don't exist caused bots are searching for
-# exploitable content. This filter is designed to have a low false postitive
+# exploitable content. This filter is designed to have a low false positive
 # rate due.
 #
 # An alternative to this is the apache-noscript filter which blocks all
@@ -37,4 +37,4 @@ webroot = /var/www/

 # DEV Notes:
 #
-# Author: Daniel Black
+# Author: Daniel Black
@@ -10,9 +10,10 @@ before = apache-common.conf
 [Definition]


-failregex = ^%(_apache_error_client)s ModSecurity:  (\[.*?\] )*Access denied with code [45]\d\d.*$
+failregex = ^%(_apache_error_client)s ModSecurity:\s+(?:\[(?:\w+ \"[^\"]*\"|[^\]]*)\]\s*)*Access denied with code [45]\d\d

 ignoreregex = 

 # https://github.com/SpiderLabs/ModSecurity/wiki/ModSecurity-2-Data-Formats
 # Author: Daniel Black
+#         Sergey G. Brester aka sebres (review, optimization)
@@ -8,11 +8,15 @@ before = apache-common.conf

 [Definition]

-failregex = ^%(_apache_error_client)s ((AH0013[456]: )?Invalid (method|URI) in request .*( - possible attempt to establish SSL connection on non-SSL port)?|(AH00565: )?request failed: URI too long \(longer than \d+\)|request failed: erroneous characters after protocol string: .*|AH00566: request failed: invalid characters in URI)(, referer: \S+)?$
+failregex = ^%(_apache_error_client)s (?:(?:AH0013[456]: )?Invalid (method|URI) in request\b|(?:AH00565: )?request failed: URI too long \(longer than \d+\)|request failed: erroneous characters after protocol string:|(?:AH00566: )?request failed: invalid characters in URI\b)

 ignoreregex =

 # DEV Notes:
+#
+# [sebres] Because this apache-log could contain very long URLs (and/or referrer), 
+#          the parsing of it anchored way may be very vulnerable (at least as regards 
+#          the system resources, see gh-1790). Thus rewritten without end-anchor ($).
 # 
 # fgrep -r 'URI too long' httpd-2.*
 #   httpd-2.2.25/server/protocol.c:                          "request failed: URI too long (longer than %d)", r->server->limit_req_line);
@@ -23,4 +23,4 @@ ignoreregex =
 # [Thu Sep 25 09:27:18.813902 2014] [cgi:error] [pid 16860] [client 89.207.132.76:59635] AH01215: /bin/bash: warning: HTTP_TEST: ignoring function definition attempt
 # [Thu Sep 25 09:29:56.141832 2014] [cgi:error] [pid 16864] [client 162.247.73.206:41273] AH01215: /bin/bash: error importing function definition for `HTTP_TEST'
 #
-# Author: Eugene Hopkinson (riot@riot.so)
+# Author: Eugene Hopkinson (e.hopkinson@gmail.com)
@@ -1,24 +1,43 @@
-# Fail2Ban filter for Anti-Spam SMTP Proxy Server also known as ASSP
-# 
-#    Honmepage:   http://www.magicvillage.de/~Fritz_Borgstedt/assp/0003D91C-8000001C/
-#    ProjektSite: http://sourceforge.net/projects/assp/?source=directory
+# Fail2Ban filter for Anti-Spam SMTP Proxy Server (ASSP)
+#    Filter works in theory for both ASSP V1 and V2. Recommended ASSP is V2.5.1 or later.
+#    Support for ASSP V1 ended in 2014 so if you are still running ASSP V1 an immediate upgrade is recommended.
+#
+#    Homepage:    http://sourceforge.net/projects/assp/
+#    ProjectSite: http://sourceforge.net/projects/assp/?source=directory
 #
 #

 [Definition] 
+# Note: First three failregex matches below are for ASSP V1 with the remaining being designed for V2. Deleting the V1 regex is recommended but I left it in for compatibility reasons.

 __assp_actions = (?:dropping|refusing)

 failregex = ^(:? \[SSL-out\])? <HOST> max sender authentication errors \(\d{,3}\) exceeded -- %(__assp_actions)s connection - after reply: \d{3} \d{1}\.\d{1}.\d{1} Error: authentication failed: \w+;$
 			^(?: \[SSL-out\])? <HOST> SSL negotiation with client failed: SSL accept attempt failed with unknown error.*:unknown protocol;$
 			^ Blocking <HOST> - too much AUTH errors \(\d{,3}\);$
+			^\s*(?:[\w\-]+\s+)*(?:\[\S+\]\s+)*<HOST> (?:\<\S+@\S+\.\S+\> )*(?:to: \S+@\S+\.\S+ )*relay attempt blocked for(?: \(parsing\))?: \S+$
+			^\s*(?:[\w\-]+\s+)*(?:\[\S+\]\s+)*<HOST> \[SMTP Error\] 535 5\.7\.8 Error: authentication failed:\s+(?:\S+|Connection lost to authentication server|Invalid authentication mechanism|Invalid base64 data in continued response)?$

 ignoreregex = 

 # DEV Notes:
+# V1 Examples matches:
+#   Apr-27-13 02:33:09 Blocking 217.194.197.97 - too much AUTH errors (41);
+#   Dec-29-12 17:10:31 [SSL-out] 200.247.87.82 SSL negotiation with client failed: SSL accept attempt failed with unknown errorerror:140760FC:SSL routines:SSL23_GET_CLIENT_HELLO:unknown protocol;
+#   Dec-30-12 04:01:47 [SSL-out] 81.82.232.66 max sender authentication errors (5) exceeded 
 #
-# Examples: Apr-27-13 02:33:09 Blocking 217.194.197.97 - too much AUTH errors (41);
-#           Dec-29-12 17:10:31 [SSL-out] 200.247.87.82 SSL negotiation with client failed: SSL accept attempt failed with unknown errorerror:140760FC:SSL routines:SSL23_GET_CLIENT_HELLO:unknown protocol;
-#           Dec-30-12 04:01:47 [SSL-out] 81.82.232.66 max sender authentication errors (5) exceeded 
+# V2 Examples matches:
+#   Jul-29-16 16:49:52 m1-25391-06124 [Worker_1] [TLS-out] [RelayAttempt] 0.0.0.0 <user@example.com> to: user@example.org relay attempt blocked for: someone@example.org
+#   Jul-30-16 16:59:42 [Worker_1] [TLS-out] 0.0.0.0 [SMTP Error] 535 5.7.8 Error: authentication failed: UGFzc3dvcmQ6
+#   Jul-30-16 00:15:36 m1-52131-09651 [Worker_1] [TLS-out] 0.0.0.0 [SMTP Error] 535 5.7.8 Error: authentication failed: UGFzc3dvcmQ6
+#   Jul-31-16 06:45:59 [Worker_1] [TLS-in] [TLS-out] 0.0.0.0 [SMTP Error] 535 5.7.8 Error: authentication failed:
+#   Jan-05-16 08:38:49 m1-01129-09140 [Worker_1] [TLS-in] [TLS-out] [RelayAttempt] 0.0.0.0 <user@example.com> relay attempt blocked for (parsing): <user2@example>
+#   Jun-12-16 16:43:37 m1-64217-12013 [Worker_1] [TLS-in] [TLS-out] [RelayAttempt] 0.0.0.0 <user@example.com> to: user2@example.com relay attempt blocked for (parsing): <a.notheruser69@example.c>
+#   Jan-22-16 22:25:51 [Worker_1] [TLS-out] 0.0.0.0 [SMTP Error] 535 5.7.8 Error: authentication failed: Invalid authentication mechanism
+#   Mar-19-16 13:42:20 [Worker_1] [TLS-out] 0.0.0.0 [SMTP Error] 535 5.7.8 Error: authentication failed: Invalid base64 data in continued response
+#   Jul-18-16 16:54:21 [Worker_2] [TLS-out] 0.0.0.0 [SMTP Error] 535 5.7.8 Error: authentication failed: Connection lost to authentication server
+#   Jul-18-16 17:14:23 m1-76453-02949 [Worker_1] [TLS-out] 0.0.0.0 [SMTP Error] 535 5.7.8 Error: authentication failed: Connection lost to authentication server
+
 #
 # Author: Enrico Labedzki (enrico.labedzki@deiwos.de)
+# V2 Filters: Robert Hardy (rhardy@webcon.ca)
@@ -11,22 +11,24 @@ before = common.conf

 _daemon = asterisk

-__pid_re = (?:\[\d+\])
+__pid_re = (?:\s*\[\d+\])

 iso8601 = \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+[+-]\d{4}

 # All Asterisk log messages begin like this:
-log_prefix= (?:NOTICE|SECURITY)%(__pid_re)s:?(?:\[C-[\da-f]*\])? \S+:\d*( in \w+:)?
+log_prefix= (?:NOTICE|SECURITY|WARNING)%(__pid_re)s:?(?:\[C-[\da-f]*\])? [^:]+:\d*(?:(?: in)? \w+:)?

-failregex = ^(%(__prefix_line)s|\[\]\s*)%(log_prefix)s Registration from '[^']*' failed for '<HOST>(:\d+)?' - (Wrong password|Username/auth name mismatch|No matching peer found|Not a local domain|Device does not match ACL|Peer is not supposed to register|ACL error \(permit/deny\)|Not a local domain)$
-            ^(%(__prefix_line)s|\[\]\s*)%(log_prefix)s Call from '[^']*' \(<HOST>:\d+\) to extension '[^']*' rejected because extension not found in context
-            ^(%(__prefix_line)s|\[\]\s*)%(log_prefix)s Host <HOST> failed to authenticate as '[^']*'$
-            ^(%(__prefix_line)s|\[\]\s*)%(log_prefix)s No registration for peer '[^']*' \(from <HOST>\)$
-            ^(%(__prefix_line)s|\[\]\s*)%(log_prefix)s Host <HOST> failed MD5 authentication for '[^']*' \([^)]+\)$
-            ^(%(__prefix_line)s|\[\]\s*)%(log_prefix)s Failed to authenticate (user|device) [^@]+@<HOST>\S*$
-            ^(%(__prefix_line)s|\[\]\s*)%(log_prefix)s hacking attempt detected '<HOST>'$
-            ^(%(__prefix_line)s|\[\]\s*)%(log_prefix)s SecurityEvent="(FailedACL|InvalidAccountID|ChallengeResponseFailed|InvalidPassword)",EventTV="([\d-]+|%(iso8601)s)",Severity="[\w]+",Service="[\w]+",EventVersion="\d+",AccountID="(\d*|<unknown>)",SessionID=".+",LocalAddress="IPV[46]/(UDP|TCP|WS)/[\da-fA-F:.]+/\d+",RemoteAddress="IPV[46]/(UDP|TCP|WS)/<HOST>/\d+"(,Challenge="[\w/]+")?(,ReceivedChallenge="\w+")?(,Response="\w+",ExpectedResponse="\w*")?(,ReceivedHash="[\da-f]+")?(,ACLName="\w+")?$
-            ^(%(__prefix_line)s|\[\]\s*WARNING%(__pid_re)s:?(?:\[C-[\da-f]*\])? )Ext\. s: "Rejecting unknown SIP connection from <HOST>"$
+failregex = ^%(__prefix_line)s%(log_prefix)s Registration from '[^']*' failed for '<HOST>(:\d+)?' - (?:Wrong password|Username/auth name mismatch|No matching peer found|Not a local domain|Device does not match ACL|Peer is not supposed to register|ACL error \(permit/deny\)|Not a local domain)$
+            ^%(__prefix_line)s%(log_prefix)s Call from '[^']*' \(<HOST>:\d+\) to extension '[^']*' rejected because extension not found in context
+            ^%(__prefix_line)s%(log_prefix)s (?:Host )?<HOST> (?:failed (?:to authenticate\b|MD5 authentication\b)|tried to authenticate with nonexistent user\b)
+            ^%(__prefix_line)s%(log_prefix)s No registration for peer '[^']*' \(from <HOST>\)$
+            ^%(__prefix_line)s%(log_prefix)s hacking attempt detected '<HOST>'$
+            ^%(__prefix_line)s%(log_prefix)s SecurityEvent="(?:FailedACL|InvalidAccountID|ChallengeResponseFailed|InvalidPassword)"(?:(?:,(?!RemoteAddress=)\w+="[^"]*")*|.*?),RemoteAddress="IPV[46]/(UDP|TCP|WS)/<HOST>/\d+"(?:,(?!RemoteAddress=)\w+="[^"]*")*$
+            ^%(__prefix_line)s%(log_prefix)s "Rejecting unknown SIP connection from <HOST>(?::\d+)?"$
+            ^%(__prefix_line)s%(log_prefix)s Request (?:'[^']*' )?from '(?:[^']*|.*?)' failed for '<HOST>(?::\d+)?'\s\(callid: [^\)]*\) - (?:No matching endpoint found|Not match Endpoint(?: Contact)? ACL|(?:Failed|Error) to authenticate)\s*$
+
+# FreePBX (todo: make optional in v.0.10):
+#            ^(%(__prefix_line)s|\[\]\s*WARNING%(__pid_re)s:?(?:\[C-[\da-f]*\])? )[^:]+: Friendly Scanner from <HOST>$

 ignoreregex =

@@ -26,7 +26,7 @@ __daemon_re = [\[\(]?%(_daemon)s(?:\(\S+\))?[\]\)]?:?

 # extra daemon info
 # EXAMPLE: [ID 800047 auth.info]
-__daemon_extra_re = (?:\[ID \d+ \S+\])
+__daemon_extra_re = \[ID \d+ \S+\]

 # Combinations of daemon name and PID
 # EXAMPLES: sshd[31607], pop(pam_unix)[4920]
@@ -44,14 +44,18 @@ __md5hex = (?:[\da-f]{2}:){15}[\da-f]{2}

 # bsdverbose is where syslogd is started with -v or -vv and results in <4.3> or
 # <auth.info> appearing before the host as per testcases/files/logs/bsd/*.
-__bsd_syslog_verbose = (<[^.]+\.[^.]+>)
+__bsd_syslog_verbose = <[^.]+\.[^.]+>
+
+__vserver = @vserver_\S+
+
+__date_ambit = (?:\[\])

 # Common line prefixes (beginnings) which could be used in filters
 #
 #      [bsdverbose]? [hostname] [vserver tag] daemon_id spaces
 #
 # This can be optional (for instance if we match named native log files)
-__prefix_line = \s*%(__bsd_syslog_verbose)s?\s*(?:%(__hostname)s )?(?:%(__kernel_prefix)s )?(?:@vserver_\S+ )?%(__daemon_combs_re)s?\s%(__daemon_extra_re)s?\s*
+__prefix_line = %(__date_ambit)s?\s*(?:%(__bsd_syslog_verbose)s\s+)?(?:%(__hostname)s\s+)?(?:%(__kernel_prefix)s\s+)?(?:%(__vserver)s\s+)?(?:%(__daemon_combs_re)s\s+)?(?:%(__daemon_extra_re)s\s+)?

 # PAM authentication mechanism check for failures, e.g.: pam_unix, pam_sss,
 # pam_ldap
@@ -13,7 +13,7 @@ before = common.conf
 _daemon = courieresmtpd

 failregex = ^%(__prefix_line)serror,relay=<HOST>,.*: 550 User (<.*> )?unknown\.?$
-            ^%(__prefix_line)serror,relay=<HOST>,msg="535 Authentication failed\.",cmd:( AUTH \S+)?( [0-9a-zA-Z\+/=]+)?$
+            ^%(__prefix_line)serror,relay=<HOST>,msg="535 Authentication failed\.",cmd:( AUTH \S+)?( [0-9a-zA-Z\+/=]+)?(?: \S+)$

 ignoreregex = 

@@ -13,7 +13,7 @@ before = common.conf

 _daemon = (?:cyrus/)?(?:imap(d|s)?|pop3(d|s)?)

-failregex = ^%(__prefix_line)sbadlogin: \S+ ?\[<HOST>\] \S+ .*?\[?SASL\(-13\): (authentication failure|user not found): .*\]?$
+failregex = ^%(__prefix_line)sbadlogin: [^\[]*\[<HOST>\] \S+ .*?\[?SASL\(-13\): (authentication failure|user not found): .*\]?$

 ignoreregex = 

@@ -0,0 +1,47 @@
+# Fail2Ban configuration file for IBM Domino SMTP Server TASK to detect failed login attempts
+#
+# Author: Christian Brandlehner
+#
+# $Revision: 003 $
+#
+# Configuration:
+# Set the following Domino Server parameters in notes.ini:
+#       console_log_enabled=1
+#       log_sessions=2
+# You also have to use a date and time format supported by fail2ban. Recommended notes.ini configuration is:
+#       DateOrder=DMY
+#       DateSeparator=-
+#       ClockType=24_Hour
+#       TimeSeparator=:
+#
+# Depending on your locale you might have to tweak the date and time format so fail2ban can read the log
+
+#[INCLUDES]
+# Read common prefixes. If any customizations available -- read them from
+# common.local
+#before = common.conf
+
+[Definition]
+# Option:  failregex
+# Notes.:  regex to match the password failure messages in the logfile. The
+#          host must be matched by a group named "host". The tag "<HOST>" can
+#          be used for standard IP/hostname matching and is only an alias for
+#          (?:::f{4,6}:)?(?P<host>\S+)
+# Values:  TEXT
+#
+# Sample log entries (used different time formats and an extra sample with process info in front of date)
+# 01-23-2009 19:54:51   SMTP Server: Authentication failed for user postmaster ; connecting host 1.2.3.4
+# [28325:00010-3735542592] 22-06-2014 09:56:12   smtp: postmaster [1.2.3.4] authentication failure using internet password
+# 08-09-2014 06:14:27   smtp: postmaster [1.2.3.4] authentication failure using internet password
+# 08-09-2014 06:14:27   SMTP Server: Authentication failed for user postmaster ; connecting host 1.2.3.4
+
+__prefix = (?:\[[^\]]+\])?\s+
+failregex = ^%(__prefix)sSMTP Server: Authentication failed for user .*? \; connecting host <HOST>$
+            ^%(__prefix)ssmtp: (?:[^\[]+ )*\[<HOST>\] authentication failure using internet password\s*$
+# Option:  ignoreregex
+# Notes.:  regex to ignore. If this regex matches, the line is ignored.
+# Values:  TEXT
+#
+
+ignoreregex =
+
@@ -7,13 +7,15 @@ before = common.conf

 [Definition]

-_daemon = (auth|dovecot(-auth)?|auth-worker)
+_auth_worker = (?:dovecot: )?auth(?:-worker)?
+_daemon = (?:dovecot(?:-auth)?|auth)

-failregex = ^%(__prefix_line)s(%(__pam_auth)s(\(dovecot:auth\))?:)?\s+authentication failure; logname=\S* uid=\S* euid=\S* tty=dovecot ruser=\S* rhost=<HOST>(\s+user=\S*)?\s*$
-            ^%(__prefix_line)s(pop3|imap)-login: (Info: )?(Aborted login|Disconnected)(: Inactivity)? \(((auth failed, \d+ attempts)( in \d+ secs)?|tried to use (disabled|disallowed) \S+ auth)\):( user=<\S*>,)?( method=\S+,)? rip=<HOST>(, lip=(\d{1,3}\.){3}\d{1,3})?(, TLS( handshaking(: SSL_accept\(\) failed: error:[\dA-F]+:SSL routines:[TLS\d]+_GET_CLIENT_HELLO:unknown protocol)?)?(: Disconnected)?)?(, session=<\S+>)?\s*$
-            ^%(__prefix_line)s(Info|dovecot: auth\(default\)|auth-worker\(\d+\)): pam\(\S+,<HOST>\): pam_authenticate\(\) failed: (User not known to the underlying authentication module: \d+ Time\(s\)|Authentication failure \(password mismatch\?\))\s*$
-            ^%(__prefix_line)s(auth|auth-worker\(\d+\)): (pam|passwd-file)\(\S+,<HOST>\): unknown user\s*$
-            ^%(__prefix_line)s(auth|auth-worker\(\d+\)): Info: ldap\(\S*,<HOST>,\S*\): invalid credentials\s*$
+__prefregex = ^%(__prefix_line)s(?:%(_auth_worker)s(?:\([^\)]+\))?: )?(?:%(__pam_auth)s(?:\(dovecot:auth\))?: |(?:pop3|imap)-login: )?(?:Info: )?
+
+failregex = ^%(__prefregex)sauthentication failure; logname=\S* uid=\S* euid=\S* tty=dovecot ruser=\S* rhost=<HOST>(?:\s+user=\S*)?\s*$
+            ^%(__prefregex)s(?:Aborted login|Disconnected)(?::(?: [^ \(]+)+)? \((?:auth failed, \d+ attempts(?: in \d+ secs)?|tried to use (?:disabled|disallowed) \S+ auth)\):(?: user=<[^>]*>,)?(?: method=\S+,)? rip=<HOST>(?:[^>]*(?:, session=<\S+>)?)\s*$
+            ^%(__prefregex)spam\(\S+,<HOST>(?:,\S*)?\): pam_authenticate\(\) failed: (?:User not known to the underlying authentication module: \d+ Time\(s\)|Authentication failure \(password mismatch\?\)|Permission denied)\s*$
+            ^%(__prefregex)s[a-z\-]{3,15}\(\S*,<HOST>(?:,\S*)?\): (?:unknown user|invalid credentials)\s*$

 ignoreregex = 

@@ -30,3 +32,4 @@ journalmatch = _SYSTEMD_UNIT=dovecot.service
 # Author: Martin Waschbuesch
 #         Daniel Black (rewrote with begin and end anchors)
 #         Martin O'Neal (added LDAP authentication failure regex)
+#         Sergey G. Brester aka sebres (reviewed, optimized, IPv6-compatibility)
@@ -9,8 +9,10 @@ after = exim-common.local

 [Definition]

-host_info = H=([\w.-]+ )?(\(\S+\) )?\[<HOST>\](:\d+)? (I=\[\S+\]:\d+ )?(U=\S+ )?(P=e?smtp )?
-pid = ( \[\d+\])?
+host_info_pre = (?:H=([\w.-]+ )?(?:\(\S+\) )?)?
+host_info_suf = (?::\d+)?(?: I=\[\S+\](:\d+)?)?(?: U=\S+)?(?: P=e?smtp)?(?: F=(?:<>|[^@]+@\S+))?\s
+host_info = %(host_info_pre)s\[<HOST>\]%(host_info_suf)s
+pid = (?: \[\d+\])?

 # DEV Notes:
 # From exim source code: ./src/receive.c:add_host_info_for_log
@@ -14,13 +14,13 @@ before = exim-common.conf
 [Definition]

 failregex = ^%(pid)s %(host_info)ssender verify fail for <\S+>: (?:Unknown user|Unrouteable address|all relevant MX records point to non-existent hosts)\s*$
-            ^%(pid)s \w+ authenticator failed for (\S+ )?\(\S+\) \[<HOST>\](:\d+)?( I=\[\S+\](:\d+)?)?: 535 Incorrect authentication data( \(set_id=.*\)|: \d+ Time\(s\))?\s*$
-            ^%(pid)s %(host_info)sF=(<>|[^@]+@\S+) rejected RCPT [^@]+@\S+: (relay not permitted|Sender verify failed|Unknown user)\s*$
-            ^%(pid)s SMTP protocol synchronization error \([^)]*\): rejected (connection from|"\S+") %(host_info)s(next )?input=".*"\s*$
-            ^%(pid)s SMTP call from \S+ \[<HOST>\](:\d+)? (I=\[\S+\](:\d+)? )?dropped: too many nonmail commands \(last was "\S+"\)\s*$
-            ^%(pid)s SMTP protocol error in "AUTH \S*(| \S*)" H=(|\S* )(|\(\S*\) )\[<HOST>\]\:\d+ I=\[\S*\]\:\d+ AUTH command used when not advertised\s*$
-            ^%(pid)s no MAIL in SMTP connection from (|\S* )(|\(\S*\) )\[<HOST>\]\:\d+ I=\[\S*\]\:\d+ D=\d+s(| C=\S*)\s*$
-            ^%(pid)s \S+ SMTP connection from (|\S* )(|\(\S*\) )\[<HOST>\]\:\d+ I=\[\S*\]\:\d+ closed by DROP in ACL\s*$
+            ^%(pid)s \w+ authenticator failed for (?:[^\[\( ]* )?(?:\(\S*\) )?\[<HOST>\](?::\d+)?(?: I=\[\S+\](:\d+)?)?: 535 Incorrect authentication data( \(set_id=.*\)|: \d+ Time\(s\))?\s*$
+            ^%(pid)s %(host_info)srejected RCPT [^@]+@\S+: (?:relay not permitted|Sender verify failed|Unknown user|Unrouteable address)\s*$
+            ^%(pid)s SMTP protocol synchronization error \([^)]*\): rejected (?:connection from|"\S+") %(host_info)s(?:next )?input=".*"\s*$
+            ^%(pid)s SMTP call from \S+ %(host_info)sdropped: too many nonmail commands \(last was "\S+"\)\s*$
+            ^%(pid)s SMTP protocol error in "[^"]+(?:"+[^"]*(?="))*?" %(host_info)sAUTH command used when not advertised\s*$
+            ^%(pid)s no MAIL in SMTP connection from (?:[^\[\( ]* )?(?:\(\S*\) )?%(host_info)sD=\d\S*s(?: C=\S*)?\s*$
+            ^%(pid)s (?:[\w\-]+ )?SMTP connection from (?:[^\[\( ]* )?(?:\(\S*\) )?%(host_info)sclosed by DROP in ACL\s*$

 ignoreregex = 

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env fail2ban-python
 # Inspired by https://isc.sans.edu/forums/diary/When+Google+isnt+Google/15968/
 #
 # Written in Python to reuse built-in Python batteries and not depend on
@@ -3,9 +3,14 @@
 [Definition]

 failregex = ^ SMTP Spam attack detected from <HOST>,
-            ^ IP address <HOST> found in DNS blacklist \S+, mail from \S+ to \S+$
+            ^ IP address <HOST> found in DNS blacklist
            ^ Relay attempt from IP address <HOST>
            ^ Attempt to deliver to unknown recipient \S+, from \S+, IP address <HOST>$
+            ^ Failed SMTP login from <HOST>
+            ^ SMTP: User \S+ doesn't exist. Attempt from IP address <HOST>
+            ^ Client with IP address <HOST> has no reverse DNS entry, connection rejected before SMTP greeting$
+            ^ Administration login into Web Administration from <HOST> failed: IP address not allowed$
+            ^ Message from IP address <HOST>, sender \S+ rejected: sender domain does not exist$

 ignoreregex =

@@ -16,5 +21,6 @@ datepattern = ^\[%%d/%%b/%%Y %%H:%%M:%%S\]
 # DEV NOTES:
 # 
 # Author: A.P. Lawrence
+# Updated by: M. Bischoff <https://github.com/herrbischoff>
 #
 # Based off: http://aplawrence.com/Kerio/fail2ban.html
@@ -3,7 +3,7 @@

 [Definition]

-failregex = ^: \(http_auth\.c\.\d+\) (password doesn\'t match .* username: .*|digest: auth failed for .*: wrong password|get_password failed), IP: <HOST>\s*$
+failregex = ^: \((?:http|mod)_auth\.c\.\d+\) (?:password doesn\'t match .* username: .*|digest: auth failed for .*: wrong password|get_password failed), IP: <HOST>\s*$

 ignoreregex = 

@@ -0,0 +1,49 @@
+# Fail2Ban filter for unsuccesfull MongoDB authentication attempts
+#
+# Logfile /var/log/mongodb/mongodb.log
+#
+# add setting in /etc/mongodb.conf
+# logpath=/var/log/mongodb/mongodb.log
+#
+# and use of the authentication
+# auth = true
+#
+
+[Definition]
+#failregex = ^\s+\[initandlisten\] connection accepted from <HOST>:\d+ \#(?P<__connid>\d+) \(1 connection now open\)<SKIPLINES>\s+\[conn(?P=__connid)\] Failed to authenticate\s+
+failregex = ^\s+\[conn(?P<__connid>\d+)\] Failed to authenticate [^\n]+<SKIPLINES>\s+\[conn(?P=__connid)\] end connection <HOST>
+
+ignoreregex =
+
+
+[Init]
+maxlines = 10
+
+# DEV Notes:
+#
+# Regarding the multiline regex:
+#
+# There can be a nunber of non-related lines between the first and second part
+# of this regex maxlines of 10 is quite generious.
+#
+# Note the capture __connid, includes the connection ID, used in second part of regex.
+#
+# The first regex is commented out (but will match also), because it is better to use
+# the host from "end connection" line (uncommented above):
+#  -  it has the same prefix, searching begins directly with failure message
+#     (so faster, because ignores success connections at all)
+#  -  it is not so vulnerable in case of possible race condition
+#
+# Log example:
+# 2016-10-20T09:54:27.108+0200 [initandlisten] connection accepted from 127.0.0.1:53276 #1 (1 connection now open)
+# 2016-10-20T09:54:27.109+0200 [conn1]  authenticate db: test { authenticate: 1, nonce: "xxx", user: "root", key: "xxx" }
+# 2016-10-20T09:54:27.110+0200 [conn1] Failed to authenticate root@test with mechanism MONGODB-CR: AuthenticationFailed UserNotFound Could not find user root@test
+# 2016-11-09T09:54:27.894+0100 [conn1] end connection 127.0.0.1:53276 (0 connections now open)
+# 2016-11-09T11:55:58.890+0100 [initandlisten] connection accepted from 127.0.0.1:54266 #1510 (1 connection now open)
+# 2016-11-09T11:55:58.892+0100 [conn1510]  authenticate db: admin { authenticate: 1, nonce: "xxx", user: "root", key: "xxx" }
+# 2016-11-09T11:55:58.892+0100 [conn1510] Failed to authenticate root@admin with mechanism MONGODB-CR: AuthenticationFailed key mismatch
+# 2016-11-09T11:55:58.894+0100 [conn1510] end connection 127.0.0.1:54266 (0 connections now open)
+#
+# Authors: Alexander Finkhäuser
+#          Sergey G. Brester (sebres)
+
@@ -1,4 +1,4 @@
-# Fail2Ban filter for unsuccesfull MySQL authentication attempts
+# Fail2Ban filter for unsuccesful MySQL authentication attempts
 #
 #
 # To log wrong MySQL access attempts add to /etc/my.cnf in [mysqld]:
@@ -4,7 +4,7 @@
 [Definition]


-failregex = ^ \[error\] \d+#\d+: \*\d+ user "\S+":? (password mismatch|was not found in ".*"), client: <HOST>, server: \S*, request: "\S+ \S+ HTTP/\d+\.\d+", host: "\S+"(, referrer: "\S+")?\s*$
+failregex = ^ \[error\] \d+#\d+: \*\d+ user "(?:[^"]+|.*?)":? (?:password mismatch|was not found in "[^\"]*"), client: <HOST>, server: \S*, request: "\S+ \S+ HTTP/\d+\.\d+", host: "\S+"(?:, referrer: "\S+")?\s*$

 ignoreregex = 

@@ -36,10 +36,10 @@ ngx_limit_req_zones = [^"]+
 # Use following full expression if you should range limit request to specified 
 # servers, requests, referrers etc. only :
 #
-# failregex = ^\s*\[error\] \d+#\d+: \*\d+ limiting requests, excess: [\d\.]+ by zone "(?:%(ngx_limit_req_zones)s)", client: <HOST>, server: \S*, request: "\S+ \S+ HTTP/\d+\.\d+", host: "\S+"(, referrer: "\S+")?\s*$
+# failregex = ^\s*\[[a-z]+\] \d+#\d+: \*\d+ limiting requests, excess: [\d\.]+ by zone "(?:%(ngx_limit_req_zones)s)", client: <HOST>, server: \S*, request: "\S+ \S+ HTTP/\d+\.\d+", host: "\S+"(, referrer: "\S+")?\s*$

 # Shortly, much faster and stable version of regexp:
-failregex = ^\s*\[error\] \d+#\d+: \*\d+ limiting requests, excess: [\d\.]+ by zone "(?:%(ngx_limit_req_zones)s)", client: <HOST>
+failregex = ^\s*\[[a-z]+\] \d+#\d+: \*\d+ limiting requests, excess: [\d\.]+ by zone "(?:%(ngx_limit_req_zones)s)", client: <HOST>,

 ignoreregex = 

@@ -22,7 +22,7 @@ _daemon = nsd
 #          (?:::f{4,6}:)?(?P<host>[\w\-.^_]+)
 # Values:  TEXT

-failregex =  ^\[\]%(__prefix_line)sinfo: ratelimit block .* query <HOST> TYPE255$
-              ^\[\]%(__prefix_line)sinfo: .* <HOST> refused, no acl matches\.$
+failregex =  ^%(__prefix_line)sinfo: ratelimit block .* query <HOST> TYPE255$
+             ^%(__prefix_line)sinfo: .* <HOST> refused, no acl matches\.$

 ignoreregex =
@@ -0,0 +1,18 @@
+# Fail2Ban fitler for the phpMyAdmin-syslog
+#
+
+[INCLUDES]
+
+before = common.conf
+
+[Definition]
+
+_daemon = phpMyAdmin
+
+failregex = ^%(__prefix_line)suser denied: (?:\S+|.*?) \(mysql-denied\) from <HOST>\s*$
+
+ignoreregex =
+
+
+# Author: Pavel Mihadyuk
+# Regex fixes: Serg G. Brester
@@ -11,8 +11,9 @@ before = common.conf
 [Definition]

 _daemon = postfix(-\w+)?/smtpd
+_port = (?::\d+)?

-failregex = ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[<HOST>\]: 454 4\.7\.1 Service unavailable; Client host \[\S+\] blocked using .* from=<\S*> to=<\S+> proto=ESMTP helo=<\S*>$
+failregex = ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[<HOST>\]%(_port)s: [45]54 [45]\.7\.1 Service unavailable; Client host \[\S+\] blocked\b

 ignoreregex =

@@ -8,8 +8,9 @@ before = common.conf
 [Definition]

 _daemon = postfix(-\w+)?/(?:submission/|smtps/)?smtp[ds]
+_port = (?::\d+)?

-failregex = ^%(__prefix_line)swarning: [-._\w]+\[<HOST>\]: SASL ((?i)LOGIN|PLAIN|(?:CRAM|DIGEST)-MD5) authentication failed(: [ A-Za-z0-9+/:]*={0,2})?\s*$
+failregex = ^%(__prefix_line)swarning: [-._\w]+\[<HOST>\]%(_port)s: SASL ((?i)LOGIN|PLAIN|(?:CRAM|DIGEST)-MD5) authentication failed(:[ A-Za-z0-9+/:]*={0,2})?\s*$

 ignoreregex = authentication failed: Connection lost to authentication server$

@@ -11,14 +11,15 @@ before = common.conf
 [Definition]

 _daemon = postfix(-\w+)?/(?:submission/|smtps/)?smtp[ds]
+_port = (?::\d+)?

-failregex = ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[<HOST>\]: 554 5\.7\.1 .*$
-            ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[<HOST>\]: 450 4\.7\.1 Client host rejected: cannot find your hostname, (\[\S*\]); from=<\S*> to=<\S+> proto=ESMTP helo=<\S*>$
-            ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[<HOST>\]: 450 4\.7\.1 : Helo command rejected: Host not found; from=<> to=<> proto=ESMTP helo= *$
-            ^%(__prefix_line)sNOQUEUE: reject: EHLO from \S+\[<HOST>\]: 504 5\.5\.2 <\S+>: Helo command rejected: need fully-qualified hostname;
-            ^%(__prefix_line)sNOQUEUE: reject: VRFY from \S+\[<HOST>\]: 550 5\.1\.1 .*$
-            ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[<HOST>\]: 450 4\.1\.8 <\S*>: Sender address rejected: Domain not found; from=<\S*> to=<\S+> proto=ESMTP helo=<\S*>$
-            ^%(__prefix_line)simproper command pipelining after \S+ from [^[]*\[<HOST>\]:?$
+failregex = ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[<HOST>\]%(_port)s: 554 5\.7\.1 .*$
+            ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[<HOST>\]%(_port)s: 450 4\.7\.1 Client host rejected: cannot find your hostname, (\[\S*\]); from=<\S*> to=<\S+> proto=ESMTP helo=<\S*>$
+            ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[<HOST>\]%(_port)s: 450 4\.7\.1 : Helo command rejected: Host not found; from=<> to=<> proto=ESMTP helo= *$
+            ^%(__prefix_line)sNOQUEUE: reject: EHLO from \S+\[<HOST>\]%(_port)s: 504 5\.5\.2 <\S+>: Helo command rejected: need fully-qualified hostname;
+            ^%(__prefix_line)sNOQUEUE: reject: VRFY from \S+\[<HOST>\]%(_port)s: 550 5\.1\.1 .*$
+            ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[<HOST>\]%(_port)s: 450 4\.1\.8 <\S*>: Sender address rejected: Domain not found; from=<\S*> to=<\S+> proto=ESMTP helo=<\S*>$
+            ^%(__prefix_line)simproper command pipelining after \S+ from [^[]*\[<HOST>\]%(_port)s:?$

 ignoreregex = 

@@ -18,10 +18,13 @@ __suffix_failed_login = (User not authorized for login|No such user found|Incorr

 failregex = ^%(__prefix_line)s%(__hostname)s \(\S+\[<HOST>\]\)[: -]+ USER .*: no such user found from \S+ \[\S+\] to \S+:\S+ *$
            ^%(__prefix_line)s%(__hostname)s \(\S+\[<HOST>\]\)[: -]+ USER .* \(Login failed\): %(__suffix_failed_login)s\s*$
-            ^%(__prefix_line)s%(__hostname)s \(\S+\[<HOST>\]\)[: -]+ SECURITY VIOLATION: .* login attempted\. *$
+            ^%(__prefix_line)s%(__hostname)s \(\S+\[<HOST>\]\)[: -]+ SECURITY VIOLATION: .* login attempted\.? *$
            ^%(__prefix_line)s%(__hostname)s \(\S+\[<HOST>\]\)[: -]+ Maximum login attempts \(\d+\) exceeded *$

 ignoreregex = 

+[Init]
+journalmatch = _SYSTEMD_UNIT=proftpd.service
+
 # Author: Yaroslav Halchenko
 #         Daniel Black - hardening of regex
@@ -7,12 +7,16 @@ before = common.conf

 [Definition]

-_daemon = (?:sm-(mta|acceptingconnections))
+_daemon = (?:sendmail|sm-(?:mta|acceptingconnections))

 failregex = ^%(__prefix_line)s\w{14}: (\S+ )?\[<HOST>\]( \(may be forged\))?: possible SMTP attack: command=AUTH, count=\d+$

 ignoreregex =

+[Init]
+
+journalmatch = _SYSTEMD_UNIT=sendmail.service
+
 # DEV Notes:
 #
 # Author: Daniel Black
@@ -23,9 +23,9 @@ _daemon = (?:(sm-(mta|acceptingconnections)|sendmail))

 failregex = ^%(__prefix_line)s\w{14}: ruleset=check_rcpt, arg1=(?P<email><\S+@\S+>), relay=(\S+ )?\[<HOST>\]( \(may be forged\))?, reject=(550 5\.7\.1 (?P=email)\.\.\. Relaying denied\. (IP name possibly forged \[(\d+\.){3}\d+\]|Proper authentication required\.|IP name lookup failed \[(\d+\.){3}\d+\])|553 5\.1\.8 (?P=email)\.\.\. Domain of sender address \S+ does not exist|550 5\.[71]\.1 (?P=email)\.\.\. (Rejected: .*|User unknown))$
            ^%(__prefix_line)sruleset=check_relay, arg1=(?P<dom>\S+), arg2=<HOST>, relay=((?P=dom) )?\[(\d+\.){3}\d+\]( \(may be forged\))?, reject=421 4\.3\.2 (Connection rate limit exceeded\.|Too many open connections\.)$
-            ^%(__prefix_line)s\w{14}: rejecting commands from  (\S+ )?\[<HOST>\] due to pre-greeting traffic after \d+ seconds$
+            ^%(__prefix_line)s\w{14}: rejecting commands from (\S* )?\[<HOST>\] due to pre-greeting traffic after \d+ seconds$
            ^%(__prefix_line)s\w{14}: (\S+ )?\[<HOST>\]: ((?i)expn|vrfy) \S+ \[rejected\]$
-            ^(?P<__prefix>%(__prefix_line)s\w+: )<[^@]+@[^>]+>\.\.\. No such user here<SKIPLINES>(?P=__prefix)from=<[^@]+@[^>]+>, size=\d+, class=\d+, nrcpts=\d+, bodytype=\w+, proto=E?SMTP, daemon=MTA, relay=\S+ \[<HOST>\]$
+            ^(?P<__prefix>%(__prefix_line)s\w+: )<[^@]+@[^>]+>\.\.\. No such user here$<SKIPLINES>^(?P=__prefix)from=<[^@]+@[^>]+>, size=\d+, class=\d+, nrcpts=\d+, bodytype=\w+, proto=E?SMTP, daemon=MTA, relay=\S+ \[<HOST>\]$


 ignoreregex =
@@ -33,6 +33,8 @@ ignoreregex =

 [Init]

+journalmatch = _SYSTEMD_UNIT=sendmail.service
+
 # "maxlines" is number of log lines to buffer for multi-line regex searches
 maxlines = 10

@@ -0,0 +1,25 @@
+# slapd (Stand-alone LDAP Daemon) openldap daemon filter
+#
+# Detecting invalid credentials: error code 49
+# http://www.openldap.org/doc/admin24/appendix-ldap-result-codes.html#invalidCredentials (49)
+
+[INCLUDES]
+
+# Read common prefixes. If any customizations available -- read them from
+# common.local
+before = common.conf
+
+[Definition]
+
+_daemon = slapd
+
+failregex = ^(?P<__prefix>%(__prefix_line)s)conn=(?P<_conn_>\d+) fd=\d+ ACCEPT from IP=<HOST>:\d{1,5} \(IP=\S+\)\s*<SKIPLINES>(?P=__prefix)conn=(?P=_conn_) op=\d+ RESULT(?:\s(?!err)\S+=\S*)* err=49 text=[\w\s]*$
+
+ignoreregex =
+
+[Init]
+
+# "maxlines" is number of log lines to buffer for multi-line regex searches
+maxlines = 20
+
+# Author: Andrii Melnyk
@@ -4,7 +4,7 @@

 [Definition]

-failregex = ^ sogod \[\d+\]: SOGoRootPage Login from '<HOST>' for user '.*' might not have worked( - password policy: \d*  grace: -?\d*  expire: -?\d*  bound: -?\d*)?\s*$
+failregex = ^ sogod \[\d+\]: SOGoRootPage Login from '<HOST>(?:,[^']*)?' for user '[^']*' might not have worked( - password policy: \d*  grace: -?\d*  expire: -?\d*  bound: -?\d*)?\s*$

 ignoreregex = 

@@ -0,0 +1,11 @@
+# Fail2Ban aggressive ssh filter for at attempted exploit
+#
+# Includes failregex of both sshd and sshd-ddos filters
+#
+[INCLUDES]
+
+before = sshd.conf
+
+[Definition]
+
+mode = %(aggressive)s
@@ -10,20 +10,8 @@

 [INCLUDES]

-# Read common prefixes. If any customizations available -- read them from
-# common.local
-before = common.conf
+before = sshd.conf

 [Definition]

-_daemon = sshd
-
-failregex = ^%(__prefix_line)sDid not receive identification string from <HOST>\s*$
-
-ignoreregex = 
-
-[Init]
-
-journalmatch = _SYSTEMD_UNIT=sshd.service + _COMM=sshd
-
-# Author: Yaroslav Halchenko
+mode = %(ddos)s
@@ -14,27 +14,54 @@
 # common.local
 before = common.conf

-[Definition]
+[DEFAULT]

 _daemon = sshd

-failregex = ^%(__prefix_line)s(?:error: PAM: )?[aA]uthentication (?:failure|error|failed) for .* from <HOST>( via \S+)?\s*$
-            ^%(__prefix_line)s(?:error: PAM: )?User not known to the underlying authentication module for .* from <HOST>\s*$
-            ^%(__prefix_line)sFailed \S+ for .*? from <HOST>(?: port \d*)?(?: ssh\d*)?(: (ruser .*|(\S+ ID \S+ \(serial \d+\) CA )?\S+ %(__md5hex)s(, client user ".*", client host ".*")?))?\s*$
-            ^%(__prefix_line)sROOT LOGIN REFUSED.* FROM <HOST>\s*$
-            ^%(__prefix_line)s[iI](?:llegal|nvalid) user .* from <HOST>\s*$
-            ^%(__prefix_line)sUser .+ from <HOST> not allowed because not listed in AllowUsers\s*$
-            ^%(__prefix_line)sUser .+ from <HOST> not allowed because listed in DenyUsers\s*$
-            ^%(__prefix_line)sUser .+ from <HOST> not allowed because not in any group\s*$
-            ^%(__prefix_line)srefused connect from \S+ \(<HOST>\)\s*$
-            ^%(__prefix_line)s(?:error: )?Received disconnect from <HOST>: 3: .*: Auth fail(?: \[preauth\])?$
-            ^%(__prefix_line)sUser .+ from <HOST> not allowed because a group is listed in DenyGroups\s*$
-            ^%(__prefix_line)sUser .+ from <HOST> not allowed because none of user's groups are listed in AllowGroups\s*$
-            ^(?P<__prefix>%(__prefix_line)s)User .+ not allowed because account is locked<SKIPLINES>(?P=__prefix)(?:error: )?Received disconnect from <HOST>: 11: .+ \[preauth\]$
-            ^(?P<__prefix>%(__prefix_line)s)Disconnecting: Too many authentication failures for .+? \[preauth\]<SKIPLINES>(?P=__prefix)(?:error: )?Connection closed by <HOST> \[preauth\]$
-            ^(?P<__prefix>%(__prefix_line)s)Connection from <HOST> port \d+(?: on \S+ port \d+)?<SKIPLINES>(?P=__prefix)Disconnecting: Too many authentication failures for .+? \[preauth\]$
-            ^%(__prefix_line)s(error: )?maximum authentication attempts exceeded for .* from <HOST>(?: port \d*)?(?: ssh\d*)? \[preauth\]$
-            ^%(__prefix_line)spam_unix\(sshd:auth\):\s+authentication failure;\s*logname=\S*\s*uid=\d*\s*euid=\d*\s*tty=\S*\s*ruser=\S*\s*rhost=<HOST>\s.*$
+# optional prefix (logged from several ssh versions) like "error: ", "error: PAM: " or "fatal: "
+__pref = (?:(?:error|fatal): (?:PAM: )?)?
+# optional suffix (logged from several ssh versions) like " [preauth]"
+__suff = (?: \[preauth\])?\s*
+__on_port_opt = (?: port \d+)?(?: on \S+(?: port \d+)?)?
+
+# single line prefix:
+__prefix_line_sl = %(__prefix_line)s%(__pref)s
+# multi line prefixes (for first and second lines):
+__prefix_line_ml1 = (?P<__prefix>%(__prefix_line)s)%(__pref)s
+__prefix_line_ml2 = %(__suff)s$<SKIPLINES>^(?P=__prefix)%(__pref)s
+
+mode = %(normal)s
+
+normal = ^%(__prefix_line_sl)s[aA]uthentication (?:failure|error|failed) for .* from <HOST>( via \S+)?\s*%(__suff)s$
+         ^%(__prefix_line_sl)sUser not known to the underlying authentication module for .* from <HOST>\s*%(__suff)s$
+         ^%(__prefix_line_sl)sFailed \S+ for (?P<cond_inv>invalid user )?(?P<user>(?P<cond_user>\S+)|(?(cond_inv)(?:(?! from ).)*?|[^:]+)) from <HOST>%(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$)
+         ^%(__prefix_line_sl)sROOT LOGIN REFUSED.* FROM <HOST>\s*%(__suff)s$
+         ^%(__prefix_line_sl)s[iI](?:llegal|nvalid) user .*? from <HOST>%(__on_port_opt)s\s*$
+         ^%(__prefix_line_sl)sUser .+ from <HOST> not allowed because not listed in AllowUsers\s*%(__suff)s$
+         ^%(__prefix_line_sl)sUser .+ from <HOST> not allowed because listed in DenyUsers\s*%(__suff)s$
+         ^%(__prefix_line_sl)sUser .+ from <HOST> not allowed because not in any group\s*%(__suff)s$
+         ^%(__prefix_line_sl)srefused connect from \S+ \(<HOST>\)\s*%(__suff)s$
+         ^%(__prefix_line_sl)sReceived disconnect from <HOST>%(__on_port_opt)s:\s*3: .*: Auth fail%(__suff)s$
+         ^%(__prefix_line_sl)sUser .+ from <HOST> not allowed because a group is listed in DenyGroups\s*%(__suff)s$
+         ^%(__prefix_line_sl)sUser .+ from <HOST> not allowed because none of user's groups are listed in AllowGroups\s*%(__suff)s$
+         ^%(__prefix_line_sl)spam_unix\(sshd:auth\):\s+authentication failure;\s*logname=\S*\s*uid=\d*\s*euid=\d*\s*tty=\S*\s*ruser=\S*\s*rhost=<HOST>\s.*%(__suff)s$
+         ^%(__prefix_line_sl)s(error: )?maximum authentication attempts exceeded for .* from <HOST>%(__on_port_opt)s(?: ssh\d*)? \[preauth\]$
+         ^%(__prefix_line_ml1)sUser .+ not allowed because account is locked%(__prefix_line_ml2)sReceived disconnect from <HOST>: 11: .+%(__suff)s$
+         ^%(__prefix_line_ml1)sDisconnecting: Too many authentication failures for .+?%(__prefix_line_ml2)sConnection closed by <HOST>%(__suff)s$
+         ^%(__prefix_line_ml1)sConnection from <HOST>%(__on_port_opt)s%(__prefix_line_ml2)sDisconnecting: Too many authentication failures for .+%(__suff)s$
+
+ddos =   ^%(__prefix_line_sl)sDid not receive identification string from <HOST>%(__suff)s$
+         ^%(__prefix_line_sl)sReceived disconnect from <HOST>%(__on_port_opt)s:\s*14: No supported authentication methods available%(__suff)s$
+         ^%(__prefix_line_sl)sUnable to negotiate with <HOST>%(__on_port_opt)s: no matching (?:cipher|key exchange method) found.
+         ^%(__prefix_line_ml1)sConnection from <HOST>%(__on_port_opt)s%(__prefix_line_ml2)sUnable to negotiate a (?:cipher|key exchange method)%(__suff)s$
+         ^%(__prefix_line_ml1)sSSH: Server;Ltype: (?:Authname|Version|Kex);Remote: <HOST>-\d+;[A-Z]\w+:.*%(__prefix_line_ml2)sRead from socket failed: Connection reset by peer%(__suff)s$
+
+aggressive = %(normal)s
+             %(ddos)s
+
+[Definition]
+
+failregex = %(mode)s

 ignoreregex = 

@@ -17,7 +17,7 @@ _daemon = (?:lighttpd|suhosin)

 _lighttpd_prefix = (?:\(mod_fastcgi\.c\.\d+\) FastCGI-stderr:\s)

-failregex = ^%(__prefix_line)s%(_lighttpd_prefix)s?ALERT - .* \(attacker '<HOST>', file '.*'(?:, line \d+)?\)$
+failregex = ^%(__prefix_line)s%(_lighttpd_prefix)s?ALERT - .*? \(attacker '<HOST>', file '[^']*'(?:, line \d+)?\)$

 ignoreregex = 

@@ -14,7 +14,7 @@ __pam_re=\(?%(__pam_auth)s(?:\(\S+\))?\)?:?
 _daemon =  vsftpd

 failregex = ^%(__prefix_line)s%(__pam_re)s\s+authentication failure; logname=\S* uid=\S* euid=\S* tty=(ftp)? ruser=\S* rhost=<HOST>(?:\s+user=.*)?\s*$
-            ^ \[pid \d+\] \[.+\] FAIL LOGIN: Client "<HOST>"\s*$
+            ^ \[pid \d+\] \[[^\]]+\] FAIL LOGIN: Client "<HOST>"(?:\s*$|,)

 ignoreregex = 

@@ -0,0 +1,21 @@
+# Fail2Ban filter for Zoneminder login failures
+
+[INCLUDES]
+before = apache-common.conf
+
+[Definition]
+
+# pattern: [Wed Apr 27 23:12:07.736196 2016] [:error] [pid 2460] [client 10.1.1.1:47296] WAR [Login denied for user "test"], referer: https://zoneminderurl/index.php
+#
+#
+# Option:  failregex
+# Notes.:  regex to match the password failure messages in the logfile.
+
+failregex = ^%(_apache_error_client)s WAR \[Login denied for user "[^"]*"\]
+
+ignoreregex =
+
+# Notes:
+#	Tested on Zoneminder 1.29.0
+#
+# Author: John Marzella
@@ -94,6 +94,7 @@ backend = auto
 #        but it will be logged as a warning.
 # no:    if a hostname is encountered, will not be used for banning,
 #        but it will be logged as info.
+# raw:   use raw value (no hostname), allow use it for no-host filters/actions (example user)
 usedns = warn

 # "logencoding" specifies the encoding of the log files handled by the jail
@@ -185,10 +186,11 @@ action_cf_mwl = cloudflare[cfuser="%(cfemail)s", cftoken="%(cfapikey)s"]

 # Report block via blocklist.de fail2ban reporting service API
 # 
-# See the IMPORTANT note in action.d/blocklist_de.conf for when to
-# use this action. Create a file jail.d/blocklist_de.local containing
-# [Init]
-# blocklist_de_apikey = {api key from registration]
+# See the IMPORTANT note in action.d/blocklist_de.conf for when to use this action.
+# Specify expected parameters in file action.d/blocklist_de.local or if the interpolation
+# `action_blocklist_de` used for the action, set value of `blocklist_de_apikey`
+# in your `jail.local` globally (section [DEFAULT]) or per specific jail section (resp. in 
+# corresponding jail.d/my-jail.local file).
 #
 action_blocklist_de  = blocklist_de[email="%(sender)s", service=%(filter)s, apikey="%(blocklist_de_apikey)s", agent="%(fail2ban_agent)s"]

@@ -222,6 +224,8 @@ action = %(action_)s

 [sshd]

+# To use more aggressive sshd filter (inclusive sshd-ddos failregex):
+#filter = sshd-aggressive
 port    = ssh
 logpath = %(sshd_log)s
 backend = %(sshd_backend)s
@@ -730,6 +734,13 @@ logpath  = %(mysql_log)s
 backend  = %(mysql_backend)s


+# Log wrong MongoDB auth (for details see filter 'filter.d/mongodb-auth.conf')
+[mongodb-auth]
+# change port when running with "--shardsvr" or "--configsvr" runtime operation
+port     = 27017
+logpath  = /var/log/mongodb/mongodb.log
+
+
 # Jail for more extended banning of persistent abusers
 # !!! WARNINGS !!!
 # 1. Make sure that your loglevel specified in fail2ban.conf/.local
@@ -778,7 +789,6 @@ logpath = /var/log/ejabberd/ejabberd.log
 [counter-strike]

 logpath = /opt/cstrike/logs/L[0-9]*.log
-# Firewall: http://www.cstrike-planet.com/faq/6
 tcpport = 27030,27031,27032,27033,27034,27035,27036,27037,27038,27039
 udpport = 1200,27000,27001,27002,27003,27004,27005,27006,27007,27008,27009,27010,27011,27012,27013,27014,27015
 action  = %(banaction)s[name=%(__name__)s-tcp, port="%(tcpport)s", protocol="tcp", chain="%(chain)s", actname=%(banaction)s-tcp]
@@ -809,8 +819,9 @@ maxretry = 1
 [pass2allow-ftp]
 # this pass2allow example allows FTP traffic after successful HTTP authentication
 port         = ftp,ftp-data,ftps,ftps-data
-# knocking_url variable must be overridden to some secret value in filter.d/apache-pass.local
-filter       = apache-pass
+# knocking_url variable must be overridden to some secret value in jail.local
+knocking_url = /knocking/
+filter       = apache-pass[knocking_url="%(knocking_url)s"]
 # access log of the website with HTTP auth
 logpath      = %(apache_access_log)s
 blocktype    = RETURN
@@ -839,3 +850,27 @@ logencoding = utf-8
 # See "haproxy-http-auth" filter for a brief cautionary note when setting
 # maxretry and findtime.
 logpath  = /var/log/haproxy.log
+
+[slapd]
+port    = ldap,ldaps
+filter  = slapd
+logpath = /var/log/slapd.log
+
+[domino-smtp]
+port    = smtp,ssmtp
+filter  = domino-smtp
+logpath = /home/domino01/data/IBM_TECHNICAL_SUPPORT/console.log
+
+[phpmyadmin-syslog]
+port    = http,https
+filter  = phpmyadmin-syslog
+logpath = %(syslog_authpriv)s
+backend = %(syslog_backend)s
+
+
+[zoneminder]
+# Zoneminder HTTP/HTTPS web interface auth
+# Logs auth failures to apache2 error log
+port    = http,https
+logpath = %(apache_error_log)s
+
@@ -40,7 +40,8 @@ lighttpd_error_log = /var/log/lighttpd/error.log
 # http://www.hardened-php.net/suhosin/configuration.html#suhosin.log.syslog.facility
 # syslog_user is the default. Lighttpd also hooks errors into its log.

-suhosin_log = %(syslog_user)s %(lighttpd_error_log)s
+suhosin_log = %(syslog_user)s
+              %(lighttpd_error_log)s

 # defaults to ftp or local2 if ftp doesn't exist
 proftpd_log = %(syslog_ftp)s
@@ -32,6 +32,8 @@ apache_access_log = /var/log/apache2/*access.log

 exim_main_log = /var/log/exim4/mainlog

+dovecot_log = %(syslog_mail)s
+
 # was in debian squeezy but not in wheezy
 # /etc/proftpd/proftpd.conf (SystemLog)
 proftpd_log = /var/log/proftpd/proftpd.log
@@ -34,13 +34,13 @@ auditd_log = /dev/null
 # http://svnweb.freebsd.org/ports/head/www/apache24/files/patch-config.layout
 # http://svnweb.freebsd.org/ports/head/www/apache22/files/patch-config.layout

-apache_error_log = /usr/local/www/logs/*error[_.]log
+apache_error_log = /var/log/httpd-error.log

-apache_access_log = /usr/local/www/logs/*access[_.]log
+apache_access_log = /var/log/httpd-access.log

 # http://svnweb.freebsd.org/ports/head/www/nginx/Makefile?view=markup

-nginx_error_log = /var/log/nginx-error.log
+nginx_error_log = /var/log/nginx/error.log

-nginx_access_log = /var/log/nginx-access.log
+nginx_access_log = /var/log/nginx/access.log

@@ -36,3 +36,15 @@ mysql_log = /var/log/mysql/mysqld.log
 roundcube_errors_log = /srv/www/roundcubemail/logs/errors

 solidpop3d_log = %(syslog_mail)s
+
+# These services will log to the journal via syslog, so use the journal by
+# default.
+syslog_backend = systemd
+sshd_backend = systemd
+dropbear_backend = systemd
+proftpd_backend = systemd
+pureftpd_backend = systemd
+wuftpd_backend = systemd
+postfix_backend = systemd
+dovecot_backend = systemd
+mysql_backend = systemd
@@ -34,7 +34,9 @@ Below derived from:
 	https://mail.python.org/pipermail/tutor/2007-August/056243.html
 """

+logging.MSG = logging.INFO - 2
 logging.NOTICE = logging.INFO + 5
+logging.addLevelName(logging.MSG, 'MSG')
 logging.addLevelName(logging.NOTICE, 'NOTICE')


@@ -168,7 +168,7 @@ after = 1.conf
 			parser, i = self._getSharedSCPWI(resource)
 			if not i:
 				return []
-		except UnicodeDecodeError, e:
+		except UnicodeDecodeError as e:
 			logSys.error("Error decoding config file '%s': %s" % (resource, e))
 			return []
 		
@@ -221,7 +221,7 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes):
 				if not pOptions is None and option[1] in pOptions:
 					continue
 				values[option[1]] = v
-			except NoSectionError, e:
+			except NoSectionError as e:
 				# No "Definition" section or wrong basedir
 				logSys.error(e)
 				values[option[1]] = option[2]
@@ -43,12 +43,12 @@ from optparse import OptionParser, Option
 from ConfigParser import NoOptionError, NoSectionError, MissingSectionHeaderError

 try:
-	from systemd import journal
 	from ..server.filtersystemd import FilterSystemd
 except ImportError:
-	journal = None
+	FilterSystemd = None

 from ..version import version
+from .jailreader import JailReader
 from .filterreader import FilterReader
 from ..server.filter import Filter, FileContainer
 from ..server.failregex import RegexException
@@ -82,7 +82,7 @@ def pprint_list(l, header=None):
 		s = ''
 	output( s + "|  " + "\n|  ".join(l) + '\n`-' )

-def journal_lines_gen(myjournal):
+def journal_lines_gen(flt, myjournal): # pragma: no cover
 	while True:
 		try:
 			entry = myjournal.get_next()
@@ -90,7 +90,7 @@ def journal_lines_gen(myjournal):
 			continue
 		if not entry:
 			break
-		yield FilterSystemd.formatJournalEntry(entry)
+		yield flt.formatJournalEntry(entry)

 def get_opt_parser():
 	# use module docstring for help output
@@ -126,6 +126,8 @@ Report bugs to https://github.com/fail2ban/fail2ban/issues
 			   help="set custom pattern used to match date/times"),
 		Option("-e", "--encoding",
 			   help="File encoding. Default: system locale"),
+		Option("-r", "--raw", action='store_true',
+			   help="Raw hosts, don't resolve dns"),
 		Option("-L", "--maxlines", type=int, default=0,
 			   help="maxlines for multi-line regex"),
 		Option("-m", "--journalmatch",
@@ -239,6 +241,7 @@ class Fail2banRegex(object):
 			self.encoding = opts.encoding
 		else:
 			self.encoding = locale.getpreferredencoding()
+		self.raw = True if opts.raw else False

 	def decode_line(self, line):
 		return FileContainer.decode_line('<LOG>', self.encoding, line)
@@ -326,7 +329,7 @@ class Fail2banRegex(object):
 			if ret is not None:
 				found = True
 				regex = self._ignoreregex[ret].inc()
-		except RegexException, e:
+		except RegexException as e:
 			output( e )
 			return False
 		return found
@@ -335,7 +338,7 @@ class Fail2banRegex(object):
 		orgLineBuffer = self._filter._Filter__lineBuffer
 		fullBuffer = len(orgLineBuffer) >= self._filter.getMaxLines()
 		try:
-			line, ret = self._filter.processLine(line, date, checkAllRegex=True)
+			line, ret = self._filter.processLine(line, date, checkAllRegex=True, returnRawHost=self.raw)
 			for match in ret:
 				# Append True/False flag depending if line was matched by
 				# more than one regex
@@ -343,7 +346,7 @@ class Fail2banRegex(object):
 				regex = self._failregex[match[0]]
 				regex.inc()
 				regex.appendIP(match)
-		except RegexException, e:
+		except RegexException as e:
 			output( e )
 			return False
 		except IndexError:
@@ -507,28 +510,25 @@ class Fail2banRegex(object):
 				output( "Use         log file : %s" % cmd_log )
 				output( "Use         encoding : %s" % self.encoding )
 				test_lines = self.file_lines_gen(hdlr)
-			except IOError, e:
+			except IOError as e:
 				output( e )
 				return False
-		elif cmd_log == "systemd-journal": # pragma: no cover
-			if not journal:
+		elif cmd_log.startswith("systemd-journal"): # pragma: no cover
+			if not FilterSystemd:
 				output( "Error: systemd library not found. Exiting..." )
 				return False
-			myjournal = journal.Reader(converters={'__CURSOR': lambda x: x})
+			output( "Use         systemd journal" )
+			output( "Use         encoding : %s" % self.encoding )
+			backend, beArgs = JailReader.extractOptions(cmd_log)
+			flt = FilterSystemd(None, **beArgs)
+			flt.setLogEncoding(self.encoding)
+			myjournal = flt.getJournalReader()
 			journalmatch = self._journalmatch
 			self.setDatePattern(None)
 			if journalmatch:
-				try:
-					for element in journalmatch:
-						if element == "+":
-							myjournal.add_disjunction()
-						else:
-							myjournal.add_match(element)
-				except ValueError:
-					output( "Error: Invalid journalmatch: %s" % shortstr(" ".join(journalmatch)) )
-					return False
+				flt.addJournalMatch(journalmatch)
 			output( "Use    journal match : %s" % " ".join(journalmatch) )
-			test_lines = journal_lines_gen(myjournal)
+			test_lines = journal_lines_gen(flt, myjournal)
 		else:
 			output( "Use      single line : %s" % shortstr(cmd_log) )
 			test_lines = [ cmd_log ]
@@ -27,14 +27,13 @@ __license__ = "GPL"
 import glob
 import json
 import os.path
-import re

 from .configreader import ConfigReaderUnshared, ConfigReader
 from .filterreader import FilterReader
 from .actionreader import ActionReader
 from ..version import version
 from ..helpers import getLogger
-from ..helpers import splitcommaspace
+from ..helpers import extractOptions, splitwords

 # Gets the instance of the logger.
 logSys = getLogger(__name__)
@@ -42,10 +41,6 @@ logSys = getLogger(__name__)

 class JailReader(ConfigReader):
 	
-	optionCRE = re.compile("^((?:\w|-|_|\.)+)(?:\[(.*)\])?$")
-	optionExtractRE = re.compile(
-		r'([\w\-_\.]+)=(?:"([^"]*)"|\'([^\']*)\'|([^,]*))(?:,|$)')
-	
 	def __init__(self, name, force_enable=False, **kwargs):
 		ConfigReader.__init__(self, **kwargs)
 		self.__name = name
@@ -121,7 +116,7 @@ class JailReader(ConfigReader):
 		if self.isEnabled():
 			# Read filter
 			if self.__opts["filter"]:
-				filterName, filterOpt = JailReader.extractOptions(
+				filterName, filterOpt = extractOptions(
 					self.__opts["filter"])
 				self.__filter = FilterReader(
 					filterName, self.__name, filterOpt, share_config=self.share_config, basedir=self.getBaseDir())
@@ -150,7 +145,7 @@ class JailReader(ConfigReader):
 				try:
 					if not act:			  # skip empty actions
 						continue
-					actName, actOpt = JailReader.extractOptions(act)
+					actName, actOpt = extractOptions(act)
 					if actName.endswith(".py"):
 						self.__actions.append([
 							"set",
@@ -171,7 +166,7 @@ class JailReader(ConfigReader):
 							self.__actions.append(action)
 						else:
 							raise AttributeError("Unable to read action")
-				except Exception, e:
+				except Exception as e:
 					logSys.error("Error in action definition " + act)
 					logSys.debug("Caught exception: %s" % (e,))
 					return False
@@ -192,7 +187,7 @@ class JailReader(ConfigReader):
 		stream = []
 		for opt in self.__opts:
 			if opt == "logpath" and	\
-					self.__opts.get('backend', None) != "systemd":
+					not self.__opts.get('backend', None).startswith("systemd"):
 				found_files = 0
 				for path in self.__opts[opt].split("\n"):
 					path = path.rsplit(" ", 1)
@@ -214,7 +209,7 @@ class JailReader(ConfigReader):
 			elif opt == "maxretry":
 				stream.append(["set", self.__name, "maxretry", self.__opts[opt]])
 			elif opt == "ignoreip":
-				for ip in splitcommaspace(self.__opts[opt]):
+				for ip in splitwords(self.__opts[opt]):
 					stream.append(["set", self.__name, "addignoreip", ip])
 			elif opt == "findtime":
 				stream.append(["set", self.__name, "findtime", self.__opts[opt]])
@@ -244,18 +239,4 @@ class JailReader(ConfigReader):
 		stream.insert(0, ["add", self.__name, backend])
 		return stream
 	
-	@staticmethod
-	def extractOptions(option):
-		match = JailReader.optionCRE.match(option)
-		if not match:
-			# TODO proper error handling
-			return None, None
-		option_name, optstr = match.groups()
-		option_opts = dict()
-		if optstr:
-			for optmatch in JailReader.optionExtractRE.finditer(optstr):
-				opt = optmatch.group(1)
-				value = [
-					val for val in optmatch.group(2,3,4) if val is not None][0]
-				option_opts[opt.strip()] = value.strip()
-		return option_name, option_opts
+JailReader.extractOptions = staticmethod(extractOptions)
@@ -128,12 +128,41 @@ def excepthook(exctype, value, traceback):
 		"Unhandled exception in Fail2Ban:", exc_info=True)
 	return sys.__excepthook__(exctype, value, traceback)

-def splitcommaspace(s):
-	"""Helper to split on any comma or space
+def splitwords(s):
+	"""Helper to split words on any comma, space, or a new line

 	Returns empty list if input is empty (or None) and filters
 	out empty entries
 	"""
 	if not s:
 		return []
-	return filter(bool, re.split('[ ,]', s))
+	return filter(bool, map(str.strip, re.split('[ ,\n]+', s)))
+
+
+#
+# Following function used for parse options from parameter (e.g. `name[p1=0, p2="..."][p3='...']`).
+#
+
+# regex, to extract list of options:
+OPTION_CRE = re.compile(r"^([^\[]+)(?:\[(.*)\])?\s*$", re.DOTALL)
+# regex, to iterate over single option in option list, syntax:
+# `action = act[p1="...", p2='...', p3=...]`, where the p3=... not contains `,` or ']'
+# since v0.10 separator extended with `]\s*[` for support of multiple option groups, syntax 
+# `action = act[p1=...][p2=...]`
+OPTION_EXTRACT_CRE = re.compile(
+	r'([\w\-_\.]+)=(?:"([^"]*)"|\'([^\']*)\'|([^,\]]*))(?:,|\]\s*\[|$)', re.DOTALL)
+
+def extractOptions(option):
+	match = OPTION_CRE.match(option)
+	if not match:
+		# TODO proper error handling
+		return None, None
+	option_name, optstr = match.groups()
+	option_opts = dict()
+	if optstr:
+		for optmatch in OPTION_EXTRACT_CRE.finditer(optstr):
+			opt = optmatch.group(1)
+			value = [
+				val for val in optmatch.group(2,3,4) if val is not None][0]
+			option_opts[opt.strip()] = value.strip()
+	return option_name, option_opts
@@ -55,6 +55,9 @@ _RETCODE_HINTS = {
 signame = dict((num, name)
 	for name, num in signal.__dict__.iteritems() if name.startswith("SIG"))

+# max tag replacement count:
+MAX_TAG_REPLACE_COUNT = 10
+

 class CallingMap(MutableMapping):
 	"""A Mapping type which returns the result of callable values.
@@ -390,22 +393,22 @@ class CommandAction(ActionBase):
 		"""
 		t = re.compile(r'<([^ <>]+)>')
 		# repeat substitution while embedded-recursive (repFlag is True)
+		done = cls._escapedTags.copy()
 		while True:
 			repFlag = False
 			# substitute each value:
 			for tag in tags.iterkeys():
-				if tag in cls._escapedTags:
-					# Escaped so won't match
-					continue
+				# ignore escaped or already done:
+				if tag in done: continue
 				value = str(tags[tag])
 				# search and replace all tags within value, that can be interpolated using other tags:
 				m = t.search(value)
-				done = []
+				refCounts = {}
 				#logSys.log(5, 'TAG: %s, value: %s' % (tag, value))
 				while m:
 					found_tag = m.group(1)
 					#logSys.log(5, 'found: %s' % found_tag)
-					if found_tag == tag or found_tag in done:
+					if found_tag == tag or refCounts.get(found_tag, 1) > MAX_TAG_REPLACE_COUNT:
 						# recursive definitions are bad
 						#logSys.log(5, 'recursion fail tag: %s value: %s' % (tag, value) )
 						return False
@@ -417,7 +420,9 @@ class CommandAction(ActionBase):
 						continue
 					value = value.replace('<%s>' % found_tag , tags[found_tag])
 					#logSys.log(5, 'value now: %s' % value)
-					done.append(found_tag)
+					# increment reference count:
+					refCounts[found_tag] = refCounts.get(found_tag, 0) + 1
+					# the next match for replace:
 					m = t.search(value, m.start())
 				#logSys.log(5, 'TAG: %s, newvalue: %s' % (tag, value))
 				# was substituted?
@@ -426,6 +431,9 @@ class CommandAction(ActionBase):
 					if t.search(value):
 						repFlag = True
 					tags[tag] = value
+				# no more sub tags (and no possible composite), add this tag to done set (just to be faster):
+				if '<' not in value: done.add(tag)
+			# stop interpolation, if no replacements anymore:
 			if not repFlag:
 				break
 		return tags
@@ -42,7 +42,7 @@ if sys.version_info >= (3,):
 		try:
 			x = json.dumps(x, ensure_ascii=False).encode(
 				locale.getpreferredencoding(), 'replace')
-		except Exception, e: # pragma: no cover
+		except Exception as e: # pragma: no cover
 			logSys.error('json dumps failed: %s', e)
 			x = '{}'
 		return x
@@ -51,7 +51,7 @@ if sys.version_info >= (3,):
 		try:
 			x = json.loads(x.decode(
 				locale.getpreferredencoding(), 'replace'))
-		except Exception, e: # pragma: no cover
+		except Exception as e: # pragma: no cover
 			logSys.error('json loads failed: %s', e)
 			x = {}
 		return x
@@ -70,7 +70,7 @@ else:
 		try:
 			x = json.dumps(_normalize(x), ensure_ascii=False).decode(
 				locale.getpreferredencoding(), 'replace')
-		except Exception, e: # pragma: no cover
+		except Exception as e: # pragma: no cover
 			logSys.error('json dumps failed: %s', e)
 			x = '{}'
 		return x
@@ -79,7 +79,7 @@ else:
 		try:
 			x = _normalize(json.loads(x.decode(
 				locale.getpreferredencoding(), 'replace')))
-		except Exception, e: # pragma: no cover
+		except Exception as e: # pragma: no cover
 			logSys.error('json loads failed: %s', e)
 			x = {}
 		return x
@@ -175,14 +175,29 @@ class Fail2BanDb(object):

 			logSys.info(
 				"Connected to fail2ban persistent database '%s'", filename)
-		except sqlite3.OperationalError, e:
+		except sqlite3.OperationalError as e:
 			logSys.error(
 				"Error connecting to fail2ban persistent database '%s': %s",
 				filename, e.args[0])
 			raise

+		# differentiate pypy: switch journal mode later (save it during the upgrade), 
+		# to prevent errors like "database table is locked":
+		try:
+			import __pypy__
+			pypy = True
+		except ImportError:
+			pypy = False
+
 		cur = self._db.cursor()
-		cur.execute("PRAGMA foreign_keys = ON;")
+		cur.execute("PRAGMA foreign_keys = ON")
+		# speedup: write data through OS without syncing (no wait):
+		cur.execute("PRAGMA synchronous = OFF")
+		# speedup: transaction log in memory, alternate using OFF (disable, rollback will be impossible):
+		if not pypy:
+			cur.execute("PRAGMA journal_mode = MEMORY")
+		# speedup: temporary tables and indices are kept in memory:
+		cur.execute("PRAGMA temp_store = MEMORY")

 		try:
 			cur.execute("SELECT version FROM fail2banDb LIMIT 1")
@@ -202,6 +217,9 @@ class Fail2BanDb(object):
 						Fail2BanDb.__version__, version, newversion)
 					raise RuntimeError('Failed to fully update')
 		finally:
+			# pypy: set journal mode after possible upgrade db:
+			if pypy:
+				cur.execute("PRAGMA journal_mode = MEMORY")
 			cur.close()

 	@property
@@ -244,13 +262,14 @@ class Fail2BanDb(object):

 		A timestamped backup is also created prior to attempting the update.
 		"""
-		self._dbBackupFilename = self.filename + '.' + time.strftime('%Y%m%d-%H%M%S', MyTime.gmtime())
-		shutil.copyfile(self.filename, self._dbBackupFilename)
-		logSys.info("Database backup created: %s", self._dbBackupFilename)
 		if version > Fail2BanDb.__version__:
 			raise NotImplementedError(
 						"Attempt to travel to future version of database ...how did you get here??")

+		self._dbBackupFilename = self.filename + '.' + time.strftime('%Y%m%d-%H%M%S', MyTime.gmtime())
+		shutil.copyfile(self.filename, self._dbBackupFilename)
+		logSys.info("Database backup created: %s", self._dbBackupFilename)
+
 		if version < 2:
 			cur.executescript("BEGIN TRANSACTION;"
 						"CREATE TEMPORARY TABLE logs_temp AS SELECT * FROM logs;"
@@ -274,8 +293,12 @@ class Fail2BanDb(object):
 			Jail to be added to the database.
 		"""
 		cur.execute(
-			"INSERT OR REPLACE INTO jails(name, enabled) VALUES(?, 1)",
+			"INSERT OR IGNORE INTO jails(name, enabled) VALUES(?, 1)",
 			(jail.name,))
+		if cur.rowcount <= 0:
+			cur.execute(
+				"UPDATE jails SET enabled = 1 WHERE name = ? AND enabled != 1",
+				(jail.name,))

 	@commitandrollback
 	def delJail(self, cur, jail):
@@ -298,7 +321,7 @@ class Fail2BanDb(object):
 		cur.execute("UPDATE jails SET enabled=0")

 	@commitandrollback
-	def getJailNames(self, cur):
+	def getJailNames(self, cur, enabled=None):
 		"""Get name of jails in database.

 		Currently only used for testing purposes.
@@ -308,7 +331,11 @@ class Fail2BanDb(object):
 		set
 			Set of jail names.
 		"""
-		cur.execute("SELECT name FROM jails")
+		if enabled is None:
+			cur.execute("SELECT name FROM jails")
+		else:
+			cur.execute("SELECT name FROM jails WHERE enabled=%s" %
+				(int(enabled),))
 		return set(row[0] for row in cur.fetchmany())

 	@commitandrollback
@@ -64,7 +64,7 @@ class DateTemplate(object):
 	def getRegex(self):
 		return self._regex

-	def setRegex(self, regex, wordBegin=True):
+	def setRegex(self, regex, wordBegin=True, wordEnd=True):
 		"""Sets regex to use for searching for date in log line.

 		Parameters
@@ -72,8 +72,12 @@ class DateTemplate(object):
 		regex : str
 			The regex the template will use for searching for a date.
 		wordBegin : bool
-			Defines whether the regex should be modified to search at
-			beginning of a word, by adding "\\b" to start of regex.
+			Defines whether the regex should be modified to search at beginning of a
+			word, by adding special boundary r'(?=^|\b|\W)' to start of regex.
+			Default True.
+		wordEnd : bool
+			Defines whether the regex should be modified to search at end of a word,
+			by adding special boundary r'(?=\b|\W|$)' to end of regex.
 			Default True.

 		Raises
@@ -82,8 +86,10 @@ class DateTemplate(object):
 			If regular expression fails to compile
 		"""
 		regex = regex.strip()
-		if (wordBegin and not re.search(r'^\^', regex)):
-			regex = r'\b' + regex
+		if wordBegin and not re.search(r'^\^', regex):
+			regex = r'(?=^|\b|\W)' + regex
+		if wordEnd and not re.search(r'\$$', regex):
+			regex += r'(?=\b|\W|$)'
 		self._regex = regex
 		self._cRegex = re.compile(regex, re.UNICODE | re.IGNORECASE)

@@ -24,6 +24,7 @@ __license__ = "GPL"
 import codecs
 import fcntl
 import locale
+import logging
 import os
 import re
 import sys
@@ -82,6 +83,8 @@ class Filter(JailThread):
 		self.__lastDate = None
 		## External command
 		self.__ignoreCommand = False
+		## Default or preferred encoding (to decode bytes from file or journal):
+		self.__encoding = locale.getpreferredencoding()

 		self.dateDetector = DateDetector()
 		self.dateDetector.addDefaultTemplate()
@@ -105,7 +108,7 @@ class Filter(JailThread):
 				logSys.warning(
 					"Mutliline regex set for jail '%s' "
 					"but maxlines not greater than 1")
-		except RegexException, e:
+		except RegexException as e:
 			logSys.error(e)
 			raise e

@@ -138,7 +141,7 @@ class Filter(JailThread):
 		try:
 			regex = Regex(value)
 			self.__ignoreRegex.append(regex)
-		except RegexException, e:
+		except RegexException as e:
 			logSys.error(e)
 			raise e 

@@ -168,7 +171,7 @@ class Filter(JailThread):
 		if isinstance(value, bool):
 			value = {True: 'yes', False: 'no'}[value]
 		value = value.lower()			  # must be a string by now
-		if not (value in ('yes', 'no', 'warn')):
+		if not (value in ('yes', 'warn', 'no', 'raw')):
 			logSys.error("Incorrect value %r specified for usedns. "
 						 "Using safe 'no'" % (value,))
 			value = 'no'
@@ -279,6 +282,27 @@ class Filter(JailThread):
 	def getMaxLines(self):
 		return self.__lineBufferSize

+	##
+	# Set the log file encoding
+	#
+	# @param encoding the encoding used with log files
+
+	def setLogEncoding(self, encoding):
+		if encoding.lower() == "auto":
+			encoding = locale.getpreferredencoding()
+		codecs.lookup(encoding) # Raise LookupError if invalid codec
+		self.__encoding = encoding
+		logSys.info("Set jail log file encoding to %s" % encoding)
+		return encoding
+
+	##
+	# Get the log file encoding
+	#
+	# @return log encoding value
+
+	def getLogEncoding(self):
+		return self.__encoding
+
 	##
 	# Main loop.
 	#
@@ -394,8 +418,31 @@ class Filter(JailThread):

 		return False

+	if sys.version_info >= (3,):
+		@staticmethod
+		def uni_decode(x, enc, errors='strict'):
+			try:
+				if isinstance(x, bytes):
+					return x.decode(enc, errors)
+				return x
+			except (UnicodeDecodeError, UnicodeEncodeError): # pragma: no cover - unsure if reachable
+				if errors != 'strict': 
+					raise
+				return uni_decode(x, enc, 'replace')
+	else:
+		@staticmethod
+		def uni_decode(x, enc, errors='strict'):
+			try:
+				if isinstance(x, unicode):
+					return x.encode(enc, errors)
+				return x
+			except (UnicodeDecodeError, UnicodeEncodeError): # pragma: no cover - unsure if reachable
+				if errors != 'strict':
+					raise
+				return uni_decode(x, enc, 'replace')
+
 	def processLine(self, line, date=None, returnRawHost=False,
-		checkAllRegex=False):
+		checkAllRegex=False, checkFindTime=False):
 		"""Split the time portion from log msg and return findFailures on them
 		"""
 		if date:
@@ -414,21 +461,17 @@ class Filter(JailThread):
 				tupleLine = (l, "", "")

 		return "".join(tupleLine[::2]), self.findFailure(
-			tupleLine, date, returnRawHost, checkAllRegex)
+			tupleLine, date, returnRawHost, checkAllRegex, checkFindTime)

 	def processLineAndAdd(self, line, date=None):
 		"""Processes the line for failures and populates failManager
 		"""
-		for element in self.processLine(line, date)[1]:
+		for element in self.processLine(line, date, checkFindTime=True)[1]:
 			ip = element[1]
 			unixTime = element[2]
 			lines = element[3]
 			logSys.debug("Processing line with time:%s and ip:%s"
 						 % (unixTime, ip))
-			if unixTime < MyTime.time() - self.getFindTime():
-				logSys.debug("Ignore line since time %s < %s - %s"
-							 % (unixTime, MyTime.time(), self.getFindTime()))
-				break
 			if self.inIgnoreIPList(ip, log_ignore=True):
 				continue
 			logSys.info("[%s] Found %s" % (self.jail.name, ip))
@@ -457,7 +500,7 @@ class Filter(JailThread):
 	# @return a dict with IP and timestamp.

 	def findFailure(self, tupleLine, date=None, returnRawHost=False,
-		checkAllRegex=False):
+		checkAllRegex=False, checkFindTime=False):
 		failList = list()

 		# Checks if we must ignore this line.
@@ -489,6 +532,11 @@ class Filter(JailThread):
 			timeText = self.__lastTimeText or "".join(tupleLine[::2])
 			date = self.__lastDate

+		if checkFindTime and date is not None and date < MyTime.time() - self.getFindTime():
+			logSys.log(5, "Ignore line since time %s < %s - %s", 
+				date, MyTime.time(), self.getFindTime())
+			return failList
+
 		self.__lineBuffer = (
 			self.__lineBuffer + [tupleLine])[-self.__lineBufferSize:]
 		logSys.log(5, "Looking for failregex match of %r" % self.__lineBuffer)
@@ -523,7 +571,7 @@ class Filter(JailThread):
 					self.__lineBuffer = failRegex.getUnmatchedTupleLines()
 					try:
 						host = failRegex.getHost()
-						if returnRawHost:
+						if returnRawHost or self.__useDns == "raw":
 							failList.append([failRegexIndex, host, date,
 								 failRegex.getMatchedLines()])
 							if not checkAllRegex:
@@ -536,7 +584,7 @@ class Filter(JailThread):
 										 failRegex.getMatchedLines()])
 								if not checkAllRegex:
 									break
-					except RegexException, e: # pragma: no cover - unsure if reachable
+					except RegexException as e: # pragma: no cover - unsure if reachable
 						logSys.error(e)
 		return failList

@@ -554,7 +602,6 @@ class FileFilter(Filter):
 		Filter.__init__(self, jail, **kwargs)
 		## The log file path.
 		self.__logs = dict()
-		self.setLogEncoding("auto")

 	##
 	# Add a log file path
@@ -625,21 +672,9 @@ class FileFilter(Filter):
 	# @param encoding the encoding used with log files

 	def setLogEncoding(self, encoding):
-		if encoding.lower() == "auto":
-			encoding = locale.getpreferredencoding()
-		codecs.lookup(encoding) # Raise LookupError if invalid codec
+		encoding = super(FileFilter, self).setLogEncoding(encoding)
 		for log in self.__logs.itervalues():
 			log.setEncoding(encoding)
-		self.__encoding = encoding
-		logSys.info("Set jail log file encoding to %s" % encoding)
-
-	##
-	# Get the log file encoding
-	#
-	# @return log encoding value
-
-	def getLogEncoding(self):
-		return self.__encoding

 	def getLog(self, path):
 		return self.__logs.get(path, None)
@@ -660,15 +695,15 @@ class FileFilter(Filter):
 		try:
 			has_content = log.open()
 		# see http://python.org/dev/peps/pep-3151/
-		except IOError, e:
+		except IOError as e:
 			logSys.error("Unable to open %s" % filename)
 			logSys.exception(e)
 			return False
-		except OSError, e: # pragma: no cover - requires race condition to tigger this
+		except OSError as e: # pragma: no cover - requires race condition to tigger this
 			logSys.error("Error opening %s" % filename)
 			logSys.exception(e)
 			return False
-		except Exception, e: # pragma: no cover - Requires implemention error in FileContainer to generate
+		except Exception as e: # pragma: no cover - Requires implemention error in FileContainer to generate
 			logSys.error("Internal errror in FileContainer open method - please report as a bug to https://github.com/fail2ban/fail2ban/issues")
 			logSys.exception(e)
 			return False
@@ -707,7 +742,12 @@ class FileFilter(Filter):

 try:
 	import hashlib
-	md5sum = hashlib.md5
+	try:
+		md5sum = hashlib.md5
+		# try to use it (several standards like FIPS forbid it):
+		md5sum(' ').hexdigest()
+	except: # pragma: no cover
+		md5sum = hashlib.sha1
 except ImportError: # pragma: no cover
 	# hashlib was introduced in Python 2.5.  For compatibility with those
 	# elderly Pythons, import from md5
@@ -780,7 +820,7 @@ class FileContainer:
 		## sys.stdout.flush()
 		# Compare hash and inode
 		if self.__hash != myHash or self.__ino != stats.st_ino:
-			logSys.info("Log rotation detected for %s" % self.__filename)
+			logSys.log(logging.MSG, "Log rotation detected for %s" % self.__filename)
 			self.__hash = myHash
 			self.__ino = stats.st_ino
 			self.__pos = 0
@@ -791,14 +831,19 @@ class FileContainer:
 	@staticmethod
 	def decode_line(filename, enc, line):
 		try:
-			line = line.decode(enc, 'strict')
-		except UnicodeDecodeError:
-			logSys.warning(
+			return line.decode(enc, 'strict')
+		except (UnicodeDecodeError, UnicodeEncodeError) as e:
+			global _decode_line_warn
+			lev = logging.DEBUG
+			if _decode_line_warn.get(filename, 0) <= MyTime.time():
+				lev = logging.WARNING
+				_decode_line_warn[filename] = MyTime.time() + 24*60*60
+			logSys.log(lev,
 				"Error decoding line from '%s' with '%s'."
 				" Consider setting logencoding=utf-8 (or another appropriate"
 				" encoding) for this jail. Continuing"
-				" to process line ignoring invalid characters: %r" %
-				(filename, enc, line))
+				" to process line ignoring invalid characters: %r",
+				filename, enc, line)
 			# decode with replacing error chars:
 			line = line.decode(enc, 'replace')
 		return line
@@ -819,6 +864,8 @@ class FileContainer:
 		## print "D: Closed %s with pos %d" % (handler, self.__pos)
 		## sys.stdout.flush()

+_decode_line_warn = {}
+

 ##
 # JournalFilter class.
@@ -858,11 +905,11 @@ class DNSUtils:
 		# retrieve ip (todo: use AF_INET6 for IPv6)
 		try:
 			return set([i[4][0] for i in socket.getaddrinfo(dns, None, socket.AF_INET, 0, socket.IPPROTO_TCP)])
-		except socket.error, e:
+		except socket.error as e:
 			logSys.warning("Unable to find a corresponding IP address for %s: %s"
 						% (dns, e))
 			return list()
-		except socket.error, e:
+		except socket.error as e:
 			logSys.warning("Socket error raised trying to resolve hostname %s: %s"
 						% (dns, e))
 			return list()
@@ -871,7 +918,7 @@ class DNSUtils:
 	def ipToName(ip):
 		try:
 			return socket.gethostbyaddr(ip)[0]
-		except socket.error, e:
+		except socket.error as e:
 			logSys.debug("Unable to find a name for the IP %s: %s" % (ip, e))
 			return None

@@ -138,7 +138,7 @@ class FilterPoll(FileFilter):
 				logSys.debug("%s has been modified", filename)
 				self.__prevStats[filename] = stats
 				return True
-		except OSError, e:
+		except OSError as e:
 			logSys.error("Unable to get stat on %s because of: %s"
 						 % (filename, e))
 			self.__file404Cnt[filename] += 1
@@ -44,7 +44,7 @@ if not hasattr(pyinotify, '__version__') \
 try:
 	manager = pyinotify.WatchManager()
 	del manager
-except Exception, e:
+except Exception as e:
 	raise ImportError("Pyinotify is probably not functional on this system: %s"
 					  % str(e))

@@ -31,9 +31,9 @@ if LooseVersion(getattr(journal, '__version__', "0")) < '204':
 	raise ImportError("Fail2Ban requires systemd >= 204")

 from .failmanager import FailManagerEmpty
-from .filter import JournalFilter
+from .filter import JournalFilter, Filter
 from .mytime import MyTime
-from ..helpers import getLogger
+from ..helpers import getLogger, logging, splitwords

 # Gets the instance of the logger.
 logSys = getLogger(__name__)
@@ -54,14 +54,45 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover
 	# @param jail the jail object

 	def __init__(self, jail, **kwargs):
+		jrnlargs = FilterSystemd._getJournalArgs(kwargs)
 		JournalFilter.__init__(self, jail, **kwargs)
-		self.__modified = False
+		self.__modified = 0
 		# Initialise systemd-journal connection
-		self.__journal = journal.Reader(converters={'__CURSOR': lambda x: x})
+		self.__journal = journal.Reader(**jrnlargs)
 		self.__matches = []
 		self.setDatePattern(None)
+		self.ticks = 0
 		logSys.debug("Created FilterSystemd")

+	@staticmethod
+	def _getJournalArgs(kwargs):
+		args = {'converters':{'__CURSOR': lambda x: x}}
+		try:
+			args['path'] = kwargs.pop('journalpath')
+		except KeyError:
+			pass
+
+		try:
+			args['files'] = kwargs.pop('journalfiles')
+		except KeyError:
+			pass
+		else:
+			import glob
+			p = args['files']
+			if not isinstance(p, (list, set, tuple)):
+				p = splitwords(p)
+			files = []
+			for p in p:
+				files.extend(glob.glob(p))
+			args['files'] = list(set(files))
+
+		try:
+			args['flags'] = kwargs.pop('journalflags')
+		except KeyError:
+			pass
+
+		return args
+
 	##
 	# Add a journal match filters from list structure
 	#
@@ -139,21 +170,17 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover
 	def getJournalMatch(self):
 		return self.__matches

-    ##
-    # Join group of log elements which may be a mix of bytes and strings
-    #
-    # @param elements list of strings and bytes
-    # @return elements joined as string
+	def uni_decode(self, x):
+		v = Filter.uni_decode(x, self.getLogEncoding())
+		return v

-	@staticmethod
-	def _joinStrAndBytes(elements):
-		strElements = []
-		for element in elements:
-			if isinstance(element, str):
-				strElements.append(element)
-			else:
-				strElements.append(str(element, errors='ignore'))
-		return " ".join(strElements)
+	##
+	# Get journal reader
+	#
+	# @return journal reader
+
+	def getJournalReader(self):
+		return self.__journal

 	##
 	# Format journal log entry into syslog style
@@ -161,52 +188,58 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover
 	# @param entry systemd journal entry dict
 	# @return format log line

-	@classmethod
-	def formatJournalEntry(cls, logentry):
-		logelements = [""]
-		if logentry.get('_HOSTNAME'):
-			logelements.append(logentry['_HOSTNAME'])
-		if logentry.get('SYSLOG_IDENTIFIER'):
-			logelements.append(logentry['SYSLOG_IDENTIFIER'])
-			if logentry.get('SYSLOG_PID'):
-				logelements[-1] += ("[%i]" % logentry['SYSLOG_PID'])
-			elif logentry.get('_PID'):
-				logelements[-1] += ("[%i]" % logentry['_PID'])
+	def formatJournalEntry(self, logentry):
+		# Be sure, all argument of line tuple should have the same type:
+		uni_decode = self.uni_decode
+		logelements = []
+		v = logentry.get('_HOSTNAME')
+		if v:
+			logelements.append(uni_decode(v))
+		v = logentry.get('SYSLOG_IDENTIFIER')
+		if not v:
+			v = logentry.get('_COMM')
+		if v:
+			logelements.append(uni_decode(v))
+			v = logentry.get('SYSLOG_PID')
+			if not v:
+				v = logentry.get('_PID')
+			if v:
+				try: # [integer] (if already numeric):
+					v = "[%i]" % v
+				except TypeError:
+					try: # as [integer] (try to convert to int):
+						v = "[%i]" % int(v, 0)
+					except (TypeError, ValueError): # fallback - [string] as it is
+						v = "[%s]" % v
+				logelements[-1] += v
 			logelements[-1] += ":"
-		elif logentry.get('_COMM'):
-			logelements.append(logentry['_COMM'])
-			if logentry.get('_PID'):
-				logelements[-1] += ("[%i]" % logentry['_PID'])
-			logelements[-1] += ":"
-		if logelements[-1] == "kernel:":
-			if '_SOURCE_MONOTONIC_TIMESTAMP' in logentry:
-				monotonic = logentry.get('_SOURCE_MONOTONIC_TIMESTAMP')
-			else:
-				monotonic = logentry.get('__MONOTONIC_TIMESTAMP')[0]
-			logelements.append("[%12.6f]" % monotonic.total_seconds())
-		if isinstance(logentry.get('MESSAGE',''), list):
-			logelements.append(" ".join(logentry['MESSAGE']))
+			if logelements[-1] == "kernel:":
+				if '_SOURCE_MONOTONIC_TIMESTAMP' in logentry:
+					monotonic = logentry.get('_SOURCE_MONOTONIC_TIMESTAMP')
+				else:
+					monotonic = logentry.get('__MONOTONIC_TIMESTAMP')[0]
+				logelements.append("[%12.6f]" % monotonic.total_seconds())
+		msg = logentry.get('MESSAGE','')
+		if isinstance(msg, list):
+			logelements.append(" ".join(uni_decode(v) for v in msg))
 		else:
-			logelements.append(logentry.get('MESSAGE', ''))
+			logelements.append(uni_decode(msg))

-		try:
-			logline = u" ".join(logelements)
-		except UnicodeDecodeError:
-			# Python 2, so treat as string
-			logline = " ".join([str(logline) for logline in logelements])
-		except TypeError:
-			# Python 3, one or more elements bytes
-			logSys.warning("Error decoding log elements from journal: %s" %
-				repr(logelements))
-			logline =  cls._joinStrAndBytes(logelements)
+		logline = " ".join(logelements)

 		date = logentry.get('_SOURCE_REALTIME_TIMESTAMP',
 				logentry.get('__REALTIME_TIMESTAMP'))
 		logSys.debug("Read systemd journal entry: %r" %
 			"".join([date.isoformat(), logline]))
-		return (('', date.isoformat(), logline),
+		## use the same type for 1st argument:
+		return ((logline[:0], date.isoformat(), logline.replace('\n', '\\n')),
 			time.mktime(date.timetuple()) + date.microsecond/1.0E6)

+	def seekToTime(self, date):
+		if not isinstance(date, datetime.datetime):
+			date = datetime.datetime.fromtimestamp(date)
+		self.__journal.seek_realtime(date)
+
 	##
 	# Main loop.
 	#
@@ -224,7 +257,7 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover
 		# Seek to now - findtime in journal
 		start_time = datetime.datetime.now() - \
 				datetime.timedelta(seconds=int(self.getFindTime()))
-		self.__journal.seek_realtime(start_time)
+		self.seekToTime(start_time)
 		# Move back one entry to ensure do not end up in dead space
 		# if start time beyond end of journal
 		try:
@@ -233,29 +266,45 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover
 			pass # Reading failure, so safe to ignore

 		while self.active:
-			if not self.idle:
-				while self.active:
-					try:
-						logentry = self.__journal.get_next()
-					except OSError:
-						logSys.warning(
-							"Error reading line from systemd journal")
-						continue
-					if logentry:
-						self.processLineAndAdd(
-							*self.formatJournalEntry(logentry))
-						self.__modified = True
-					else:
-						break
-				if self.__modified:
-					try:
-						while True:
-							ticket = self.failManager.toBan()
-							self.jail.putFailTicket(ticket)
-					except FailManagerEmpty:
-						self.failManager.cleanup(MyTime.time())
-					self.__modified = False
+			# wait for records (or for timeout in sleeptime seconds):
 			self.__journal.wait(self.sleeptime)
+			if self.idle:
+				# because journal.wait will returns immediatelly if we have records in journal,
+				# just wait a little bit here for not idle, to prevent hi-load:
+				time.sleep(self.sleeptime)
+				continue
+			self.__modified = 0
+			while self.active:
+				logentry = None
+				try:
+					logentry = self.__journal.get_next()
+				except OSError as e:
+					logSys.error("Error reading line from systemd journal: %s",
+						e, exc_info=logSys.getEffectiveLevel() <= logging.DEBUG)
+				self.ticks += 1
+				if logentry:
+					self.processLineAndAdd(
+						*self.formatJournalEntry(logentry))
+					self.__modified += 1
+					if self.__modified >= 100: # todo: should be configurable
+						break
+				else:
+					break
+			if self.__modified:
+				try:
+					while True:
+						ticket = self.failManager.toBan()
+						self.jail.putFailTicket(ticket)
+				except FailManagerEmpty:
+					self.failManager.cleanup(MyTime.time())
+
+		# close journal:
+		try:
+			if self.__journal:
+				self.__journal.close()
+		except Exception as e: # pragma: no cover
+			logSys.error("Close journal failed: %r", e,
+				exc_info=logSys.getEffectiveLevel()<=logging.DEBUG)
 		logSys.debug((self.jail is not None and self.jail.name
                      or "jailless") +" filter terminated")
 		return True
@@ -27,6 +27,7 @@ import logging
 import Queue

 from .actions import Actions
+from ..client.jailreader import JailReader
 from ..helpers import getLogger

 # Gets the instance of the logger.
@@ -82,6 +83,7 @@ class Jail:
 		return "%s(%r)" % (self.__class__.__name__, self.name)

 	def _setBackend(self, backend):
+		backend, beArgs = JailReader.extractOptions(backend)
 		backend = backend.lower()		# to assure consistent matching

 		backends = self._BACKENDS
@@ -98,7 +100,7 @@ class Jail:
 		for b in backends:
 			initmethod = getattr(self, '_init%s' % b.capitalize())
 			try:
-				initmethod()
+				initmethod(**beArgs)
 				if backend != 'auto' and b != backend:
 					logSys.warning("Could only initiated %r backend whenever "
 								   "%r was requested" % (b, backend))
@@ -106,7 +108,7 @@ class Jail:
 					logSys.info("Initiated %r backend" % b)
 				self.__actions = Actions(self)
 				return					# we are done
-			except ImportError, e:
+			except ImportError as e:
 				# Log debug if auto, but error if specific
 				logSys.log(
 					logging.DEBUG if backend == "auto" else logging.ERROR,
@@ -117,28 +119,28 @@ class Jail:
 		raise RuntimeError(
 			"Failed to initialize any backend for Jail %r" % self.name)

-	def _initPolling(self):
+	def _initPolling(self, **kwargs):
 		from filterpoll import FilterPoll
-		logSys.info("Jail '%s' uses poller" % self.name)
-		self.__filter = FilterPoll(self)
+		logSys.info("Jail '%s' uses poller %r" % (self.name, kwargs))
+		self.__filter = FilterPoll(self, **kwargs)

-	def _initGamin(self):
+	def _initGamin(self, **kwargs):
 		# Try to import gamin
 		from filtergamin import FilterGamin
-		logSys.info("Jail '%s' uses Gamin" % self.name)
-		self.__filter = FilterGamin(self)
+		logSys.info("Jail '%s' uses Gamin %r" % (self.name, kwargs))
+		self.__filter = FilterGamin(self, **kwargs)

-	def _initPyinotify(self):
+	def _initPyinotify(self, **kwargs):
 		# Try to import pyinotify
 		from filterpyinotify import FilterPyinotify
-		logSys.info("Jail '%s' uses pyinotify" % self.name)
-		self.__filter = FilterPyinotify(self)
+		logSys.info("Jail '%s' uses pyinotify %r" % (self.name, kwargs))
+		self.__filter = FilterPyinotify(self, **kwargs)

-	def _initSystemd(self): # pragma: systemd no cover
+	def _initSystemd(self, **kwargs): # pragma: systemd no cover
 		# Try to import systemd
 		from filtersystemd import FilterSystemd
-		logSys.info("Jail '%s' uses systemd" % self.name)
-		self.__filter = FilterSystemd(self)
+		logSys.info("Jail '%s' uses systemd %r" % (self.name, kwargs))
+		self.__filter = FilterSystemd(self, **kwargs)

 	@property
 	def name(self):
@@ -108,20 +108,20 @@ class Server:
 			pidFile = open(pidfile, 'w')
 			pidFile.write("%s\n" % os.getpid())
 			pidFile.close()
-		except IOError, e:
+		except IOError as e:
 			logSys.error("Unable to create PID file: %s" % e)
 		
 		# Start the communication
 		logSys.debug("Starting communication")
 		try:
 			self.__asyncServer.start(sock, force)
-		except AsyncServerException, e:
+		except AsyncServerException as e:
 			logSys.error("Could not start server: %s", e)
 		# Removes the PID file.
 		try:
 			logSys.debug("Remove PID file %s" % pidfile)
 			os.remove(pidfile)
-		except OSError, e:
+		except OSError as e:
 			logSys.error("Unable to remove PID file: %s" % e)
 		logSys.info("Exiting Fail2ban")
 	
@@ -237,13 +237,11 @@ class Server:
 	
 	def setLogEncoding(self, name, encoding):
 		filter_ = self.__jails[name].filter
-		if isinstance(filter_, FileFilter):
-			filter_.setLogEncoding(encoding)
+		filter_.setLogEncoding(encoding)
 	
 	def getLogEncoding(self, name):
 		filter_ = self.__jails[name].filter
-		if isinstance(filter_, FileFilter):
-			return filter_.getLogEncoding()
+		return filter_.getLogEncoding()
 	
 	def setFindTime(self, name, value):
 		self.__jails[name].filter.setFindTime(value)
@@ -543,7 +541,7 @@ class Server:
 			# the child gets a new PID, making it impossible for its PID to equal its
 			# PGID.
 			pid = os.fork()
-		except OSError, e:
+		except OSError as e:
 			return((e.errno, e.strerror))	 # ERROR (return a tuple)
 		
 		if pid == 0:	   # The first child.
@@ -564,7 +562,7 @@ class Server:
 				# fork guarantees that the child is no longer a session leader, thus
 				# preventing the daemon from ever acquiring a controlling terminal.
 				pid = os.fork()		# Fork a second child.
-			except OSError, e:
+			except OSError as e:
 				return((e.errno, e.strerror))  # ERROR (return a tuple)
 		
 			if (pid == 0):	  # The second child.
@@ -56,7 +56,7 @@ class Transmitter:
 		try:
 			ret = self.__commandHandler(command)
 			ack = 0, ret
-		except Exception, e:
+		except Exception as e:
 			logSys.warning("Command %r has failed. Received %r"
 						% (command, e))
 			ack = 1, e
@@ -0,0 +1,42 @@
+# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*-
+# vi: set ft=python sts=4 ts=4 sw=4 noet :
+
+# This file is part of Fail2Ban.
+#
+# Fail2Ban is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# Fail2Ban is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Fail2Ban; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+
+__author__ = "Serg G. Brester"
+__license__ = "GPL"
+
+import os
+import sys
+
+
+def updatePyExec(bindir, executable=None):
+	"""Update fail2ban-python link to current python version (where f2b-modules located/installed)
+	"""
+	bindir = os.path.realpath(bindir)
+	if executable is None:
+		executable = sys.executable
+	pypath = os.path.join(bindir, 'fail2ban-python')
+	# if not exists or point to another version - update link:
+	isfile = os.path.isfile(pypath)
+	if not isfile or os.path.realpath(pypath) != os.path.realpath(executable):
+		if isfile:
+			os.unlink(pypath)
+		os.symlink(executable, pypath)
+	# extend current environment path (e.g. if fail2ban not yet installed):
+	if bindir not in os.environ["PATH"].split(os.pathsep):
+		os.environ["PATH"] = os.environ["PATH"] + os.pathsep + bindir;
@@ -37,6 +37,7 @@ if sys.version_info >= (2,7):
 			self.jail.actions.add("badips", pythonModule, initOpts={
 				'category': "ssh",
 				'banaction': "test",
+				'timeout': 30,
 				})
 			self.action = self.jail.actions["badips"]

@@ -48,7 +49,7 @@ if sys.version_info >= (2,7):

 		def testCategory(self):
 			categories = self.action.getCategories()
-			self.assertTrue("ssh" in categories)
+			self.assertIn("ssh", categories)
 			self.assertTrue(len(categories) >= 10)

 			self.assertRaises(
@@ -101,21 +101,21 @@ class SMTPActionTest(unittest.TestCase):
 		self.assertEqual(self.smtpd.rcpttos, ["root"])
 		subject = "Subject: [Fail2Ban] %s: banned %s" % (
 			self.jail.name, aInfo['ip'])
-		self.assertTrue(subject in self.smtpd.data.replace("\n", ""))
+		self.assertIn(subject, self.smtpd.data.replace("\n", ""))
 		self.assertTrue(
 			"%i attempts" % aInfo['failures'] in self.smtpd.data)

 		self.action.matches = "matches"
 		self.action.ban(aInfo)
-		self.assertTrue(aInfo['matches'] in self.smtpd.data)
+		self.assertIn(aInfo['matches'], self.smtpd.data)

 		self.action.matches = "ipjailmatches"
 		self.action.ban(aInfo)
-		self.assertTrue(aInfo['ipjailmatches'] in self.smtpd.data)
+		self.assertIn(aInfo['ipjailmatches'], self.smtpd.data)

 		self.action.matches = "ipmatches"
 		self.action.ban(aInfo)
-		self.assertTrue(aInfo['ipmatches'] in self.smtpd.data)
+		self.assertIn(aInfo['ipmatches'], self.smtpd.data)

 	def testOptions(self):
 		self.action.start()
@@ -65,12 +65,12 @@ class ExecuteActions(LogCaptureTestCase):
 	def testActionsManipulation(self):
 		self.__actions.add('test')
 		self.assertTrue(self.__actions['test'])
-		self.assertTrue('test' in self.__actions)
-		self.assertFalse('nonexistant action' in self.__actions)
+		self.assertIn('test', self.__actions)
+		self.assertNotIn('nonexistant action', self.__actions)
 		self.__actions.add('test1')
 		del self.__actions['test']
 		del self.__actions['test1']
-		self.assertFalse('test' in self.__actions)
+		self.assertNotIn('test', self.__actions)
 		self.assertEqual(len(self.__actions), 0)

 		self.__actions.setBanTime(127)
@@ -29,6 +29,7 @@ import time
 import tempfile

 from ..server.action import CommandAction, CallingMap
+from ..server.actions import OrderedDict

 from .utils import LogCaptureTestCase
 from .utils import pid_exists
@@ -58,6 +59,62 @@ class CommandActionTest(LogCaptureTestCase):
 		# Unresolveable substition
 		self.assertFalse(CommandAction.substituteRecursiveTags({'A': 'to=<B> fromip=<IP>', 'C': '<B>', 'B': '<C>', 'D': ''}))
 		self.assertFalse(CommandAction.substituteRecursiveTags({'failregex': 'to=<honeypot> fromip=<IP>', 'sweet': '<honeypot>', 'honeypot': '<sweet>', 'ignoreregex': ''}))
+		# We need here an ordered, because the sequence of iteration is very important for this test
+		if OrderedDict:
+			# No cyclic recursion, just multiple replacement of tag <T>, should be successful:
+			self.assertEqual(CommandAction.substituteRecursiveTags( OrderedDict(
+					(('X', 'x=x<T>'), ('T', '1'), ('Z', '<X> <T> <Y>'), ('Y', 'y=y<T>')))
+				), {'X': 'x=x1', 'T': '1', 'Y': 'y=y1', 'Z': 'x=x1 1 y=y1'}
+			)
+			# No cyclic recursion, just multiple replacement of tag <T> in composite tags, should be successful:
+			self.assertEqual(CommandAction.substituteRecursiveTags( OrderedDict(
+				  (('X', 'x=x<T> <Z> <<R1>> <<R2>>'), ('R1', 'Z'), ('R2', 'Y'), ('T', '1'), ('Z', '<T> <Y>'), ('Y', 'y=y<T>')))
+				), {'X': 'x=x1 1 y=y1 1 y=y1 y=y1', 'R1': 'Z', 'R2': 'Y', 'T': '1', 'Z': '1 y=y1', 'Y': 'y=y1'}
+			)
+			# No cyclic recursion, just multiple replacement of same tags, should be successful:
+			self.assertEqual(CommandAction.substituteRecursiveTags( OrderedDict((
+					('actionstart', 'ipset create <ipmset> hash:ip timeout <bantime> family <ipsetfamily>\n<iptables> -I <chain> <actiontype>'),
+					('ipmset', 'f2b-<name>'),
+					('name', 'any'),
+					('bantime', '600'),
+					('ipsetfamily', 'inet'),
+					('iptables', 'iptables <lockingopt>'),
+					('lockingopt', '-w'),
+					('chain', 'INPUT'),
+					('actiontype', '<multiport>'),
+					('multiport', '-p <protocol> -m multiport --dports <port> -m set --match-set <ipmset> src -j <blocktype>'),
+					('protocol', 'tcp'),
+					('port', 'ssh'),
+					('blocktype', 'REJECT',),
+				))
+				), OrderedDict((
+					('actionstart', 'ipset create f2b-any hash:ip timeout 600 family inet\niptables -w -I INPUT -p tcp -m multiport --dports ssh -m set --match-set f2b-any src -j REJECT'),
+					('ipmset', 'f2b-any'),
+					('name', 'any'),
+					('bantime', '600'),
+					('ipsetfamily', 'inet'),
+					('iptables', 'iptables -w'),
+					('lockingopt', '-w'),
+					('chain', 'INPUT'),
+					('actiontype', '-p tcp -m multiport --dports ssh -m set --match-set f2b-any src -j REJECT'),
+					('multiport', '-p tcp -m multiport --dports ssh -m set --match-set f2b-any src -j REJECT'),
+					('protocol', 'tcp'),
+					('port', 'ssh'),
+					('blocktype', 'REJECT')
+				))
+			)
+			# Cyclic recursion by composite tag creation, tags "create" another tag, that closes cycle:
+			self.assertFalse(CommandAction.substituteRecursiveTags( OrderedDict((
+					('A', '<<B><C>>'),
+					('B', 'D'), ('C', 'E'),
+					('DE', 'cycle <A>'),
+			)) ))
+			self.assertFalse(CommandAction.substituteRecursiveTags( OrderedDict((
+					('DE', 'cycle <A>'),
+					('A', '<<B><C>>'),
+					('B', 'D'), ('C', 'E'),
+			)) ))
+			
 		# missing tags are ok
 		self.assertEqual(CommandAction.substituteRecursiveTags({'A': '<C>'}), {'A': '<C>'})
 		self.assertEqual(CommandAction.substituteRecursiveTags({'A': '<C> <D> <X>','X':'fun'}), {'A': '<C> <D> fun', 'X':'fun'})
@@ -125,6 +182,7 @@ class CommandActionTest(LogCaptureTestCase):
 				CallingMap(matches=lambda: str(10))),
 			"09 10 11")

+	def testReplaceNoTag(self):
 		# As tag not present, therefore callable should not be called
 		# Will raise ValueError if it is
 		self.assertEqual(
@@ -106,13 +106,12 @@ class StatusExtendedCymruInfo(unittest.TestCase):
 						   "country": ["nxdomain"],
 						   "rir": ["nxdomain"]})

-		# even for private IPs ASNs defined
 		# Since it outputs for all active tickets we would get previous results
 		# and new ones
-		ticket = BanTicket("10.0.0.0", 1167606000.0)
+		ticket = BanTicket("8.0.0.0", 1167606000.0)
 		self.assertTrue(self.__banManager.addBanTicket(ticket))
 		cymru_info = self.__banManager.getBanListExtendedCymruInfo()
 		assert_dict_equal(cymru_info,
-						  {"asn": ["nxdomain", "4565",],
-						   "country": ["nxdomain", "unknown"],
-						   "rir": ["nxdomain", "other"]})
+						  {"asn": ["nxdomain", "3356",],
+						   "country": ["nxdomain", "US"],
+						   "rir": ["nxdomain", "arin"]})
@@ -36,7 +36,7 @@ from ..client.jailsreader import JailsReader
 from ..client.actionreader import ActionReader
 from ..client.configurator import Configurator
 from ..version import version
-from .utils import LogCaptureTestCase
+from .utils import LogCaptureTestCase, with_tmpdir

 TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "files")

@@ -94,9 +94,8 @@ option = %s
 		if not os.access(f, os.R_OK):
 			self.assertFalse(self.c.read('d'))	# should not be readable BUT present
 		else:
-			# SkipTest introduced only in 2.7 thus can't yet use generally
-			# raise unittest.SkipTest("Skipping on %s -- access rights are not enforced" % platform)
-			pass
+			import platform
+			raise unittest.SkipTest("Skipping on %s -- access rights are not enforced" % platform.platform())

 	def testOptionalDotDDir(self):
 		self.assertFalse(self.c.read('c'))	# nothing is there yet
@@ -281,8 +280,8 @@ class JailReaderTest(LogCaptureTestCase):
 		self.assertEqual(eval(act[2][5]).get('agent', '<wrong>'), useragent)
 		self.assertEqual(act[3], ['set', 'blocklisttest', 'action', 'mynetwatchman', 'agent', useragent])

-	def testGlob(self):
-		d = tempfile.mkdtemp(prefix="f2b-temp")
+	@with_tmpdir
+	def testGlob(self, d):
 		# Generate few files
 		# regular file
 		f1 = os.path.join(d, 'f1')
@@ -297,9 +296,6 @@ class JailReaderTest(LogCaptureTestCase):
 		self.assertEqual(JailReader._glob(f2), [])
 		self.assertLogged('File %s is a dangling link, thus cannot be monitored' % f2)
 		self.assertEqual(JailReader._glob(os.path.join(d, 'nonexisting')), [])
-		os.remove(f1)
-		os.remove(f2)
-		os.rmdir(d)

 		
 class FilterReaderTest(unittest.TestCase):
@@ -410,7 +406,7 @@ class FilterReaderTest(unittest.TestCase):
 			# from testcase01
 			filterReader.get('Definition', 'failregex')
 			filterReader.get('Definition', 'ignoreregex')
-		except Exception, e: # pragma: no cover - failed if reachable
+		except Exception as e: # pragma: no cover - failed if reachable
 			self.fail('unexpected options after readexplicit: %s' % (e))


@@ -433,10 +429,10 @@ class JailsReaderTestCache(LogCaptureTestCase):
 				cnt += 1
 		return cnt

-	def testTestJailConfCache(self):
+	@with_tmpdir
+	def testTestJailConfCache(self, basedir):
 		saved_ll = configparserinc.logLevel
 		configparserinc.logLevel = logging.DEBUG
-		basedir = tempfile.mkdtemp("fail2ban_conf")
 		try:
 			shutil.rmtree(basedir)
 			shutil.copytree(CONFIG_DIR, basedir)
@@ -468,7 +464,6 @@ class JailsReaderTestCache(LogCaptureTestCase):
 			cnt = self._getLoggedReadCount(r'action\.d/iptables-common\.conf')
 			self.assertTrue(cnt == 1, "Unexpected count by reading of action files, cnt = %s" % cnt)
 		finally:
-			shutil.rmtree(basedir)
 			configparserinc.logLevel = saved_ll


@@ -525,12 +520,12 @@ class JailsReaderTest(LogCaptureTestCase):
 				self.assertTrue(actionReader.read())
 				actionReader.getOptions({})	  # populate _opts
 				if not actionName.endswith('-common'):
-					self.assertTrue('Definition' in actionReader.sections(),
+					self.assertIn('Definition', actionReader.sections(),
 						msg="Action file %r is lacking [Definition] section" % actionConfig)
 					# all must have some actionban defined
 					self.assertTrue(actionReader._opts.get('actionban', '').strip(),
 						msg="Action file %r is lacking actionban" % actionConfig)
-				self.assertTrue('Init' in actionReader.sections(),
+				self.assertIn('Init', actionReader.sections(),
 						msg="Action file %r is lacking [Init] section" % actionConfig)

 		def testReadStockJailConf(self):
@@ -582,7 +577,7 @@ class JailsReaderTest(LogCaptureTestCase):
 					self.assertTrue(len(actName))
 					self.assertTrue(isinstance(actOpt, dict))
 					if actName == 'iptables-multiport':
-						self.assertTrue('port' in actOpt)
+						self.assertIn('port', actOpt)

 					actionReader = ActionReader(
 						actName, jail, {}, basedir=CONFIG_DIR)
@@ -602,7 +597,7 @@ class JailsReaderTest(LogCaptureTestCase):
 			# grab all filter names
 			filters = set(os.path.splitext(os.path.split(a)[1])[0]
 				for a in glob.glob(os.path.join('config', 'filter.d', '*.conf'))
-					if not a.endswith('common.conf'))
+					if not (a.endswith('common.conf') or a.endswith('-aggressive.conf')))
 			# get filters of all jails (filter names without options inside filter[...])
 			filters_jail = set(
 				JailReader.extractOptions(jail.options['filter'])[0] for jail in jails.jails
@@ -632,11 +627,13 @@ class JailsReaderTest(LogCaptureTestCase):

 			# and we know even some of them by heart
 			for j in ['sshd', 'recidive']:
-				# by default we have 'auto' backend ATM
-				self.assertTrue(['add', j, 'auto'] in comm_commands)
+				# by default we have 'auto' backend ATM, but some distributions can overwrite it, 
+				# (e.g. fedora default is 'systemd') therefore let check it without backend...
+				self.assertIn(['add', j], 
+					(cmd[:2] for cmd in comm_commands if len(cmd) == 3 and cmd[0] == 'add'))
 				# and warn on useDNS
-				self.assertTrue(['set', j, 'usedns', 'warn'] in comm_commands)
-				self.assertTrue(['start', j] in comm_commands)
+				self.assertIn(['set', j, 'usedns', 'warn'], comm_commands)
+				self.assertIn(['start', j], comm_commands)

 			# last commands should be the 'start' commands
 			self.assertEqual(comm_commands[-1][0], 'start')
@@ -655,7 +652,7 @@ class JailsReaderTest(LogCaptureTestCase):
 					action_name = action.getName()
 					if '<blocktype>' in str(commands):
 						# Verify that it is among cInfo
-						self.assertTrue('blocktype' in action._initOpts)
+						self.assertIn('blocktype', action._initOpts)
 						# Verify that we have a call to set it up
 						blocktype_present = False
 						target_command = ['set', jail_name, 'action', action_name, 'blocktype']
@@ -716,8 +713,8 @@ class JailsReaderTest(LogCaptureTestCase):
 			self.assertEqual(configurator._Configurator__jails.getBaseDir(), '/tmp')
 			self.assertEqual(configurator.getBaseDir(), CONFIG_DIR)

-	def testMultipleSameAction(self):
-		basedir = tempfile.mkdtemp("fail2ban_conf")
+	@with_tmpdir
+	def testMultipleSameAction(self, basedir):
 		os.mkdir(os.path.join(basedir, "filter.d"))
 		os.mkdir(os.path.join(basedir, "action.d"))
 		open(os.path.join(basedir, "action.d", "testaction1.conf"), 'w').close()
@@ -746,4 +743,33 @@ filter = testfilter1
 		# Python actions should not be passed `actname`
 		self.assertEqual(add_actions[-1][-1], "{}")

-		shutil.rmtree(basedir)
+	def testLogPathFileFilterBackend(self):
+		self.assertRaisesRegexp(ValueError, r"Have not found any log file for .* jail", 
+			self._testLogPath, backend='polling')
+
+	def testLogPathSystemdBackend(self):
+		try: # pragma: systemd no cover
+			from ..server.filtersystemd import FilterSystemd
+		except Exception, e: # pragma: no cover
+			raise unittest.SkipTest("systemd python interface not available")
+		self._testLogPath(backend='systemd')
+		self._testLogPath(backend='systemd[journalflags=2]')
+	
+	@with_tmpdir
+	def _testLogPath(self, basedir, backend):
+		jailfd = open(os.path.join(basedir, "jail.conf"), 'w')
+		jailfd.write("""
+[testjail1]
+enabled = true
+backend = %s
+logpath = %s/not/exist.log
+          /this/path/should/not/exist.log
+action = 
+filter = 
+failregex = test <HOST>
+""" % (backend, basedir))
+		jailfd.close()
+		jails = JailsReader(basedir=basedir)
+		self.assertTrue(jails.read())
+		self.assertTrue(jails.getOptions())
+		jails.convert()
@@ -0,0 +1,18 @@
+# Fail2Ban generic example resp. test filter 
+#
+# Author: Serg G. Brester (sebres)
+#
+
+[INCLUDES]
+
+# Read common prefixes. If any customizations available -- read them from
+# common.local.  common.conf is a symlink to the original common.conf and
+# should be copied (dereferenced) during installation
+before = ../../../../config/filter.d/common.conf
+
+[Definition]
+
+_daemon = test-demo
+
+failregex = ^%(__prefix_line)sF2B: failure from <HOST>$
+ignoreregex = 
@@ -48,12 +48,10 @@ class DatabaseTest(LogCaptureTestCase):
 	def setUp(self):
 		"""Call before every test case."""
 		super(DatabaseTest, self).setUp()
-		if Fail2BanDb is None and sys.version_info >= (2,7): # pragma: no cover
+		if Fail2BanDb is None: # pragma: no cover
 			raise unittest.SkipTest(
 				"Unable to import fail2ban database module as sqlite is not "
 				"available.")
-		elif Fail2BanDb is None:
-			return
 		_, self.dbFilename = tempfile.mkstemp(".db", "fail2ban_")
 		self.db = Fail2BanDb(self.dbFilename)

@@ -123,7 +121,7 @@ class DatabaseTest(LogCaptureTestCase):

 		self.db.addLog(self.jail, self.fileContainer)

-		self.assertTrue(filename in self.db.getLogPaths(self.jail))
+		self.assertIn(filename, self.db.getLogPaths(self.jail))
 		os.remove(filename)

 	def testUpdateLog(self):
@@ -318,6 +316,25 @@ class DatabaseTest(LogCaptureTestCase):
 		actions._Actions__checkBan()
 		self.assertLogged("ban ainfo %s, %s, %s, %s" % (True, True, True, True))

+	def testDelAndAddJail(self):
+		self.testAddJail() # Add jail
+		# Delete jail (just disabled it):
+		self.db.delJail(self.jail)
+		jails = self.db.getJailNames()
+		self.assertIn(len(jails) == 1 and self.jail.name, jails)
+		jails = self.db.getJailNames(enabled=False)
+		self.assertIn(len(jails) == 1 and self.jail.name, jails)
+		jails = self.db.getJailNames(enabled=True)
+		self.assertTrue(len(jails) == 0)
+		# Add it again - should just enable it:
+		self.db.addJail(self.jail)
+		jails = self.db.getJailNames()
+		self.assertIn(len(jails) == 1 and self.jail.name, jails)
+		jails = self.db.getJailNames(enabled=True)
+		self.assertIn(len(jails) == 1 and self.jail.name, jails)
+		jails = self.db.getJailNames(enabled=False)
+		self.assertTrue(len(jails) == 0)
+
 	def testPurge(self):
 		if Fail2BanDb is None: # pragma: no cover
 			return
@@ -39,7 +39,7 @@ except ImportError:

 from ..client import fail2banregex
 from ..client.fail2banregex import Fail2banRegex, get_opt_parser, output
-from .utils import LogCaptureTestCase, logSys
+from .utils import setUpMyTime, tearDownMyTime, LogCaptureTestCase, logSys
 from .utils import CONFIG_DIR


@@ -70,10 +70,12 @@ class Fail2banRegexTest(LogCaptureTestCase):
 	def setUp(self):
 		"""Call before every test case."""
 		LogCaptureTestCase.setUp(self)
+		setUpMyTime()

 	def tearDown(self):
 		"""Call after every test case."""
 		LogCaptureTestCase.tearDown(self)
+		tearDownMyTime()

 	def testWrongRE(self):
 		(opts, args, fail2banRegex) = _Fail2banRegex(
@@ -132,6 +134,15 @@ class Fail2banRegexTest(LogCaptureTestCase):
 		self.assertLogged('Dez 31 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 193.168.0.128')
 		self.assertLogged('Dec 31 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 87.142.124.10')

+	def testDirectRE_1raw(self):
+		(opts, args, fail2banRegex) = _Fail2banRegex(
+			"--print-all-matched", "--raw",
+			Fail2banRegexTest.FILENAME_01, 
+			Fail2banRegexTest.RE_00
+		)
+		self.assertTrue(fail2banRegex.start(opts, args))
+		self.assertLogged('Lines: 19 lines, 0 ignored, 16 matched, 3 missed')
+
 	def testDirectRE_2(self):
 		(opts, args, fail2banRegex) = _Fail2banRegex(
 			"--print-all-matched",
@@ -150,8 +161,8 @@ class Fail2banRegexTest(LogCaptureTestCase):
 		self.assertTrue(fail2banRegex.start(opts, args))
 		self.assertLogged('Lines: 13 lines, 0 ignored, 5 matched, 8 missed')

-		self.assertLogged('141.3.81.106  Fri Aug 14 11:53:59 2015')
-		self.assertLogged('141.3.81.106  Fri Aug 14 11:54:59 2015')
+		self.assertLogged('141.3.81.106  Sun Aug 14 11:53:59 2005')
+		self.assertLogged('141.3.81.106  Sun Aug 14 11:54:59 2005')

 	def testWronChar(self):
 		(opts, args, fail2banRegex) = _Fail2banRegex(
@@ -160,9 +171,8 @@ class Fail2banRegexTest(LogCaptureTestCase):
 		self.assertTrue(fail2banRegex.start(opts, args))
 		self.assertLogged('Lines: 4 lines, 0 ignored, 2 matched, 2 missed')

-		self.assertLogged('Error decoding line');
-		self.assertLogged('Continuing to process line ignoring invalid characters:', '2015-01-14 20:00:58 user ');
-		self.assertLogged('Continuing to process line ignoring invalid characters:', '2015-01-14 20:00:59 user ');
+		self.assertLogged('Error decoding line')
+		self.assertLogged('Continuing to process line ignoring invalid characters:')

 		self.assertLogged('Nov  8 00:16:12 main sshd[32548]: input_userauth_request: invalid user llinco')
 		self.assertLogged('Nov  8 00:16:12 main sshd[32547]: pam_succeed_if(sshd:auth): error retrieving information about user llinco')
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env fail2ban-python
 import requests

 try:
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env fail2ban-python
 import sys
 if sys.argv[1] == "10.0.0.1":
 	exit(0)
@@ -1,5 +1,5 @@
 # failJSON: { "time": "2013-12-23T13:12:31", "match": true , "host": "173.255.225.101" }
 [Mon Dec 23 13:12:31 2013] [error] [client 173.255.225.101] ModSecurity:  [file "/etc/httpd/modsecurity.d/activated_rules/modsecurity_crs_21_protocol_anomalies.conf"] [line "47"] [id "960015"] [rev "1"] [msg "Request Missing an Accept Header"] [severity "NOTICE"] [ver "OWASP_CRS/2.2.8"] [maturity "9"] [accuracy "9"] [tag "OWASP_CRS/PROTOCOL_VIOLATION/MISSING_HEADER_ACCEPT"] [tag "WASCTC/WASC-21"][tag "OWASP_TOP_10/A7"] [tag "PCI/6.5.10"] Access denied with code 403 (phase 2). Operator EQ matched 0 at REQUEST_HEADERS. [hostname "www.mysite.net"] [uri "/"] [unique_id "Urf@f12qgHIAACrFOlgAAABA"]

-# failJSON: { "time": "2013-12-28T09:18:05", "match": true , "host": "32.65.254.69" }
-[Sat Dec 28 09:18:05 2013] [error] [client 32.65.254.69] ModSecurity:  [file "/etc/httpd/modsecurity.d/10_asl_rules.conf"] [line "635"] [id "340069"] [rev "4"] [msg "Atomicorp.com UNSUPPORTED DELAYED Rules: Web vulnerability scanner"] [severity "CRITICAL"] Access denied with code 403 (phase 2). Pattern match "(?:nessus(?:_is_probing_you_|test)|^/w00tw00t\\\\.at\\\\.)" at REQUEST_URI. [hostname "192.81.249.191"] [uri "/w00tw00t.at.blackhats.romanian.anti-sec:)"] [unique_id "4Q6RdsBR@b4AAA65LRUAAAAA"] 
+# failJSON: { "time": "2013-12-28T09:18:05", "match": true , "host": "32.65.254.69", "desc": "additional entry (and exact one space)" }
+[Sat Dec 28 09:18:05 2013] [error] [client 32.65.254.69] ModSecurity: [file "/etc/httpd/modsecurity.d/10_asl_rules.conf"] [line "635"] [id "340069"] [rev "4"] [msg "Atomicorp.com UNSUPPORTED DELAYED Rules: Web vulnerability scanner"] [severity "CRITICAL"] Access denied with code 403 (phase 2). Pattern match "(?:nessus(?:_is_probing_you_|test)|^/w00tw00t\\\\.at\\\\.)" at REQUEST_URI. [hostname "192.81.249.191"] [uri "/w00tw00t.at.blackhats.romanian.anti-sec:)"] [unique_id "4Q6RdsBR@b4AAA65LRUAAAAA"] 
--- a/Show More
+++ b/Show More