suppress unneeded info (moved to debug level)

see #3186
Merge branch 'gh-actions--test-systemd' into 0.10 + several revisions cherry-picked from 0.11/master
2021-12-20 15:39:57 +01:00 · 2021-12-02 18:48:59 +01:00 · 2021-12-02 18:44:24 +01:00 · 2021-12-02 18:44:22 +01:00 · 2021-12-02 18:33:10 +01:00 · 2021-12-02 18:33:08 +01:00
137 changed files with 3136 additions and 1824 deletions
@@ -0,0 +1,98 @@
+name: CI
+
+# Controls when the action will run. Triggers the workflow on push or pull request
+# events but only for the master branch
+on:
+  push:
+    paths-ignore:
+    - 'doc/**'
+    - 'files/**'
+    - 'man/**'
+  pull_request:
+    paths-ignore:
+    - 'doc/**'
+    - 'files/**'
+    - 'man/**'
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+jobs:
+  # This workflow contains a single job called "build"
+  build:
+    # The type of runner that the job will run on
+    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, '3.10', '3.11.0-alpha.1', pypy2, pypy3]
+      fail-fast: false
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - uses: actions/checkout@v2
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Grant systemd-journal access
+        run: sudo usermod -a -G systemd-journal "$USER" || echo 'no systemd-journal access'
+
+      - name: Python version
+        run: |
+          F2B_PY=$(python -c "import sys; print(sys.version)")
+          echo "Python: ${{ matrix.python-version }} -- ${F2B_PY/$'\n'/ }"
+          F2B_PYV=$(echo "${F2B_PY}" | grep -oP '^\d+(?:\.\d+)')
+          F2B_PY=${F2B_PY:0:1}
+          echo "Set F2B_PY=$F2B_PY, F2B_PYV=$F2B_PYV"
+          echo "F2B_PY=$F2B_PY" >> $GITHUB_ENV
+          echo "F2B_PYV=$F2B_PYV" >> $GITHUB_ENV
+          # for GHA we need to monitor all journals, since it cannot be found using SYSTEM_ONLY(4):
+          echo "F2B_SYSTEMD_DEFAULT_FLAGS=0" >> $GITHUB_ENV
+
+      - name: Install dependencies
+        run: |
+          if [[ "$F2B_PY" = 3 ]]; then python -m pip install --upgrade pip || echo "can't upgrade pip"; fi
+          if [[ "$F2B_PY" = 3 ]] && ! command -v 2to3x -v 2to3 > /dev/null; then
+            #pip install 2to3
+            sudo apt-get -y install 2to3
+          fi
+          #sudo apt-get -y install python${F2B_PY/2/}-pyinotify || echo 'inotify not available'
+          python -m pip install pyinotify || echo 'inotify not available'
+          #sudo apt-get -y install python${F2B_PY/2/}-systemd || echo 'systemd not available'
+          sudo apt-get -y install libsystemd-dev || echo 'systemd dependencies seems to be unavailable'
+          python -m pip install systemd-python || echo 'systemd not available'
+
+      - name: Before scripts
+        run: |
+          cd "$GITHUB_WORKSPACE"
+          # Manually execute 2to3 for now
+          if [[ "$F2B_PY" = 3 ]]; then echo "2to3 ..." && ./fail2ban-2to3; fi
+          _debug() { echo -n "$1 "; err=$("${@:2}" 2>&1) && echo 'OK' || echo -e "FAIL\n$err"; }
+          # (debug) output current preferred encoding:
+          _debug 'Encodings:' python -c 'import locale, sys; from fail2ban.helpers import PREFER_ENC; print(PREFER_ENC, locale.getpreferredencoding(), (sys.stdout and sys.stdout.encoding))'
+          # (debug) backend availabilities:
+          echo 'Backends:'
+          _debug '- systemd:' python -c 'from fail2ban.server.filtersystemd import FilterSystemd'
+          #_debug '- systemd (root): ' sudo python -c 'from fail2ban.server.filtersystemd import FilterSystemd'
+          _debug '- pyinotify:' python -c 'from fail2ban.server.filterpyinotify import FilterPyinotify'
+
+      - name: Test suite
+        run: |
+          if [[ "$F2B_PY" = 2 ]]; then
+            python setup.py test
+          elif dpkg --compare-versions "$F2B_PYV" lt 3.10; then 
+            python bin/fail2ban-testcases --verbosity=2
+          else
+            echo "Skip systemd backend since systemd-python module must be fixed for python >= v.3.10 in GHA ..."
+            python bin/fail2ban-testcases --verbosity=2 -i "[sS]ystemd|[jJ]ournal"
+          fi
+
+      #- name: Test suite (debug some systemd tests only)
+         #run: python bin/fail2ban-testcases --verbosity=2 "[sS]ystemd|[jJ]ournal"
+         #run: python bin/fail2ban-testcases --verbosity=2 -l 5 "test_WrongChar"
+
+      - name: Build
+        run: python setup.py build
+
+      #- name: Test initd scripts
+      #  run: shellcheck -s bash -e SC1090,SC1091 files/debian-initd
@@ -10,22 +10,14 @@ dist: xenial
 matrix:
  fast_finish: true
  include:
-  - python: 2.6
-    dist: trusty   # required for Python 2.6
  - python: 2.7
-    dist: trusty   # required for packages like gamin
-    name: 2.7 (trusty)
-  - python: 2.7
-    name: 2.7 (xenial)
-  - python: pypy
-    dist: trusty
-  - python: 3.3
-    dist: trusty
+  #- python: pypy
  - python:  3.4
  - python:  3.5
  - python:  3.6
  - python:  3.7
-  - python:  3.8-dev
+  - python:  3.8
+  - python:  3.9-dev
  - python:  pypy3.5
 before_install:
  - echo "running under $TRAVIS_PYTHON_VERSION"
@@ -69,8 +61,8 @@ script:
  - if [[ "$F2B_PY" = 3 ]]; then coverage run bin/fail2ban-testcases --verbosity=2; fi
  # Use $VENV_BIN (not python) or else sudo will always run the system's python (2.7)
  - sudo $VENV_BIN/pip install .
-  # Doc files should get installed on Travis under Linux (python >= 3.8 seem to use another path segment)
-  - if [[ $TRAVIS_PYTHON_VERSION < 3.8 ]]; then test -e /usr/share/doc/fail2ban/FILTERS; fi
+  # Doc files should get installed on Travis under Linux (some builds/python's seem to use another path segment)
+  - test -e /usr/share/doc/fail2ban/FILTERS && echo 'found' || echo 'not found'
  # Test initd script
  - shellcheck -s bash -e SC1090,SC1091 files/debian-initd
 after_success:
@@ -6,9 +6,10 @@
 Fail2Ban: Changelog
 ===================

-Incompatibility list (compared to v.0.9):
+ver. 0.10.6 (2020/11/23) - just-what-the-doctor-ordered
 -----------

+### Incompatibility list (v.0.10 compared to v.0.9):
 * Filter (or `failregex`) internal capture-groups:

  - If you've your own `failregex` or custom filters using conditional match `(?P=host)`, you should
@@ -30,6 +31,67 @@ Incompatibility list (compared to v.0.9):
 * Since v0.10 fail2ban supports the matching of IPv6 addresses, but not all ban actions are
  IPv6-capable now.

+### Fixes
+* [stability] prevent race condition - no ban if filter (backend) is continuously busy if
+  too many messages will be found in log, e. g. initial scan of large log-file or journal (gh-2660)
+* pyinotify-backend sporadically avoided initial scanning of log-file by start
+* python 3.9 compatibility (and Travis CI support)
+* restoring a large number (500+ depending on files ulimit) of current bans when using PyPy fixed
+* manual ban is written to database, so can be restored by restart (gh-2647)
+* `jail.conf`: don't specify `action` directly in jails (use `action_` or `banaction` instead)
+* no mails-action added per default anymore (e. g. to allow that `action = %(action_mw)s` should be specified
+  per jail or in default section in jail.local), closes gh-2357
+* ensure we've unique action name per jail (also if parameter `actname` is not set but name deviates from standard name, gh-2686)
+* don't use `%(banaction)s` interpolation because it can be complex value (containing `[...]` and/or quotes), 
+  so would bother the action interpolation
+* fixed type conversion in config readers (take place after all interpolations get ready), that allows to 
+  specify typed parameters variable (as substitutions) as well as to supply it in other sections or as init parameters.
+* `action.d/*-ipset*.conf`: several ipset actions fixed (no timeout per default anymore), so no discrepancy
+  between ipset and fail2ban (removal from ipset will be managed by fail2ban only, gh-2703)
+* `action.d/cloudflare.conf`: fixed `actionunban` (considering new-line chars and optionally real json-parsing
+   with `jq`, gh-2140, gh-2656)
+* `action.d/nftables.conf` (type=multiport only): fixed port range selector, replacing `:` with `-` (gh-2763)
+* `action.d/firewallcmd-*.conf` (multiport only): fixed port range selector, replacing `:` with `-` (gh-2821)
+* `action.d/bsd-ipfw.conf`: fixed selection of rule-no by large list or initial `lowest_rule_num` (gh-2836)
+* `filter.d/common.conf`: avoid substitute of default values in related `lt_*` section, `__prefix_line`
+  should be interpolated in definition section (inside the filter-config, gh-2650)
+* `filter.d/dovecot.conf`: 
+  - add managesieve and submission support (gh-2795);
+  - accept messages with more verbose logging (gh-2573);
+* `filter.d/courier-smtp.conf`: prefregex extended to consider port in log-message (gh-2697)
+* `filter.d/traefik-auth.conf`: filter extended with parameter mode (`normal`, `ddos`, `aggressive`) to handle 
+  the match of username differently (gh-2693):
+  - `normal`: matches 401 with supplied username only
+  - `ddos`: matches 401 without supplied username only
+  - `aggressive`: matches 401 and any variant (with and without username)
+* `filter.d/sshd.conf`: normalizing of user pattern in all RE's, allowing empty user (gh-2749)
+
+### New Features and Enhancements
+* fail2ban-regex:
+  - speedup formatted output (bypass unneeded stats creation)
+  - extended with prefregex statistic
+  - more informative output for `datepattern` (e. g. set from filter) - pattern : description
+* parsing of action in jail-configs considers space between action-names as separator also
+  (previously only new-line was allowed), for example `action = a b` would specify 2 actions `a` and `b`
+* new filter and jail for GitLab recognizing failed application logins (gh-2689)
+* new filter and jail for Grafana recognizing failed application logins (gh-2855)
+* new filter and jail for SoftEtherVPN recognizing failed application logins (gh-2723)
+* `filter.d/guacamole.conf` extended with `logging` parameter to follow webapp-logging if it's configured (gh-2631)
+* `filter.d/bitwarden.conf` enhanced to support syslog (gh-2778)
+* introduced new prefix `{UNB}` for `datepattern` to disable word boundaries in regex;
+* datetemplate: improved anchor detection for capturing groups `(^...)`;
+* datepattern: improved handling with wrong recognized timestamps (timezones, no datepattern, etc)
+  as well as some warnings signaling user about invalid pattern or zone (gh-2814):
+  - filter gets mode in-operation, which gets activated if filter starts processing of new messages;
+    in this mode a timestamp read from log-line that appeared recently (not an old line), deviating too much
+    from now (up too 24h), will be considered as now (assuming a timezone issue), so could avoid unexpected 
+    bypass of failure (previously exceeding `findtime`);
+  - better interaction with non-matching optional datepattern or invalid timestamps;
+  - implements special datepattern `{NONE}` - allow to find failures totally without date-time in log messages,
+    whereas filter will use now as timestamp (gh-2802)
+* performance optimization of `datepattern` (better search algorithm in datedetector, especially for single template);
+* fail2ban-client: extended to unban IP range(s) by subnet (CIDR/mask) or hostname (DNS), gh-2791;
+

 ver. 0.10.5 (2020/01/10) - deserve-more-respect-a-jedis-weapon-must
 -----------
@@ -5,8 +5,6 @@ bin/fail2ban-testcases
 ChangeLog
 config/action.d/abuseipdb.conf
 config/action.d/apf.conf
-config/action.d/badips.conf
-config/action.d/badips.py
 config/action.d/blocklist_de.conf
 config/action.d/bsd-ipfw.conf
 config/action.d/cloudflare.conf
@@ -100,6 +98,8 @@ config/filter.d/exim.conf
 config/filter.d/exim-spam.conf
 config/filter.d/freeswitch.conf
 config/filter.d/froxlor-auth.conf
+config/filter.d/gitlab.conf
+config/filter.d/grafana.conf
 config/filter.d/groupoffice.conf
 config/filter.d/gssftpd.conf
 config/filter.d/guacamole.conf
@@ -139,6 +139,7 @@ config/filter.d/sendmail-auth.conf
 config/filter.d/sendmail-reject.conf
 config/filter.d/sieve.conf
 config/filter.d/slapd.conf
+config/filter.d/softethervpn.conf
 config/filter.d/sogo-auth.conf
 config/filter.d/solid-pop3d.conf
 config/filter.d/squid.conf
@@ -216,7 +217,6 @@ fail2ban/setup.py
 fail2ban-testcases-all
 fail2ban-testcases-all-python3
 fail2ban/tests/action_d/__init__.py
-fail2ban/tests/action_d/test_badips.py
 fail2ban/tests/action_d/test_smtp.py
 fail2ban/tests/actionstestcase.py
 fail2ban/tests/actiontestcase.py
@@ -226,6 +226,8 @@ fail2ban/tests/clientreadertestcase.py
 fail2ban/tests/config/action.d/action.conf
 fail2ban/tests/config/action.d/brokenaction.conf
 fail2ban/tests/config/fail2ban.conf
+fail2ban/tests/config/filter.d/checklogtype.conf
+fail2ban/tests/config/filter.d/checklogtype_test.conf
 fail2ban/tests/config/filter.d/simple.conf
 fail2ban/tests/config/filter.d/test.conf
 fail2ban/tests/config/filter.d/test.local
@@ -263,6 +265,8 @@ fail2ban/tests/files/config/apache-auth/README
 fail2ban/tests/files/database_v1.db
 fail2ban/tests/files/filter.d/substition.conf
 fail2ban/tests/files/filter.d/testcase01.conf
+fail2ban/tests/files/filter.d/testcase02.conf
+fail2ban/tests/files/filter.d/testcase02.local
 fail2ban/tests/files/filter.d/testcase-common.conf
 fail2ban/tests/files/ignorecommand.py
 fail2ban/tests/files/logs/3proxy
@@ -297,6 +301,8 @@ fail2ban/tests/files/logs/exim
 fail2ban/tests/files/logs/exim-spam
 fail2ban/tests/files/logs/freeswitch
 fail2ban/tests/files/logs/froxlor-auth
+fail2ban/tests/files/logs/gitlab
+fail2ban/tests/files/logs/grafana
 fail2ban/tests/files/logs/groupoffice
 fail2ban/tests/files/logs/gssftpd
 fail2ban/tests/files/logs/guacamole
@@ -334,6 +340,7 @@ fail2ban/tests/files/logs/sendmail-auth
 fail2ban/tests/files/logs/sendmail-reject
 fail2ban/tests/files/logs/sieve
 fail2ban/tests/files/logs/slapd
+fail2ban/tests/files/logs/softethervpn
 fail2ban/tests/files/logs/sogo-auth
 fail2ban/tests/files/logs/solid-pop3d
 fail2ban/tests/files/logs/squid
@@ -21,14 +21,13 @@
 #
 # Example, for ssh bruteforce (in section [sshd] of `jail.local`): 
 #   action = %(known/action)s
-#            %(action_abuseipdb)s[abuseipdb_apikey="my-api-key", abuseipdb_category="18,22"]
+#            abuseipdb[abuseipdb_apikey="my-api-key", abuseipdb_category="18,22"]
 #
-# See below for catagories.
+# See below for categories.
 #
-# Original Ref: https://wiki.shaunc.com/wikka.php?wakka=ReportingToAbuseIPDBWithFail2Ban
 # Added to fail2ban by Andrew James Collett (ajcollett)

-## abuseIPDB Catagories, `the abuseipdb_category` MUST be set in the jail.conf action call.
+## abuseIPDB Categories, `the abuseipdb_category` MUST be set in the jail.conf action call.
 # Example, for ssh bruteforce: action = %(action_abuseipdb)s[abuseipdb_category="18,22"]
 # ID	Title	Description
 # 3	  Fraud Orders
@@ -101,5 +100,5 @@ actionunban =
 # Notes    Your API key from abuseipdb.com
 # Values:  STRING  Default: None
 # Register for abuseipdb [https://www.abuseipdb.com], get api key and set below.
-# You will need to set the catagory in the action call.
+# You will need to set the category in the action call.
 abuseipdb_apikey =
@@ -1,19 +0,0 @@
-# Fail2ban reporting to badips.com
-#
-# Note: This reports an IP only and does not actually ban traffic. Use
-# another action in the same jail if you want bans to occur.
-#
-# Set the category to the appropriate value before use.
-#
-# To get see register and optional key to get personalised graphs see:
-# http://www.badips.com/blog/personalized-statistics-track-the-attackers-of-all-your-servers-with-one-key
-
-[Definition]
-
-actionban = curl --fail  --user-agent "<agent>" http://www.badips.com/add/<category>/<ip>
-
-[Init]
-
-# Option: category
-# Notes.: Values are from the list here: http://www.badips.com/get/categories
-category = 
@@ -1,391 +0,0 @@
-# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*-
-# vi: set ft=python sts=4 ts=4 sw=4 noet :
-
-# This file is part of Fail2Ban.
-#
-# Fail2Ban is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# Fail2Ban is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Fail2Ban; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
-
-import sys
-if sys.version_info < (2, 7): # pragma: no cover
-	raise ImportError("badips.py action requires Python >= 2.7")
-import json
-import threading
-import logging
-if sys.version_info >= (3, ): # pragma: 2.x no cover
-	from urllib.request import Request, urlopen
-	from urllib.parse import urlencode
-	from urllib.error import HTTPError
-else: # pragma: 3.x no cover
-	from urllib2 import Request, urlopen, HTTPError
-	from urllib import urlencode
-
-from fail2ban.server.actions import Actions, ActionBase, BanTicket
-from fail2ban.helpers import splitwords, str2LogLevel
-
-
-
-class BadIPsAction(ActionBase): # pragma: no cover - may be unavailable
-	"""Fail2Ban action which reports bans to badips.com, and also
-	blacklist bad IPs listed on badips.com by using another action's
-	ban method.
-
-	Parameters
-	----------
-	jail : Jail
-		The jail which the action belongs to.
-	name : str
-		Name assigned to the action.
-	category : str
-		Valid badips.com category for reporting failures.
-	score : int, optional
-		Minimum score for bad IPs. Default 3.
-	age : str, optional
-		Age of last report for bad IPs, per badips.com syntax.
-		Default "24h" (24 hours)
-	banaction : str, optional
-		Name of banaction to use for blacklisting bad IPs. If `None`,
-		no blacklist of IPs will take place.
-		Default `None`.
-	bancategory : str, optional
-		Name of category to use for blacklisting, which can differ
-		from category used for reporting. e.g. may want to report
-		"postfix", but want to use whole "mail" category for blacklist.
-		Default `category`.
-	bankey : str, optional
-		Key issued by badips.com to retrieve personal list
-		of blacklist IPs.
-	updateperiod : int, optional
-		Time in seconds between updating bad IPs blacklist.
-		Default 900 (15 minutes)
-	loglevel : int/str, optional
-		Log level of the message when an IP is (un)banned.
-		Default `DEBUG`.
-		Can be also supplied as two-value list (comma- or space separated) to
-		provide level of the summary message when a group of IPs is (un)banned.
-		Example `DEBUG,INFO`.
-	agent : str, optional
-		User agent transmitted to server.
-		Default `Fail2Ban/ver.`
-
-	Raises
-	------
-	ValueError
-		If invalid `category`, `score`, `banaction` or `updateperiod`.
-	"""
-
-	TIMEOUT = 10
-	_badips = "https://www.badips.com"
-	def _Request(self, url, **argv):
-		return Request(url, headers={'User-Agent': self.agent}, **argv)
-
-	def __init__(self, jail, name, category, score=3, age="24h",
-		banaction=None, bancategory=None, bankey=None, updateperiod=900, 
-		loglevel='DEBUG', agent="Fail2Ban", timeout=TIMEOUT):
-		super(BadIPsAction, self).__init__(jail, name)
-
-		self.timeout = timeout
-		self.agent = agent
-		self.category = category
-		self.score = score
-		self.age = age
-		self.banaction = banaction
-		self.bancategory = bancategory or category
-		self.bankey = bankey
-		loglevel = splitwords(loglevel)
-		self.sumloglevel = str2LogLevel(loglevel[-1])
-		self.loglevel = str2LogLevel(loglevel[0])
-		self.updateperiod = updateperiod
-
-		self._bannedips = set()
-		# Used later for threading.Timer for updating badips
-		self._timer = None
-
-	@staticmethod
-	def isAvailable(timeout=1):
-		try:
-			response = urlopen(Request("/".join([BadIPsAction._badips]),
-					headers={'User-Agent': "Fail2Ban"}), timeout=timeout)
-			return True, ''
-		except Exception as e: # pragma: no cover
-			return False, e
-
-	def logError(self, response, what=''): # pragma: no cover - sporadical (502: Bad Gateway, etc)
-		messages = {}
-		try:
-			messages = json.loads(response.read().decode('utf-8'))
-		except:
-			pass
-		self._logSys.error(
-			"%s. badips.com response: '%s'", what,
-				messages.get('err', 'Unknown'))
-
-	def getCategories(self, incParents=False):
-		"""Get badips.com categories.
-
-		Returns
-		-------
-		set
-			Set of categories.
-
-		Raises
-		------
-		HTTPError
-			Any issues with badips.com request.
-		ValueError
-			If badips.com response didn't contain necessary information
-		"""
-		try:
-			response = urlopen(
-				self._Request("/".join([self._badips, "get", "categories"])), timeout=self.timeout)
-		except HTTPError as response: # pragma: no cover
-			self.logError(response, "Failed to fetch categories")
-			raise
-		else:
-			response_json = json.loads(response.read().decode('utf-8'))
-			if not 'categories' in response_json:
-				err = "badips.com response lacked categories specification. Response was: %s" \
-				  % (response_json,)
-				self._logSys.error(err)
-				raise ValueError(err)
-			categories = response_json['categories']
-			categories_names = set(
-				value['Name'] for value in categories)
-			if incParents:
-				categories_names.update(set(
-					value['Parent'] for value in categories
-					if "Parent" in value))
-			return categories_names
-
-	def getList(self, category, score, age, key=None):
-		"""Get badips.com list of bad IPs.
-
-		Parameters
-		----------
-		category : str
-			Valid badips.com category.
-		score : int
-			Minimum score for bad IPs.
-		age : str
-			Age of last report for bad IPs, per badips.com syntax.
-		key : str, optional
-			Key issued by badips.com to fetch IPs reported with the
-			associated key.
-
-		Returns
-		-------
-		set
-			Set of bad IPs.
-
-		Raises
-		------
-		HTTPError
-			Any issues with badips.com request.
-		"""
-		try:
-			url = "?".join([
-				"/".join([self._badips, "get", "list", category, str(score)]),
-				urlencode({'age': age})])
-			if key:
-				url = "&".join([url, urlencode({'key': key})])
-			self._logSys.debug('badips.com: get list, url: %r', url)
-			response = urlopen(self._Request(url), timeout=self.timeout)
-		except HTTPError as response: # pragma: no cover
-			self.logError(response, "Failed to fetch bad IP list")
-			raise
-		else:
-			return set(response.read().decode('utf-8').split())
-
-	@property
-	def category(self):
-		"""badips.com category for reporting IPs.
-		"""
-		return self._category
-
-	@category.setter
-	def category(self, category):
-		if category not in self.getCategories():
-			self._logSys.error("Category name '%s' not valid. "
-				"see badips.com for list of valid categories",
-				category)
-			raise ValueError("Invalid category: %s" % category)
-		self._category = category
-
-	@property
-	def bancategory(self):
-		"""badips.com bancategory for fetching IPs.
-		"""
-		return self._bancategory
-
-	@bancategory.setter
-	def bancategory(self, bancategory):
-		if bancategory != "any" and bancategory not in self.getCategories(incParents=True):
-			self._logSys.error("Category name '%s' not valid. "
-				"see badips.com for list of valid categories",
-				bancategory)
-			raise ValueError("Invalid bancategory: %s" % bancategory)
-		self._bancategory = bancategory
-
-	@property
-	def score(self):
-		"""badips.com minimum score for fetching IPs.
-		"""
-		return self._score
-
-	@score.setter
-	def score(self, score):
-		score = int(score)
-		if 0 <= score <= 5:
-			self._score = score
-		else:
-			raise ValueError("Score must be 0-5")
-
-	@property
-	def banaction(self):
-		"""Jail action to use for banning/unbanning.
-		"""
-		return self._banaction
-
-	@banaction.setter
-	def banaction(self, banaction):
-		if banaction is not None and banaction not in self._jail.actions:
-			self._logSys.error("Action name '%s' not in jail '%s'",
-				banaction, self._jail.name)
-			raise ValueError("Invalid banaction")
-		self._banaction = banaction
-
-	@property
-	def updateperiod(self):
-		"""Period in seconds between banned bad IPs will be updated.
-		"""
-		return self._updateperiod
-
-	@updateperiod.setter
-	def updateperiod(self, updateperiod):
-		updateperiod = int(updateperiod)
-		if updateperiod > 0:
-			self._updateperiod = updateperiod
-		else:
-			raise ValueError("Update period must be integer greater than 0")
-
-	def _banIPs(self, ips):
-		for ip in ips:
-			try:
-				ai = Actions.ActionInfo(BanTicket(ip), self._jail)
-				self._jail.actions[self.banaction].ban(ai)
-			except Exception as e:
-				self._logSys.error(
-					"Error banning IP %s for jail '%s' with action '%s': %s",
-					ip, self._jail.name, self.banaction, e,
-					exc_info=self._logSys.getEffectiveLevel()<=logging.DEBUG)
-			else:
-				self._bannedips.add(ip)
-				self._logSys.log(self.loglevel,
-					"Banned IP %s for jail '%s' with action '%s'",
-					ip, self._jail.name, self.banaction)
-
-	def _unbanIPs(self, ips):
-		for ip in ips:
-			try:
-				ai = Actions.ActionInfo(BanTicket(ip), self._jail)
-				self._jail.actions[self.banaction].unban(ai)
-			except Exception as e:
-				self._logSys.error(
-					"Error unbanning IP %s for jail '%s' with action '%s': %s",
-					ip, self._jail.name, self.banaction, e,
-					exc_info=self._logSys.getEffectiveLevel()<=logging.DEBUG)
-			else:
-				self._logSys.log(self.loglevel,
-					"Unbanned IP %s for jail '%s' with action '%s'",
-					ip, self._jail.name, self.banaction)
-			finally:
-				self._bannedips.remove(ip)
-
-	def start(self):
-		"""If `banaction` set, blacklists bad IPs.
-		"""
-		if self.banaction is not None:
-			self.update()
-
-	def update(self):
-		"""If `banaction` set, updates blacklisted IPs.
-
-		Queries badips.com for list of bad IPs, removing IPs from the
-		blacklist if no longer present, and adds new bad IPs to the
-		blacklist.
-		"""
-		if self.banaction is not None:
-			if self._timer:
-				self._timer.cancel()
-				self._timer = None
-
-			try:
-				ips = self.getList(
-					self.bancategory, self.score, self.age, self.bankey)
-				# Remove old IPs no longer listed
-				s = self._bannedips - ips
-				m = len(s)
-				self._unbanIPs(s)
-				# Add new IPs which are now listed
-				s = ips - self._bannedips
-				p = len(s)
-				self._banIPs(s)
-				if m != 0 or p != 0:
-					self._logSys.log(self.sumloglevel,
-						"Updated IPs for jail '%s' (-%d/+%d)",
-						self._jail.name, m, p)
-				self._logSys.debug(
-					"Next update for jail '%' in %i seconds",
-					self._jail.name, self.updateperiod)
-			finally:
-				self._timer = threading.Timer(self.updateperiod, self.update)
-				self._timer.start()
-
-	def stop(self):
-		"""If `banaction` set, clears blacklisted IPs.
-		"""
-		if self.banaction is not None:
-			if self._timer:
-				self._timer.cancel()
-				self._timer = None
-			self._unbanIPs(self._bannedips.copy())
-
-	def ban(self, aInfo):
-		"""Reports banned IP to badips.com.
-
-		Parameters
-		----------
-		aInfo : dict
-			Dictionary which includes information in relation to
-			the ban.
-
-		Raises
-		------
-		HTTPError
-			Any issues with badips.com request.
-		"""
-		try:
-			url = "/".join([self._badips, "add", self.category, str(aInfo['ip'])])
-			self._logSys.debug('badips.com: ban, url: %r', url)
-			response = urlopen(self._Request(url), timeout=self.timeout)
-		except HTTPError as response: # pragma: no cover
-			self.logError(response, "Failed to ban")
-			raise
-		else:
-			messages = json.loads(response.read().decode('utf-8'))
-			self._logSys.debug(
-				"Response from badips.com report: '%s'",
-				messages['suc'])
-
-Action = BadIPsAction
@@ -14,7 +14,10 @@
 # Notes.:  command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false).
 # Values:  CMD
 #
-actionstart = ipfw show | fgrep -c -m 1 -s 'table(<table>)' > /dev/null 2>&1 || ( ipfw show | awk 'BEGIN { b = <lowest_rule_num> } { if ($1 < b) {} else if ($1 == b) { b = $1 + 1 } else { e = b } } END { if (e) exit e <br> else exit b }'; num=$?; ipfw -q add $num <blocktype> <block> from table\(<table>\) to me <port>; echo $num > "<startstatefile>" )
+actionstart = ipfw show | fgrep -c -m 1 -s 'table(<table>)' > /dev/null 2>&1 || (
+    num=$(ipfw show | awk 'BEGIN { b = <lowest_rule_num> } { if ($1 == b) { b = $1 + 1 } } END { print b }');
+    ipfw -q add "$num" <blocktype> <block> from table\(<table>\) to me <port>; echo "$num" > "<startstatefile>"
+  )


 # Option:  actionstop
@@ -5,7 +5,7 @@
 #
 # Please set jail.local's permission to 640 because it contains your CF API key.
 #
-# This action depends on curl.
+# This action depends on curl (and optionally jq).
 # Referenced from http://www.normyee.net/blog/2012/02/02/adding-cloudflare-support-to-fail2ban by NORM YEE
 #
 # To get your CloudFlare API Key: https://www.cloudflare.com/a/account/my-account
@@ -43,9 +43,9 @@ actioncheck =
 # API v1
 #actionban = curl -s -o /dev/null https://www.cloudflare.com/api_json.html -d 'a=ban' -d 'tkn=<cftoken>' -d 'email=<cfuser>' -d 'key=<ip>'
 # API v4
-actionban = curl -s -o /dev/null -X POST -H 'X-Auth-Email: <cfuser>' -H 'X-Auth-Key: <cftoken>' \
-            -H 'Content-Type: application/json' -d '{ "mode": "block", "configuration": { "target": "ip", "value": "<ip>" } }' \
-            https://api.cloudflare.com/client/v4/user/firewall/access_rules/rules
+actionban = curl -s -o /dev/null -X POST <_cf_api_prms> \
+            -d '{"mode":"block","configuration":{"target":"<cftarget>","value":"<ip>"},"notes":"Fail2Ban <name>"}' \
+            <_cf_api_url>

 # Option:  actionunban
 # Notes.:  command executed when unbanning an IP. Take care that the
@@ -58,9 +58,14 @@ actionban = curl -s -o /dev/null -X POST -H 'X-Auth-Email: <cfuser>' -H 'X-Auth-
 # API v1
 #actionunban = curl -s -o /dev/null https://www.cloudflare.com/api_json.html -d 'a=nul' -d 'tkn=<cftoken>' -d 'email=<cfuser>' -d 'key=<ip>'
 # API v4
-actionunban = curl -s -o /dev/null -X DELETE -H 'X-Auth-Email: <cfuser>' -H 'X-Auth-Key: <cftoken>' \
-              https://api.cloudflare.com/client/v4/user/firewall/access_rules/rules/$(curl -s -X GET -H 'X-Auth-Email: <cfuser>' -H 'X-Auth-Key: <cftoken>' \
-              'https://api.cloudflare.com/client/v4/user/firewall/access_rules/rules?mode=block&configuration_target=ip&configuration_value=<ip>&page=1&per_page=1' | cut -d'"' -f6)
+actionunban = id=$(curl -s -X GET <_cf_api_prms> \
+                   "<_cf_api_url>?mode=block&configuration_target=<cftarget>&configuration_value=<ip>&page=1&per_page=1&notes=Fail2Ban%%20<name>" \
+                   | { jq -r '.result[0].id' 2>/dev/null || tr -d '\n' | sed -nE 's/^.*"result"\s*:\s*\[\s*\{\s*"id"\s*:\s*"([^"]+)".*$/\1/p'; })
+              if [ -z "$id" ]; then echo "<name>: id for <ip> cannot be found"; exit 0; fi;
+              curl -s -o /dev/null -X DELETE <_cf_api_prms> "<_cf_api_url>/$id"
+
+_cf_api_url = https://api.cloudflare.com/client/v4/user/firewall/access_rules/rules
+_cf_api_prms = -H 'X-Auth-Email: <cfuser>' -H 'X-Auth-Key: <cftoken>' -H 'Content-Type: application/json'

 [Init]

@@ -76,3 +81,8 @@ actionunban = curl -s -o /dev/null -X DELETE -H 'X-Auth-Email: <cfuser>' -H 'X-A
 cftoken =

 cfuser =
+
+cftarget = ip
+
+[Init?family=inet6]
+cftarget = ip6
@@ -102,7 +102,7 @@ logpath = /dev/null
 # Notes.:  Your system mail command. Is passed 2 args: subject and recipient
 # Values:  CMD
 #
-mailcmd = mail -s
+mailcmd = mail -E 'set escape' -s

 # Option:  mailargs
 # Notes.:  Additional arguments to mail command. e.g. for standard Unix mail:
@@ -179,7 +179,7 @@ tcpflags =
 # Notes.:  Your system mail command. Is passed 2 args: subject and recipient
 # Values:  CMD
 #
-mailcmd = mail -s
+mailcmd = mail -E 'set escape' -s

 # Option:  mailargs
 # Notes.:  Additional arguments to mail command. e.g. for standard Unix mail:
@@ -18,7 +18,7 @@ before = firewallcmd-common.conf

 [Definition]

-actionstart = ipset create <ipmset> hash:ip timeout <bantime><familyopt>
+actionstart = ipset create <ipmset> hash:ip timeout <default-ipsettime> <familyopt>
              firewall-cmd --direct --add-rule <family> filter <chain> 0 <actiontype> -m set --match-set <ipmset> src -j <blocktype>

 actionflush = ipset flush <ipmset>
@@ -27,7 +27,7 @@ actionstop = firewall-cmd --direct --remove-rule <family> filter <chain> 0 <acti
             <actionflush>
             ipset destroy <ipmset>

-actionban = ipset add <ipmset> <ip> timeout <bantime> -exist
+actionban = ipset add <ipmset> <ip> timeout <ipsettime> -exist

 actionunban = ipset del <ipmset> <ip> -exist

@@ -40,11 +40,19 @@ actionunban = ipset del <ipmset> <ip> -exist
 #
 chain = INPUT_direct

-# Option: bantime
-# Notes:  specifies the bantime in seconds (handled internally rather than by fail2ban)
-# Values:  [ NUM ]  Default: 600
+# Option: default-ipsettime
+# Notes:  specifies default timeout in seconds (handled default ipset timeout only)
+# Values:  [ NUM ]  Default: 0 (no timeout, managed by fail2ban by unban)
+default-ipsettime = 0

-bantime = 600
+# Option: ipsettime
+# Notes:  specifies ticket timeout (handled ipset timeout only)
+# Values:  [ NUM ]  Default: 0 (managed by fail2ban by unban)
+ipsettime = 0
+
+# expresion to caclulate timeout from bantime, example:
+# banaction = %(known/banaction)s[ipsettime='<timeout-bantime>']
+timeout-bantime = $([ "<bantime>" -le 2147483 ] && echo "<bantime>" || echo 0)

 # Option: actiontype
 # Notes.: defines additions to the blocking rule
@@ -69,7 +77,7 @@ familyopt =
 [Init?family=inet6]

 ipmset = f2b-<name>6
-familyopt = <sp>family inet6
+familyopt = family inet6


 # DEV NOTES:
@@ -1,6 +1,6 @@
 # Fail2Ban configuration file
 #
-# Author: Donald Yandt 
+# Authors: Donald Yandt, Sergey G. Brester
 # 
 # Because of the rich rule commands requires firewalld-0.3.1+
 # This action uses firewalld rich-rules which gives you a cleaner iptables since it stores rules according to zones and not
@@ -10,36 +10,15 @@
 #
 # If you use the --permanent rule you get a xml file in /etc/firewalld/zones/<zone>.xml that can be shared and parsed easliy
 #
-# Example commands to view rules:
-# firewall-cmd [--zone=<zone>] --list-rich-rules
-# firewall-cmd [--zone=<zone>] --list-all
-# firewall-cmd [--zone=zone] --query-rich-rule='rule'
+# This is an derivative of firewallcmd-rich-rules.conf, see there for details and other parameters.

 [INCLUDES]

-before = firewallcmd-common.conf
+before = firewallcmd-rich-rules.conf

 [Definition]

-actionstart = 
-
-actionstop = 
-
-actioncheck = 
-
-# you can also use zones and/or service names. 
-#
-# zone example: 
-# firewall-cmd --zone=<zone> --add-rich-rule="rule family='<family>' source address='<ip>' port port='<port>' protocol='<protocol>' log prefix='f2b-<name>' level='<level>' limit value='<rate>/m' <rich-blocktype>"
-#
-# service name example:
-# firewall-cmd --zone=<zone> --add-rich-rule="rule family='<family>' source address='<ip>' service name='<service>' log prefix='f2b-<name>' level='<level>' limit value='<rate>/m' <rich-blocktype>"
-#
-# Because rich rules can only handle single or a range of ports we must split ports and execute the command for each port. Ports can be single and ranges separated by a comma or space for an example: http, https, 22-60, 18 smtp 
-
-actionban = ports="<port>"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --add-rich-rule="rule family='<family>' source address='<ip>' port port='$p' protocol='<protocol>' log prefix='f2b-<name>' level='<level>' limit value='<rate>/m' <rich-blocktype>"; done
-	   
-actionunban = ports="<port>"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --remove-rich-rule="rule family='<family>' source address='<ip>' port port='$p' protocol='<protocol>' log prefix='f2b-<name>' level='<level>' limit value='<rate>/m' <rich-blocktype>"; done
+rich-suffix = log prefix='f2b-<name>' level='<level>' limit value='<rate>/m' <rich-blocktype>

 [Init]

@@ -48,4 +27,3 @@ level = info

 # log rate per minute
 rate = 1
-
@@ -35,8 +35,10 @@ actioncheck =
 #
 # Because rich rules can only handle single or a range of ports we must split ports and execute the command for each port. Ports can be single and ranges separated by a comma or space for an example: http, https, 22-60, 18 smtp 

-actionban = ports="<port>"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --add-rich-rule="rule family='<family>' source address='<ip>' port port='$p' protocol='<protocol>' <rich-blocktype>"; done
+fwcmd_rich_rule = rule family='<family>' source address='<ip>' port port='$p' protocol='<protocol>' %(rich-suffix)s
+
+actionban = ports="<port>"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --add-rich-rule="%(fwcmd_rich_rule)s"; done
 	   
-actionunban = ports="<port>"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --remove-rich-rule="rule family='<family>' source address='<ip>' port port='$p' protocol='<protocol>' <rich-blocktype>"; done
-
+actionunban = ports="<port>"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --remove-rich-rule="%(fwcmd_rich_rule)s"; done

+rich-suffix = <rich-blocktype>
@@ -26,7 +26,7 @@ before = iptables-common.conf
 # Notes.:  command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false).
 # Values:  CMD
 #
-actionstart = ipset create <ipmset> hash:ip timeout <bantime><familyopt>
+actionstart = ipset create <ipmset> hash:ip timeout <default-ipsettime> <familyopt>
              <iptables> -I <chain> -m set --match-set <ipmset> src -j <blocktype>

 # Option:  actionflush
@@ -49,7 +49,7 @@ actionstop = <iptables> -D <chain> -m set --match-set <ipmset> src -j <blocktype
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = ipset add <ipmset> <ip> timeout <bantime> -exist
+actionban = ipset add <ipmset> <ip> timeout <ipsettime> -exist

 # Option:  actionunban
 # Notes.:  command executed when unbanning an IP. Take care that the
@@ -61,11 +61,19 @@ actionunban = ipset del <ipmset> <ip> -exist

 [Init]

-# Option: bantime
-# Notes:  specifies the bantime in seconds (handled internally rather than by fail2ban)
-# Values:  [ NUM ]  Default: 600
-#
-bantime = 600
+# Option: default-ipsettime
+# Notes:  specifies default timeout in seconds (handled default ipset timeout only)
+# Values:  [ NUM ]  Default: 0 (no timeout, managed by fail2ban by unban)
+default-ipsettime = 0
+
+# Option: ipsettime
+# Notes:  specifies ticket timeout (handled ipset timeout only)
+# Values:  [ NUM ]  Default: 0 (managed by fail2ban by unban)
+ipsettime = 0
+
+# expresion to caclulate timeout from bantime, example:
+# banaction = %(known/banaction)s[ipsettime='<timeout-bantime>']
+timeout-bantime = $([ "<bantime>" -le 2147483 ] && echo "<bantime>" || echo 0)

 ipmset = f2b-<name>
 familyopt =
@@ -74,4 +82,4 @@ familyopt =
 [Init?family=inet6]

 ipmset = f2b-<name>6
-familyopt = <sp>family inet6
+familyopt = family inet6
@@ -26,7 +26,7 @@ before = iptables-common.conf
 # Notes.:  command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false).
 # Values:  CMD
 #
-actionstart = ipset create <ipmset> hash:ip timeout <bantime><familyopt>
+actionstart = ipset create <ipmset> hash:ip timeout <default-ipsettime> <familyopt>
              <iptables> -I <chain> -p <protocol> -m multiport --dports <port> -m set --match-set <ipmset> src -j <blocktype>

 # Option:  actionflush
@@ -49,7 +49,7 @@ actionstop = <iptables> -D <chain> -p <protocol> -m multiport --dports <port> -m
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = ipset add <ipmset> <ip> timeout <bantime> -exist
+actionban = ipset add <ipmset> <ip> timeout <ipsettime> -exist

 # Option:  actionunban
 # Notes.:  command executed when unbanning an IP. Take care that the
@@ -61,11 +61,19 @@ actionunban = ipset del <ipmset> <ip> -exist

 [Init]

-# Option: bantime
-# Notes:  specifies the bantime in seconds (handled internally rather than by fail2ban)
-# Values:  [ NUM ]  Default: 600
-#
-bantime = 600
+# Option: default-ipsettime
+# Notes:  specifies default timeout in seconds (handled default ipset timeout only)
+# Values:  [ NUM ]  Default: 0 (no timeout, managed by fail2ban by unban)
+default-ipsettime = 0
+
+# Option: ipsettime
+# Notes:  specifies ticket timeout (handled ipset timeout only)
+# Values:  [ NUM ]  Default: 0 (managed by fail2ban by unban)
+ipsettime = 0
+
+# expresion to caclulate timeout from bantime, example:
+# banaction = %(known/banaction)s[ipsettime='<timeout-bantime>']
+timeout-bantime = $([ "<bantime>" -le 2147483 ] && echo "<bantime>" || echo 0)

 ipmset = f2b-<name>
 familyopt =
@@ -74,4 +82,4 @@ familyopt =
 [Init?family=inet6]

 ipmset = f2b-<name>6
-familyopt = <sp>family inet6
+familyopt = family inet6
@@ -17,7 +17,7 @@ actionstart = printf %%b "Hi,\n
              The jail <name> has been started successfully.\n
              Output will be buffered until <lines> lines are available.\n
              Regards,\n
-              Fail2Ban"|mail -s "[Fail2Ban] <name>: started on <fq-hostname>" <dest>
+              Fail2Ban"|mail -E 'set escape' -s "[Fail2Ban] <name>: started on <fq-hostname>" <dest>

 # Option:  actionstop
 # Notes.:  command executed at the stop of jail (or at the end of Fail2Ban)
@@ -28,13 +28,13 @@ actionstop = if [ -f <tmpfile> ]; then
                 These hosts have been banned by Fail2Ban.\n
                 `cat <tmpfile>`
                 Regards,\n
-                 Fail2Ban"|mail -s "[Fail2Ban] <name>: Summary from <fq-hostname>" <dest>
+                 Fail2Ban"|mail -E 'set escape' -s "[Fail2Ban] <name>: Summary from <fq-hostname>" <dest>
                 rm <tmpfile>
             fi
             printf %%b "Hi,\n
             The jail <name> has been stopped.\n
             Regards,\n
-             Fail2Ban"|mail -s "[Fail2Ban] <name>: stopped on <fq-hostname>" <dest>
+             Fail2Ban"|mail -E 'set escape' -s "[Fail2Ban] <name>: stopped on <fq-hostname>" <dest>

 # Option:  actioncheck
 # Notes.:  command executed once before each actionban command
@@ -55,7 +55,7 @@ actionban = printf %%b "`date`: <ip> (<failures> failures)\n" >> <tmpfile>
                These hosts have been banned by Fail2Ban.\n
                `cat <tmpfile>`
                \nRegards,\n
-                Fail2Ban"|mail -s "[Fail2Ban] <name>: Summary" <dest>
+                Fail2Ban"|mail -E 'set escape' -s "[Fail2Ban] <name>: Summary" <dest>
                rm <tmpfile>
            fi

@@ -72,7 +72,7 @@ actionunban =
 # Notes.:  Your system mail command. Is passed 2 args: subject and recipient
 # Values:  CMD
 #
-mailcmd = mail -s
+mailcmd = mail -E 'set escape' -s

 # Default name of the chain
 #
@@ -20,7 +20,7 @@ norestored = 1
 actionstart = printf %%b "Hi,\n
              The jail <name> has been started successfully.\n
              Regards,\n
-              Fail2Ban"|mail -s "[Fail2Ban] <name>: started on <fq-hostname>" <dest>
+              Fail2Ban"|mail -E 'set escape' -s "[Fail2Ban] <name>: started on <fq-hostname>" <dest>

 # Option:  actionstop
 # Notes.:  command executed at the stop of jail (or at the end of Fail2Ban)
@@ -29,7 +29,7 @@ actionstart = printf %%b "Hi,\n
 actionstop = printf %%b "Hi,\n
             The jail <name> has been stopped.\n
             Regards,\n
-             Fail2Ban"|mail -s "[Fail2Ban] <name>: stopped on <fq-hostname>" <dest>
+             Fail2Ban"|mail -E 'set escape' -s "[Fail2Ban] <name>: stopped on <fq-hostname>" <dest>

 # Option:  actioncheck
 # Notes.:  command executed once before each actionban command
@@ -49,7 +49,7 @@ actionban = printf %%b "Hi,\n
            Here is more information about <ip> :\n
            `%(_whois_command)s`\n
            Regards,\n
-            Fail2Ban"|mail -s "[Fail2Ban] <name>: banned <ip> from <fq-hostname>" <dest>
+            Fail2Ban"|mail -E 'set escape' -s "[Fail2Ban] <name>: banned <ip> from <fq-hostname>" <dest>

 # Option:  actionunban
 # Notes.:  command executed when unbanning an IP. Take care that the
@@ -16,7 +16,7 @@ norestored = 1
 actionstart = printf %%b "Hi,\n
              The jail <name> has been started successfully.\n
              Regards,\n
-              Fail2Ban"|mail -s "[Fail2Ban] <name>: started  on <fq-hostname>" <dest>
+              Fail2Ban"|mail -E 'set escape' -s "[Fail2Ban] <name>: started  on <fq-hostname>" <dest>

 # Option:  actionstop
 # Notes.:  command executed at the stop of jail (or at the end of Fail2Ban)
@@ -25,7 +25,7 @@ actionstart = printf %%b "Hi,\n
 actionstop = printf %%b "Hi,\n
             The jail <name> has been stopped.\n
             Regards,\n
-             Fail2Ban"|mail -s "[Fail2Ban] <name>: stopped on <fq-hostname>" <dest>
+             Fail2Ban"|mail -E 'set escape' -s "[Fail2Ban] <name>: stopped on <fq-hostname>" <dest>

 # Option:  actioncheck
 # Notes.:  command executed once before each actionban command
@@ -43,7 +43,7 @@ actionban = printf %%b "Hi,\n
            The IP <ip> has just been banned by Fail2Ban after
            <failures> attempts against <name>.\n
            Regards,\n
-            Fail2Ban"|mail -s "[Fail2Ban] <name>: banned <ip> from <fq-hostname>" <dest>
+            Fail2Ban"|mail -E 'set escape' -s "[Fail2Ban] <name>: banned <ip> from <fq-hostname>" <dest>

 # Option:  actionunban
 # Notes.:  command executed when unbanning an IP. Take care that the
@@ -34,7 +34,7 @@ type = multiport

 rule_match-custom =
 rule_match-allports = meta l4proto \{ <protocol> \}
-rule_match-multiport = $proto dport \{ <port> \}
+rule_match-multiport = $proto dport \{ $(echo '<port>' | sed s/:/-/g) \}
 match = <rule_match-<type>>

 # Option:  rule_stat
@@ -84,8 +84,15 @@ srv_cfg_path = /etc/nginx/
 #srv_cmd = nginx -c %(srv_cfg_path)s/nginx.conf
 srv_cmd = nginx

-# first test configuration is correct, hereafter send reload signal:
-blck_lst_reload = %(srv_cmd)s -qt; if [ $? -eq 0 ]; then
+# pid file (used to check nginx is running):
+srv_pid = /run/nginx.pid
+
+# command used to check whether nginx is running and configuration is valid:
+srv_is_running = [ -f "%(srv_pid)s" ]
+srv_check_cmd = %(srv_is_running)s && %(srv_cmd)s -qt
+
+# first test nginx is running and configuration is correct, hereafter send reload signal:
+blck_lst_reload = %(srv_check_cmd)s; if [ $? -eq 0 ]; then
                    %(srv_cmd)s -s reload; if [ $? -ne 0 ]; then echo 'reload failed.'; fi;
                  fi;

@@ -103,6 +110,8 @@ actionstop = %(actionflush)s

 actioncheck = 

-actionban = echo "\\\\<fid> 1;" >> '%(blck_lst_file)s'; %(blck_lst_reload)s
+_echo_blck_row = printf '\%%s 1;\n' "<fid>"

-actionunban = id=$(echo "<fid>" | sed -e 's/[]\/$*.^|[]/\\&/g'); sed -i "/^\\\\$id 1;$/d" %(blck_lst_file)s; %(blck_lst_reload)s
+actionban = %(_echo_blck_row)s >> '%(blck_lst_file)s'; %(blck_lst_reload)s
+
+actionunban = id=$(%(_echo_blck_row)s | sed -e 's/[]\/$*.^|[]/\\&/g'); sed -i "/^$id$/d" %(blck_lst_file)s; %(blck_lst_reload)s
@@ -51,7 +51,7 @@
 # Values:  CMD
 #
 actionstart = if ! ipset -quiet -name list f2b-<name> >/dev/null;
-              then ipset -quiet -exist create f2b-<name> hash:ip timeout <bantime>;
+              then ipset -quiet -exist create f2b-<name> hash:ip timeout <default-ipsettime>;
              fi

 # Option:  actionstop
@@ -66,7 +66,7 @@ actionstop = ipset flush f2b-<name>
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = ipset add f2b-<name> <ip> timeout <bantime> -exist
+actionban = ipset add f2b-<name> <ip> timeout <ipsettime> -exist

 # Option:  actionunban
 # Notes.:  command executed when unbanning an IP. Take care that the
@@ -76,10 +76,16 @@ actionban = ipset add f2b-<name> <ip> timeout <bantime> -exist
 #
 actionunban = ipset del f2b-<name> <ip> -exist

-[Init]
+# Option: default-ipsettime
+# Notes:  specifies default timeout in seconds (handled default ipset timeout only)
+# Values:  [ NUM ]  Default: 0 (no timeout, managed by fail2ban by unban)
+default-ipsettime = 0

-# Option: bantime
-# Notes:  specifies the bantime in seconds (handled internally rather than by fail2ban)
-# Values:  [ NUM ]  Default: 600
-#
-bantime = 600
+# Option: ipsettime
+# Notes:  specifies ticket timeout (handled ipset timeout only)
+# Values:  [ NUM ]  Default: 0 (managed by fail2ban by unban)
+ipsettime = 0
+
+# expresion to caclulate timeout from bantime, example:
+# banaction = %(known/banaction)s[ipsettime='<timeout-bantime>']
+timeout-bantime = $([ "<bantime>" -le 2147483 ] && echo "<bantime>" || echo 0)
@@ -19,7 +19,7 @@
 #         NOTICE
 #         INFO
 #         DEBUG
-# Values: [ LEVEL ]  Default: ERROR
+# Values: [ LEVEL ]  Default: INFO
 #
 loglevel = INFO

@@ -55,6 +55,12 @@ socket = /var/run/fail2ban/fail2ban.sock
 #
 pidfile = /var/run/fail2ban/fail2ban.pid

+# Option: allowipv6
+# Notes.: Allows IPv6 interface:
+#         Default: auto
+# Values: [ auto yes (on, true, 1) no (off, false, 0) ] Default: auto
+#allowipv6 = auto
+
 # Options: dbfile
 # Notes.: Set the file for the fail2ban persistent data to be stored.
 #         A value of ":memory:" means database is only stored in memory 
@@ -17,9 +17,9 @@ before = apache-common.conf

 [Definition]

-script = /\S*(?:php(?:[45]|[.-]cgi)?|\.asp|\.exe|\.pl)
+script = /\S*(?:php(?:[45]|[.-]cgi)?|\.asp|\.exe|\.pl|\bcgi-bin/)

-prefregex = ^%(_apache_error_client)s (?:AH0(?:01(?:28|30)|1(?:264|071)): )?(?:(?:[Ff]ile|script|[Gg]ot) )<F-CONTENT>.+</F-CONTENT>$
+prefregex = ^%(_apache_error_client)s (?:AH0(?:01(?:28|30)|1(?:264|071)|2811): )?(?:(?:[Ff]ile|script|[Gg]ot) )<F-CONTENT>.+</F-CONTENT>$

 failregex = ^(?:does not exist|not found or unable to stat): <script>\b
            ^'<script>\S*' not found or unable to stat
@@ -8,7 +8,7 @@ before = apache-common.conf

 [Definition]

-failregex = ^%(_apache_error_client)s (?:(?:AH0013[456]: )?Invalid (method|URI) in request\b|(?:AH00565: )?request failed: URI too long \(longer than \d+\)|request failed: erroneous characters after protocol string:|(?:AH00566: )?request failed: invalid characters in URI\b)
+failregex = ^%(_apache_error_client)s (?:(?:AH001[23][456]: )?Invalid (method|URI) in request\b|(?:AH00565: )?request failed: URI too long \(longer than \d+\)|request failed: erroneous characters after protocol string:|(?:AH00566: )?request failed: invalid characters in URI\b)

 ignoreregex =

@@ -2,5 +2,12 @@
 # Detecting failed login attempts
 # Logged in bwdata/logs/identity/Identity/log.txt

+[INCLUDES]
+before = common.conf
+
 [Definition]
-failregex = ^\s*\[WRN\]\s+Failed login attempt(?:, 2FA invalid)?\. <HOST>$
+_daemon = Bitwarden-Identity
+failregex = ^%(__prefix_line)s\s*\[(?:W(?:RN|arning)|Bit\.Core\.[^\]]+)\]\s+Failed login attempt(?:, 2FA invalid)?\. <ADDR>$
+
+# DEV Notes:
+# __prefix_line can result to an empty string, so it can support syslog and non-syslog at once.
@@ -25,7 +25,7 @@ __pid_re = (?:\[\d+\])

 # Daemon name (with optional source_file:line or whatever)
 # EXAMPLES: pam_rhosts_auth, [sshd], pop(pam_unix)
-__daemon_re = [\[\(]?%(_daemon)s(?:\(\S+\))?[\]\)]?:?
+__daemon_re = [\[\(]?<_daemon>(?:\(\S+\))?[\]\)]?:?

 # extra daemon info
 # EXAMPLE: [ID 800047 auth.info]
@@ -33,7 +33,7 @@ __daemon_extra_re = \[ID \d+ \S+\]

 # Combinations of daemon name and PID
 # EXAMPLES: sshd[31607], pop(pam_unix)[4920]
-__daemon_combs_re = (?:%(__pid_re)s?:\s+%(__daemon_re)s|%(__daemon_re)s%(__pid_re)s?:?)
+__daemon_combs_re = (?:<__pid_re>?:\s+<__daemon_re>|<__daemon_re><__pid_re>?:?)

 # Some messages have a kernel prefix with a timestamp
 # EXAMPLES: kernel: [769570.846956]
@@ -69,12 +69,12 @@ datepattern = <lt_<logtype>/datepattern>

 [lt_file]
 # Common line prefixes for logtype "file":
-__prefix_line = %(__date_ambit)s?\s*(?:%(__bsd_syslog_verbose)s\s+)?(?:%(__hostname)s\s+)?(?:%(__kernel_prefix)s\s+)?(?:%(__vserver)s\s+)?(?:%(__daemon_combs_re)s\s+)?(?:%(__daemon_extra_re)s\s+)?
+__prefix_line = <__date_ambit>?\s*(?:<__bsd_syslog_verbose>\s+)?(?:<__hostname>\s+)?(?:<__kernel_prefix>\s+)?(?:<__vserver>\s+)?(?:<__daemon_combs_re>\s+)?(?:<__daemon_extra_re>\s+)?
 datepattern = {^LN-BEG}

 [lt_short]
 # Common (short) line prefix for logtype "journal" (corresponds output of formatJournalEntry):
-__prefix_line = \s*(?:%(__hostname)s\s+)?(?:%(_daemon)s%(__pid_re)s?:?\s+)?(?:%(__kernel_prefix)s\s+)?
+__prefix_line = \s*(?:<__hostname>\s+)?(?:<_daemon><__pid_re>?:?\s+)?(?:<__kernel_prefix>\s+)?
 datepattern = %(lt_file/datepattern)s
 [lt_journal]
 __prefix_line = %(lt_short/__prefix_line)s
@@ -12,7 +12,7 @@ before = common.conf

 _daemon = courieresmtpd

-prefregex = ^%(__prefix_line)serror,relay=<HOST>,<F-CONTENT>.+</F-CONTENT>$
+prefregex = ^%(__prefix_line)serror,relay=<HOST>,(?:port=\d+,)?<F-CONTENT>.+</F-CONTENT>$

 failregex = ^[^:]*: 550 User (<.*> )?unknown\.?$
            ^msg="535 Authentication failed\.",cmd:( AUTH \S+)?( [0-9a-zA-Z\+/=]+)?(?: \S+)$
@@ -8,17 +8,18 @@ before = common.conf
 [Definition]

 _auth_worker = (?:dovecot: )?auth(?:-worker)?
+_auth_worker_info = (?:conn \w+:auth(?:-worker)? \(uid=\w+\): auth(?:-worker)?<\d+>: )?
 _daemon = (?:dovecot(?:-auth)?|auth)

-prefregex = ^%(__prefix_line)s(?:%(_auth_worker)s(?:\([^\)]+\))?: )?(?:%(__pam_auth)s(?:\(dovecot:auth\))?: |(?:pop3|imap)-login: )?(?:Info: )?<F-CONTENT>.+</F-CONTENT>$
+prefregex = ^%(__prefix_line)s(?:%(_auth_worker)s(?:\([^\)]+\))?: )?(?:%(__pam_auth)s(?:\(dovecot:auth\))?: |(?:pop3|imap|managesieve|submission)-login: )?(?:Info: )?%(_auth_worker_info)s<F-CONTENT>.+</F-CONTENT>$

 failregex = ^authentication failure; logname=<F-ALT_USER1>\S*</F-ALT_USER1> uid=\S* euid=\S* tty=dovecot ruser=<F-USER>\S*</F-USER> rhost=<HOST>(?:\s+user=<F-ALT_USER>\S*</F-ALT_USER>)?\s*$
-            ^(?:Aborted login|Disconnected)(?::(?: [^ \(]+)+)? \((?:auth failed, \d+ attempts(?: in \d+ secs)?|tried to use (?:disabled|disallowed) \S+ auth|proxy dest auth failed)\):(?: user=<<F-USER>[^>]*</F-USER>>,)?(?: method=\S+,)? rip=<HOST>(?:[^>]*(?:, session=<\S+>)?)\s*$
-            ^pam\(\S+,<HOST>(?:,\S*)?\): pam_authenticate\(\) failed: (?:User not known to the underlying authentication module: \d+ Time\(s\)|Authentication failure \(password mismatch\?\)|Permission denied)\s*$
-            ^[a-z\-]{3,15}\(\S*,<HOST>(?:,\S*)?\): (?:unknown user|invalid credentials|Password mismatch)\s*$
+            ^(?:Aborted login|Disconnected|Remote closed connection|Client has quit the connection)(?::(?: [^ \(]+)+)? \((?:auth failed, \d+ attempts(?: in \d+ secs)?|tried to use (?:disabled|disallowed) \S+ auth|proxy dest auth failed)\):(?: user=<<F-USER>[^>]*</F-USER>>,)?(?: method=\S+,)? rip=<HOST>(?:[^>]*(?:, session=<\S+>)?)\s*$
+            ^pam\(\S+,<HOST>(?:,\S*)?\): pam_authenticate\(\) failed: (?:User not known to the underlying authentication module: \d+ Time\(s\)|Authentication failure \([Pp]assword mismatch\?\)|Permission denied)\s*$
+            ^[a-z\-]{3,15}\(\S*,<HOST>(?:,\S*)?\): (?:[Uu]nknown user|[Ii]nvalid credentials|[Pp]assword mismatch)
            <mdre-<mode>>

-mdre-aggressive = ^(?:Aborted login|Disconnected)(?::(?: [^ \(]+)+)? \((?:no auth attempts|disconnected before auth was ready,|client didn't finish \S+ auth,)(?: (?:in|waited) \d+ secs)?\):(?: user=<[^>]*>,)?(?: method=\S+,)? rip=<HOST>(?:[^>]*(?:, session=<\S+>)?)\s*$
+mdre-aggressive = ^(?:Aborted login|Disconnected|Remote closed connection|Client has quit the connection)(?::(?: [^ \(]+)+)? \((?:no auth attempts|disconnected before auth was ready,|client didn't finish \S+ auth,)(?: (?:in|waited) \d+ secs)?\):(?: user=<[^>]*>,)?(?: method=\S+,)? rip=<HOST>(?:[^>]*(?:, session=<\S+>)?)\s*$

 mdre-normal = 

@@ -12,7 +12,7 @@ after = exim-common.local
 host_info_pre = (?:H=([\w.-]+ )?(?:\(\S+\) )?)?
 host_info_suf = (?::\d+)?(?: I=\[\S+\](:\d+)?)?(?: U=\S+)?(?: P=e?smtp)?(?: F=(?:<>|[^@]+@\S+))?\s
 host_info = %(host_info_pre)s\[<HOST>\]%(host_info_suf)s
-pid = (?: \[\d+\])?
+pid = (?: \[\d+\]| \w+ exim\[\d+\]:)?

 # DEV Notes:
 # From exim source code: ./src/receive.c:add_host_info_for_log
@@ -0,0 +1,6 @@
+# Fail2Ban filter for Gitlab
+# Detecting unauthorized access to the Gitlab Web portal
+# typically logged in /var/log/gitlab/gitlab-rails/application.log
+
+[Definition]
+failregex = ^: Failed Login: username=<F-USER>.+</F-USER> ip=<HOST>$
@@ -0,0 +1,9 @@
+# Fail2Ban filter for Grafana
+# Detecting unauthorized access
+# Typically logged in /var/log/grafana/grafana.log
+
+[Init]
+datepattern = ^t=%%Y-%%m-%%dT%%H:%%M:%%S%%z
+
+[Definition]
+failregex = ^(?: lvl=err?or)? msg="Invalid username or password"(?: uname=(?:"<F-ALT_USER>[^"]+</F-ALT_USER>"|<F-USER>\S+</F-USER>)| error="<F-ERROR>[^"]+</F-ERROR>"| \S+=(?:\S*|"[^"]+"))* remote_addr=<ADDR>$
@@ -5,21 +5,47 @@

 [Definition]

-# Option: failregex
-# Notes.: regex to match the password failures messages in the logfile.
-# Values: TEXT
-#
+logging = catalina
+failregex = <L_<logging>/failregex>
+maxlines = <L_<logging>/maxlines>
+datepattern = <L_<logging>/datepattern>
+
+[L_catalina]
+
 failregex = ^.*\nWARNING: Authentication attempt from <HOST> for user "[^"]*" failed\.$

-# Option:  ignoreregex
-# Notes.:  regex to ignore. If this regex matches, the line is ignored.
-# Values:  TEXT
-#
-ignoreregex = 
-
-# "maxlines" is number of log lines to buffer for multi-line regex searches
 maxlines = 2

 datepattern = ^%%b %%d, %%ExY %%I:%%M:%%S %%p
              ^WARNING:()**
-              {^LN-BEG}
+              {^LN-BEG}
+
+[L_webapp]
+
+failregex = ^ \[\S+\] WARN  \S+ - Authentication attempt from <HOST> for user "<F-USER>[^"]+</F-USER>" failed.
+
+maxlines = 1
+
+datepattern = ^%%H:%%M:%%S.%%f
+
+# DEV Notes:
+#
+# failregex is based on the default pattern given in Guacamole documentation :
+# https://guacamole.apache.org/doc/gug/configuring-guacamole.html#webapp-logging
+#
+# The following logback.xml Guacamole configuration file can then be used accordingly :
+# <configuration>
+#   <appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
+#     <file>/var/log/guacamole.log</file>
+#     <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+#       <fileNamePattern>/var/log/guacamole.%d.log.gz</fileNamePattern>
+#       <maxHistory>32</maxHistory>
+#     </rollingPolicy>
+#     <encoder>
+#       <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
+#     </encoder>
+#   </appender>
+#   <root level="info">
+#     <appender-ref ref="FILE" />
+#   </root>
+# </configuration>
@@ -6,24 +6,35 @@
 #
 import sys
 from fail2ban.server.ipdns import DNSUtils, IPAddr
+from threading import Thread

 def process_args(argv):
-    if len(argv) != 2:
-       raise ValueError("Please provide a single IP as an argument. Got: %s\n"
-                        % (argv[1:]))
+    if len(argv) - 1 not in (1, 2):
+       raise ValueError("Usage %s ip ?timeout?. Got: %s\n"
+                        % (argv[0], argv[1:]))
    ip = argv[1]

    if not IPAddr(ip).isValid:
       raise ValueError("Argument must be a single valid IP. Got: %s\n"
                        % ip)
-    return ip
+    return argv[1:]

 google_ips = None

-def is_googlebot(ip):
+def is_googlebot(ip, timeout=55):
    import re

-    host = DNSUtils.ipToName(ip)
+    timeout = float(timeout or 0)
+    if timeout:
+      def ipToNameTO(host, ip, timeout):
+        host[0] = DNSUtils.ipToName(ip)
+      host = [None]
+      th = Thread(target=ipToNameTO, args=(host, ip, timeout)); th.daemon=True; th.start()
+      th.join(timeout)
+      host = host[0]
+    else:
+      host = DNSUtils.ipToName(ip)
+
    if not host or not re.match(r'.*\.google(bot)?\.com$', host):
       return False
    host_ips = DNSUtils.dnsToIp(host)
@@ -31,7 +42,7 @@ def is_googlebot(ip):

 if __name__ == '__main__': # pragma: no cover
    try:
-      ret = is_googlebot(process_args(sys.argv))
+      ret = is_googlebot(*process_args(sys.argv))
    except ValueError as e:
      sys.stderr.write(str(e))
      sys.exit(2)
@@ -3,7 +3,7 @@

 [Definition]

-failregex = ^: \((?:http|mod)_auth\.c\.\d+\) (?:password doesn\'t match .* username: .*|digest: auth failed for .*: wrong password|get_password failed), IP: <HOST>\s*$
+failregex = ^\s*(?:: )?\(?(?:http|mod)_auth\.c\.\d+\) (?:password doesn\'t match for (?:\S+|.*?) username:\s+<F-USER>(?:\S+|.*?)</F-USER>\s*|digest: auth failed(?: for\s+<F-ALT_USER>(?:\S+|.*?)</F-ALT_USER>\s*)?: (?:wrong password|uri mismatch \([^\)]*\))|get_password failed),? IP: <HOST>\s*$

 ignoreregex = 

@@ -8,13 +8,17 @@
 # common.local
 before = common.conf

+# [DEFAULT]
+# logtype = short
+
 [Definition]

 _daemon = monit

+_prefix = Warning|HttpRequest
+
 # Regexp for previous (accessing monit httpd) and new (access denied) versions
-failregex = ^\[\s*\]\s*error\s*:\s*Warning:\s+Client '<HOST>' supplied (?:unknown user '[^']+'|wrong password for user '[^']*') accessing monit httpd$
-            ^%(__prefix_line)s\w+: access denied -- client <HOST>: (?:unknown user '[^']+'|wrong password for user '[^']*'|empty password)$
+failregex = ^%(__prefix_line)s(?:error\s*:\s+)?(?:%(_prefix)s):\s+(?:access denied\s+--\s+)?[Cc]lient '?<HOST>'?(?:\s+supplied|\s*:)\s+(?:unknown user '<F-ALT_USER>[^']+</F-ALT_USER>'|wrong password for user '<F-USER>[^']*</F-USER>'|empty password)

 # Ignore login with empty user (first connect, no user specified)
 # ignoreregex = %(__prefix_line)s\w+: access denied -- client <HOST>: (?:unknown user '')
@@ -17,7 +17,7 @@ before = common.conf

 _daemon = mysqld

-failregex = ^%(__prefix_line)s(?:(?:\d{6}|\d{4}-\d{2}-\d{2})[ T]\s?\d{1,2}:\d{2}:\d{2} )?(?:\d+ )?\[\w+\] (?:\[[^\]]+\] )*Access denied for user '[^']+'@'<HOST>' (to database '[^']*'|\(using password: (YES|NO)\))*\s*$
+failregex = ^%(__prefix_line)s(?:(?:\d{6}|\d{4}-\d{2}-\d{2})[ T]\s?\d{1,2}:\d{2}:\d{2} )?(?:\d+ )?\[\w+\] (?:\[[^\]]+\] )*Access denied for user '<F-USER>[^']+</F-USER>'@'<HOST>' (to database '[^']*'|\(using password: (YES|NO)\))*\s*$

 ignoreregex = 

@@ -32,7 +32,7 @@ __daemon_combs_re=(?:%(__pid_re)s?:\s+%(__daemon_re)s|%(__daemon_re)s%(__pid_re)

 #       hostname       daemon_id         spaces
 # this can be optional (for instance if we match named native log files)
-__line_prefix=(?:\s\S+ %(__daemon_combs_re)s\s+)?
+__line_prefix=(?:\s*\S+ %(__daemon_combs_re)s\s+)?

 prefregex = ^%(__line_prefix)s(?: error:)?\s*client(?: @\S*)? <HOST>#\S+(?: \([\S.]+\))?: <F-CONTENT>.+</F-CONTENT>\s(?:denied|\(NOTAUTH\))\s*$

@@ -1,4 +1,4 @@
-# Fail2Ban fitler for the phpMyAdmin-syslog
+# Fail2Ban filter for the phpMyAdmin-syslog
 #

 [INCLUDES]
@@ -12,16 +12,12 @@ before = common.conf

 _daemon = postfix(-\w+)?/\w+(?:/smtp[ds])?
 _port = (?::\d+)?
+_pref = [A-Z]{4}

 prefregex = ^%(__prefix_line)s<mdpr-<mode>> <F-CONTENT>.+</F-CONTENT>$

-mdpr-normal = (?:\w+: reject:|(?:improper command pipelining|too many errors) after \S+)
-mdre-normal=^RCPT from [^[]*\[<HOST>\]%(_port)s: 55[04] 5\.7\.1\s
-            ^RCPT from [^[]*\[<HOST>\]%(_port)s: 45[04] 4\.7\.\d+ (?:Service unavailable\b|Client host rejected: cannot find your (reverse )?hostname\b)
-            ^RCPT from [^[]*\[<HOST>\]%(_port)s: 450 4\.7\.\d+ (<[^>]*>)?: Helo command rejected: Host not found\b
-            ^EHLO from [^[]*\[<HOST>\]%(_port)s: 504 5\.5\.\d+ (<[^>]*>)?: Helo command rejected: need fully-qualified hostname\b
-            ^(RCPT|VRFY) from [^[]*\[<HOST>\]%(_port)s: 550 5\.1\.1\s
-            ^RCPT from [^[]*\[<HOST>\]%(_port)s: 450 4\.1\.\d+ (<[^>]*>)?: Sender address rejected: Domain not found\b
+mdpr-normal = (?:\w+: (?:milter-)?reject:|(?:improper command pipelining|too many errors) after \S+)
+mdre-normal=^%(_pref)s from [^[]*\[<HOST>\]%(_port)s: [45][50][04] [45]\.\d\.\d+ (?:(?:<[^>]*>)?: )?(?:(?:Helo command|(?:Sender|Recipient) address) rejected: )?(?:Service unavailable|User unknown|(?:Client host|Command|Data command) rejected|Relay access denied|(?:Host|Domain) not found|need fully-qualified hostname|match)\b
            ^from [^[]*\[<HOST>\]%(_port)s:?

 mdpr-auth = warning:
@@ -31,13 +27,13 @@ mdre-auth2= ^[^[]*\[<HOST>\]%(_port)s: SASL ((?i)LOGIN|PLAIN|(?:CRAM|DIGEST)-MD5

 # Mode "rbl" currently included in mode "normal", but if needed for jail "postfix-rbl" only:
 mdpr-rbl = %(mdpr-normal)s
-mdre-rbl  = ^RCPT from [^[]*\[<HOST>\]%(_port)s: [45]54 [45]\.7\.1 Service unavailable; Client host \[\S+\] blocked\b
+mdre-rbl  = ^%(_pref)s from [^[]*\[<HOST>\]%(_port)s: [45]54 [45]\.7\.1 Service unavailable; Client host \[\S+\] blocked\b

 # Mode "rbl" currently included in mode "normal" (within 1st rule)
 mdpr-more = %(mdpr-normal)s
 mdre-more = %(mdre-normal)s

-mdpr-ddos = lost connection after(?! DATA) [A-Z]+
+mdpr-ddos = (?:lost connection after(?! DATA) [A-Z]+|disconnect(?= from \S+(?: \S+=\d+)* auth=0/(?:[1-9]|\d\d+)))
 mdre-ddos = ^from [^[]*\[<HOST>\]%(_port)s:?

 mdpr-extra = (?:%(mdpr-auth)s|%(mdpr-normal)s)
@@ -1,4 +1,4 @@
-# Fail2Ban fitler for the Proftpd FTP daemon
+# Fail2Ban filter for the Proftpd FTP daemon
 #
 # Set "UseReverseDNS off" in proftpd.conf to avoid the need for DNS.
 # See: http://www.proftpd.org/docs/howto/DNS.html
@@ -14,16 +14,15 @@ before = common.conf

 _daemon = proftpd

-__suffix_failed_login = (User not authorized for login|No such user found|Incorrect password|Password expired|Account disabled|Invalid shell: '\S+'|User in \S+|Limit (access|configuration) denies login|Not a UserAlias|maximum login length exceeded).?
+__suffix_failed_login = ([uU]ser not authorized for login|[nN]o such user found|[iI]ncorrect password|[pP]assword expired|[aA]ccount disabled|[iI]nvalid shell: '\S+'|[uU]ser in \S+|[lL]imit (access|configuration) denies login|[nN]ot a UserAlias|[mM]aximum login length exceeded)


-prefregex = ^%(__prefix_line)s%(__hostname)s \(\S+\[<HOST>\]\)[: -]+ <F-CONTENT>(?:USER|SECURITY|Maximum).+</F-CONTENT>$
+prefregex = ^%(__prefix_line)s%(__hostname)s \(\S+\[<HOST>\]\)[: -]+ <F-CONTENT>(?:USER|SECURITY|Maximum) .+</F-CONTENT>$


-failregex = ^USER .*: no such user found from \S+ \[\S+\] to \S+:\S+ *$
-            ^USER .* \(Login failed\): %(__suffix_failed_login)s\s*$
-            ^SECURITY VIOLATION: .* login attempted\. *$
-            ^Maximum login attempts \(\d+\) exceeded *$
+failregex = ^USER <F-USER>\S+|.*?</F-USER>(?: \(Login failed\))?: %(__suffix_failed_login)s
+            ^SECURITY VIOLATION: <F-USER>\S+|.*?</F-USER> login attempted
+            ^Maximum login attempts \(\d+\) exceeded

 ignoreregex = 

@@ -8,11 +8,14 @@ before = common.conf
 [Definition]

 _daemon = (?:sendmail|sm-(?:mta|acceptingconnections))
+# "\w{14,20}" will give support for IDs from 14 up to 20 characters long
 __prefix_line = %(known/__prefix_line)s(?:\w{14,20}: )?
+addr = (?:IPv6:<IP6>|<IP4>)

-# "w{14,20}" will give support for IDs from 14 up to 20 characters long
-failregex = ^%(__prefix_line)s(\S+ )?\[(?:IPv6:<IP6>|<IP4>)\]( \(may be forged\))?: possible SMTP attack: command=AUTH, count=\d+$
+prefregex = ^<F-MLFID>%(__prefix_line)s</F-MLFID><F-CONTENT>.+</F-CONTENT>$

+failregex = ^(\S+ )?\[%(addr)s\]( \(may be forged\))?: possible SMTP attack: command=AUTH, count=\d+$
+            ^AUTH failure \([^\)]+\):(?: [^:]+:)? (?:authentication failure|user not found): [^,]*, user=<F-USER>(?:\S+|.*?)</F-USER>, relay=(?:\S+ )?\[%(addr)s\](?: \(may be forged\))?$
 ignoreregex =

 journalmatch = _SYSTEMD_UNIT=sendmail.service
@@ -21,19 +21,20 @@ before = common.conf

 _daemon = (?:(sm-(mta|acceptingconnections)|sendmail))
 __prefix_line = %(known/__prefix_line)s(?:\w{14,20}: )?
+addr = (?:(?:IPv6:)?<IP6>|<IP4>)

 prefregex = ^<F-MLFID>%(__prefix_line)s</F-MLFID><F-CONTENT>.+</F-CONTENT>$

-cmnfailre = ^ruleset=check_rcpt, arg1=(?P<email><\S+@\S+>), relay=(\S+ )?\[(?:IPv6:<IP6>|<IP4>)\](?: \(may be forged\))?, reject=(550 5\.7\.1 (?P=email)\.\.\. Relaying denied\. (IP name possibly forged \[(\d+\.){3}\d+\]|Proper authentication required\.|IP name lookup failed \[(\d+\.){3}\d+\])|553 5\.1\.8 (?P=email)\.\.\. Domain of sender address \S+ does not exist|550 5\.[71]\.1 (?P=email)\.\.\. (Rejected: .*|User unknown))$
-            ^ruleset=check_relay, arg1=(?P<dom>\S+), arg2=(?:IPv6:<IP6>|<IP4>), relay=((?P=dom) )?\[(\d+\.){3}\d+\](?: \(may be forged\))?, reject=421 4\.3\.2 (Connection rate limit exceeded\.|Too many open connections\.)$
-            ^rejecting commands from (\S* )?\[(?:IPv6:<IP6>|<IP4>)\] due to pre-greeting traffic after \d+ seconds$
-            ^(?:\S+ )?\[(?:IPv6:<IP6>|<IP4>)\]: (?:(?i)expn|vrfy) \S+ \[rejected\]$
+cmnfailre = ^ruleset=check_rcpt, arg1=(?P<email><\S+@\S+>), relay=(\S+ )?\[%(addr)s\](?: \(may be forged\))?, reject=(?:550 5\.7\.1(?: (?P=email)\.\.\.)?(?: Relaying denied\.)? (?:IP name possibly forged \[(\d+\.){3}\d+\]|Proper authentication required\.|IP name lookup failed \[(\d+\.){3}\d+\]|Fix reverse DNS for \S+)|553 5\.1\.8(?: (?P=email)\.\.\.)? Domain of sender address \S+ does not exist|550 5\.[71]\.1 (?P=email)\.\.\. (Rejected: .*|User unknown))$
+            ^ruleset=check_relay(?:, arg\d+=\S*)*, relay=(\S+ )?\[%(addr)s\](?: \(may be forged\))?, reject=421 4\.3\.2 (Connection rate limit exceeded\.|Too many open connections\.)$
+            ^rejecting commands from (\S* )?\[%(addr)s\] due to pre-greeting traffic after \d+ seconds$
+            ^(?:\S+ )?\[%(addr)s\]: (?:(?i)expn|vrfy) \S+ \[rejected\]$
            ^<[^@]+@[^>]+>\.\.\. No such user here$
-            ^<F-NOFAIL>from=<[^@]+@[^>]+></F-NOFAIL>, size=\d+, class=\d+, nrcpts=\d+, bodytype=\w+, proto=E?SMTP, daemon=MTA, relay=\S+ \[(?:IPv6:<IP6>|<IP4>)\]$
+            ^<F-NOFAIL>from=<[^@]+@[^>]+></F-NOFAIL>, size=\d+, class=\d+, nrcpts=\d+, bodytype=\w+, proto=E?SMTP, daemon=MTA, relay=\S+ \[%(addr)s\]$

 mdre-normal =

-mdre-extra = ^(?:\S+ )?\[(?:IPv6:<IP6>|<IP4>)\](?: \(may be forged\))? did not issue (?:[A-Z]{4}[/ ]?)+during connection to (?:TLS)?M(?:TA|S[PA])(?:-\w+)?$
+mdre-extra = ^(?:\S+ )?\[%(addr)s\](?: \(may be forged\))? did not issue \S+ during connection

 mdre-aggressive = %(mdre-extra)s

@@ -0,0 +1,9 @@
+# Fail2Ban filter for SoftEtherVPN
+# Detecting unauthorized access to SoftEtherVPN
+# typically logged in /usr/local/vpnserver/security_log/*/sec.log, or in syslog, depending on configuration
+
+[INCLUDES]
+before = common.conf
+
+[Definition]
+failregex = ^%(__prefix_line)s(?:(?:\([\d\-]+ [\d:.]+\) )?<SECURITY_LOG>: )?Connection "[^"]+": User authentication failed. The user name that has been provided was "<F-USER>(?:[^"]+|.+)</F-USER>", from <ADDR>\.$
@@ -25,7 +25,7 @@ __pref = (?:(?:error|fatal): (?:PAM: )?)?
 __suff = (?: (?:port \d+|on \S+|\[preauth\])){0,3}\s*
 __on_port_opt = (?: (?:port \d+|on \S+)){0,2}
 # close by authenticating user:
-__authng_user = (?: (?:invalid|authenticating) user <F-USER>\S+|.+?</F-USER>)?
+__authng_user = (?: (?:invalid|authenticating) user <F-USER>\S+|.*?</F-USER>)?

 # for all possible (also future) forms of "no matching (cipher|mac|MAC|compression method|key exchange method|host key type) found",
 # see ssherr.c for all possible SSH_ERR_..._ALG_MATCH errors.
@@ -40,39 +40,45 @@ prefregex = ^<F-MLFID>%(__prefix_line)s</F-MLFID>%(__pref)s<F-CONTENT>.+</F-CONT

 cmnfailre = ^[aA]uthentication (?:failure|error|failed) for <F-USER>.*</F-USER> from <HOST>( via \S+)?%(__suff)s$
            ^User not known to the underlying authentication module for <F-USER>.*</F-USER> from <HOST>%(__suff)s$
-            ^Failed publickey for invalid user <F-USER>(?P<cond_user>\S+)|(?:(?! from ).)*?</F-USER> from <HOST>%(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$)
-            ^Failed \b(?!publickey)\S+ for (?P<cond_inv>invalid user )?<F-USER>(?P<cond_user>\S+)|(?(cond_inv)(?:(?! from ).)*?|[^:]+)</F-USER> from <HOST>%(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$)
+            <cmnfailre-failed-pub-<publickey>>
+            ^Failed <cmnfailed> for (?P<cond_inv>invalid user )?<F-USER>(?P<cond_user>\S+)|(?(cond_inv)(?:(?! from ).)*?|[^:]+)</F-USER> from <HOST>%(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$)
            ^<F-USER>ROOT</F-USER> LOGIN REFUSED FROM <HOST>
            ^[iI](?:llegal|nvalid) user <F-USER>.*?</F-USER> from <HOST>%(__suff)s$
-            ^User <F-USER>.+</F-USER> from <HOST> not allowed because not listed in AllowUsers%(__suff)s$
-            ^User <F-USER>.+</F-USER> from <HOST> not allowed because listed in DenyUsers%(__suff)s$
-            ^User <F-USER>.+</F-USER> from <HOST> not allowed because not in any group%(__suff)s$
+            ^User <F-USER>\S+|.*?</F-USER> from <HOST> not allowed because not listed in AllowUsers%(__suff)s$
+            ^User <F-USER>\S+|.*?</F-USER> from <HOST> not allowed because listed in DenyUsers%(__suff)s$
+            ^User <F-USER>\S+|.*?</F-USER> from <HOST> not allowed because not in any group%(__suff)s$
            ^refused connect from \S+ \(<HOST>\)
            ^Received <F-MLFFORGET>disconnect</F-MLFFORGET> from <HOST>%(__on_port_opt)s:\s*3: .*: Auth fail%(__suff)s$
-            ^User <F-USER>.+</F-USER> from <HOST> not allowed because a group is listed in DenyGroups%(__suff)s$
-            ^User <F-USER>.+</F-USER> from <HOST> not allowed because none of user's groups are listed in AllowGroups%(__suff)s$
+            ^User <F-USER>\S+|.*?</F-USER> from <HOST> not allowed because a group is listed in DenyGroups%(__suff)s$
+            ^User <F-USER>\S+|.*?</F-USER> from <HOST> not allowed because none of user's groups are listed in AllowGroups%(__suff)s$
            ^<F-NOFAIL>%(__pam_auth)s\(sshd:auth\):\s+authentication failure;</F-NOFAIL>(?:\s+(?:(?:logname|e?uid|tty)=\S*)){0,4}\s+ruser=<F-ALT_USER>\S*</F-ALT_USER>\s+rhost=<HOST>(?:\s+user=<F-USER>\S*</F-USER>)?%(__suff)s$
-            ^(error: )?maximum authentication attempts exceeded for <F-USER>.*</F-USER> from <HOST>%(__on_port_opt)s(?: ssh\d*)?%(__suff)s$
-            ^User <F-USER>.+</F-USER> not allowed because account is locked%(__suff)s
+            ^maximum authentication attempts exceeded for <F-USER>.*</F-USER> from <HOST>%(__on_port_opt)s(?: ssh\d*)?%(__suff)s$
+            ^User <F-USER>\S+|.*?</F-USER> not allowed because account is locked%(__suff)s
            ^<F-MLFFORGET>Disconnecting</F-MLFFORGET>(?: from)?(?: (?:invalid|authenticating)) user <F-USER>\S+</F-USER> <HOST>%(__on_port_opt)s:\s*Change of username or service not allowed:\s*.*\[preauth\]\s*$
-            ^<F-MLFFORGET>Disconnecting</F-MLFFORGET>: Too many authentication failures(?: for <F-USER>.+?</F-USER>)?%(__suff)s$
+            ^Disconnecting: Too many authentication failures(?: for <F-USER>\S+|.*?</F-USER>)?%(__suff)s$
            ^<F-NOFAIL>Received <F-MLFFORGET>disconnect</F-MLFFORGET></F-NOFAIL> from <HOST>%(__on_port_opt)s:\s*11:
            <mdre-<mode>-other>
            ^<F-MLFFORGET><F-MLFGAINED>Accepted \w+</F-MLFGAINED></F-MLFFORGET> for <F-USER>\S+</F-USER> from <HOST>(?:\s|$)

+cmnfailed-any = \S+
+cmnfailed-ignore = \b(?!publickey)\S+
+cmnfailed-invalid = <cmnfailed-ignore>
+cmnfailed-nofail = (?:<F-NOFAIL>publickey</F-NOFAIL>|\S+)
+cmnfailed = <cmnfailed-<publickey>>
+
 mdre-normal =
 # used to differentiate "connection closed" with and without `[preauth]` (fail/nofail cases in ddos mode)
 mdre-normal-other = ^<F-NOFAIL><F-MLFFORGET>(Connection closed|Disconnected)</F-MLFFORGET></F-NOFAIL> (?:by|from)%(__authng_user)s <HOST>(?:%(__suff)s|\s*)$

 mdre-ddos = ^Did not receive identification string from <HOST>
+            ^kex_exchange_identification: (?:[Cc]lient sent invalid protocol identifier|[Cc]onnection closed by remote host)
            ^Bad protocol version identification '.*' from <HOST>
-            ^Connection <F-MLFFORGET>reset</F-MLFFORGET> by <HOST>
            ^<F-NOFAIL>SSH: Server;Ltype:</F-NOFAIL> (?:Authname|Version|Kex);Remote: <HOST>-\d+;[A-Z]\w+:
            ^Read from socket failed: Connection <F-MLFFORGET>reset</F-MLFFORGET> by peer
 # same as mdre-normal-other, but as failure (without <F-NOFAIL>) and [preauth] only:
-mdre-ddos-other = ^<F-MLFFORGET>(Connection closed|Disconnected)</F-MLFFORGET> (?:by|from)%(__authng_user)s <HOST>%(__on_port_opt)s\s+\[preauth\]\s*$
+mdre-ddos-other = ^<F-MLFFORGET>(Connection (?:closed|reset)|Disconnected)</F-MLFFORGET> (?:by|from)%(__authng_user)s <HOST>%(__on_port_opt)s\s+\[preauth\]\s*$

-mdre-extra = ^Received <F-MLFFORGET>disconnect</F-MLFFORGET> from <HOST>%(__on_port_opt)s:\s*14: No supported authentication methods available
+mdre-extra = ^Received <F-MLFFORGET>disconnect</F-MLFFORGET> from <HOST>%(__on_port_opt)s:\s*14: No(?: supported)? authentication methods available
            ^Unable to negotiate with <HOST>%(__on_port_opt)s: no matching <__alg_match> found.
            ^Unable to negotiate a <__alg_match>
            ^no matching <__alg_match> found:
@@ -84,6 +90,17 @@ mdre-aggressive = %(mdre-ddos)s
 # mdre-extra-other is fully included within mdre-ddos-other:
 mdre-aggressive-other = %(mdre-ddos-other)s

+# Parameter "publickey": nofail (default), invalid, any, ignore
+publickey = nofail
+# consider failed publickey for invalid users only:
+cmnfailre-failed-pub-invalid = ^Failed publickey for invalid user <F-USER>(?P<cond_user>\S+)|(?:(?! from ).)*?</F-USER> from <HOST>%(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$)
+# consider failed publickey for valid users too (don't need RE, see cmnfailed):
+cmnfailre-failed-pub-any =
+# same as invalid, but consider failed publickey for valid users too, just as no failure (helper to get IP and user-name only, see cmnfailed):
+cmnfailre-failed-pub-nofail = <cmnfailre-failed-pub-invalid>
+# don't consider failed publickey as failures (don't need RE, see cmnfailed):
+cmnfailre-failed-pub-ignore =
+
 cfooterre = ^<F-NOFAIL>Connection from</F-NOFAIL> <HOST>

 failregex = %(cmnfailre)s
@@ -51,6 +51,26 @@

 [Definition]

-failregex = ^<HOST> \- (?!- )\S+ \[\] \"(GET|POST|HEAD) [^\"]+\" 401\b
+# Parameter "method" can be used to specifiy request method
+req-method = \S+
+# Usage example (for jail.local):
+#   filter = traefik-auth[req-method="GET|POST|HEAD"]
+
+failregex = ^<HOST> \- <usrre-<mode>> \[\] \"(?:<req-method>) [^\"]+\" 401\b

 ignoreregex =
+
+# Parameter "mode": normal (default), ddos or aggressive
+# Usage example (for jail.local):
+#   [traefik-auth]
+#   mode = aggressive
+#   # or another jail (rewrite filter parameters of jail):
+#   [traefik-auth-ddos]
+#   filter = traefik-auth[mode=ddos]
+#
+mode = normal
+
+# part of failregex matches user name (must be available in normal mode, must be empty in ddos mode, and both for aggressive mode):
+usrre-normal = (?!- )<F-USER>\S+</F-USER>
+usrre-ddos = -
+usrre-aggressive = <F-USER>\S+</F-USER>
@@ -171,28 +171,28 @@ banaction = iptables-multiport
 banaction_allports = iptables-allports

 # The simplest action to take: ban only
-action_ = %(banaction)s[name=%(__name__)s, bantime="%(bantime)s", port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"]
+action_ = %(banaction)s[bantime="%(bantime)s", port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"]

 # ban & send an e-mail with whois report to the destemail.
-action_mw = %(banaction)s[name=%(__name__)s, bantime="%(bantime)s", port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"]
-            %(mta)s-whois[name=%(__name__)s, sender="%(sender)s", dest="%(destemail)s", protocol="%(protocol)s", chain="%(chain)s"]
+action_mw = %(action_)s
+            %(mta)s-whois[sender="%(sender)s", dest="%(destemail)s", protocol="%(protocol)s", chain="%(chain)s"]

 # ban & send an e-mail with whois report and relevant log lines
 # to the destemail.
-action_mwl = %(banaction)s[name=%(__name__)s, bantime="%(bantime)s", port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"]
-             %(mta)s-whois-lines[name=%(__name__)s, sender="%(sender)s", dest="%(destemail)s", logpath="%(logpath)s", chain="%(chain)s"]
+action_mwl = %(action_)s
+             %(mta)s-whois-lines[sender="%(sender)s", dest="%(destemail)s", logpath="%(logpath)s", chain="%(chain)s"]

 # See the IMPORTANT note in action.d/xarf-login-attack for when to use this action
 #
 # ban & send a xarf e-mail to abuse contact of IP address and include relevant log lines
 # to the destemail.
-action_xarf = %(banaction)s[name=%(__name__)s, bantime="%(bantime)s", port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"]
+action_xarf = %(action_)s
             xarf-login-attack[service=%(__name__)s, sender="%(sender)s", logpath="%(logpath)s", port="%(port)s"]

 # ban IP on CloudFlare & send an e-mail with whois report and relevant log lines
 # to the destemail.
 action_cf_mwl = cloudflare[cfuser="%(cfemail)s", cftoken="%(cfapikey)s"]
-                %(mta)s-whois-lines[name=%(__name__)s, sender="%(sender)s", dest="%(destemail)s", logpath="%(logpath)s", chain="%(chain)s"]
+                %(mta)s-whois-lines[sender="%(sender)s", dest="%(destemail)s", logpath="%(logpath)s", chain="%(chain)s"]

 # Report block via blocklist.de fail2ban reporting service API
 # 
@@ -202,21 +202,7 @@ action_cf_mwl = cloudflare[cfuser="%(cfemail)s", cftoken="%(cfapikey)s"]
 # in your `jail.local` globally (section [DEFAULT]) or per specific jail section (resp. in 
 # corresponding jail.d/my-jail.local file).
 #
-action_blocklist_de  = blocklist_de[email="%(sender)s", service=%(filter)s, apikey="%(blocklist_de_apikey)s", agent="%(fail2ban_agent)s"]
-
-# Report ban via badips.com, and use as blacklist
-#
-# See BadIPsAction docstring in config/action.d/badips.py for
-# documentation for this action.
-#
-# NOTE: This action relies on banaction being present on start and therefore
-# should be last action defined for a jail.
-#
-action_badips = badips.py[category="%(__name__)s", banaction="%(banaction)s", agent="%(fail2ban_agent)s"]
-#
-# Report ban via badips.com (uses action.d/badips.conf for reporting only)
-#
-action_badips_report = badips[category="%(__name__)s", agent="%(fail2ban_agent)s"]
+action_blocklist_de  = blocklist_de[email="%(sender)s", service="%(__name__)s", apikey="%(blocklist_de_apikey)s", agent="%(fail2ban_agent)s"]

 # Report ban via abuseipdb.com.
 #
@@ -333,7 +319,7 @@ maxretry = 1
 [openhab-auth]

 filter = openhab
-action = iptables-allports[name=NoAuthFailures]
+banaction = %(banaction_allports)s
 logpath = /opt/openhab/logs/request.log


@@ -440,6 +426,7 @@ backend  = %(syslog_backend)s

 port     = http,https
 logpath  = /var/log/tomcat*/catalina.out
+#logpath  = /var/log/guacamole.log

 [monit]
 #Ban clients brute-forcing the monit gui login
@@ -706,8 +693,8 @@ logpath  = /var/log/named/security.log
 [nsd]

 port     = 53
-action   = %(banaction)s[name=%(__name__)s-tcp, port="%(port)s", protocol="tcp", chain="%(chain)s", actname=%(banaction)s-tcp]
-           %(banaction)s[name=%(__name__)s-udp, port="%(port)s", protocol="udp", chain="%(chain)s", actname=%(banaction)s-udp]
+action_  = %(default/action_)s[name=%(__name__)s-tcp, protocol="tcp"]
+           %(default/action_)s[name=%(__name__)s-udp, protocol="udp"]
 logpath = /var/log/nsd.log


@@ -718,9 +705,8 @@ logpath = /var/log/nsd.log
 [asterisk]

 port     = 5060,5061
-action   = %(banaction)s[name=%(__name__)s-tcp, port="%(port)s", protocol="tcp", chain="%(chain)s", actname=%(banaction)s-tcp]
-           %(banaction)s[name=%(__name__)s-udp, port="%(port)s", protocol="udp", chain="%(chain)s", actname=%(banaction)s-udp]
-           %(mta)s-whois[name=%(__name__)s, dest="%(destemail)s"]
+action_  = %(default/action_)s[name=%(__name__)s-tcp, protocol="tcp"]
+           %(default/action_)s[name=%(__name__)s-udp, protocol="udp"]
 logpath  = /var/log/asterisk/messages
 maxretry = 10

@@ -728,9 +714,8 @@ maxretry = 10
 [freeswitch]

 port     = 5060,5061
-action   = %(banaction)s[name=%(__name__)s-tcp, port="%(port)s", protocol="tcp", chain="%(chain)s", actname=%(banaction)s-tcp]
-           %(banaction)s[name=%(__name__)s-udp, port="%(port)s", protocol="udp", chain="%(chain)s", actname=%(banaction)s-udp]
-           %(mta)s-whois[name=%(__name__)s, dest="%(destemail)s"]
+action_  = %(default/action_)s[name=%(__name__)s-tcp, protocol="tcp"]
+           %(default/action_)s[name=%(__name__)s-udp, protocol="udp"]
 logpath  = /var/log/freeswitch.log
 maxretry = 10

@@ -815,11 +800,23 @@ logpath = /var/log/ejabberd/ejabberd.log
 [counter-strike]

 logpath = /opt/cstrike/logs/L[0-9]*.log
-# Firewall: http://www.cstrike-planet.com/faq/6
 tcpport = 27030,27031,27032,27033,27034,27035,27036,27037,27038,27039
 udpport = 1200,27000,27001,27002,27003,27004,27005,27006,27007,27008,27009,27010,27011,27012,27013,27014,27015
-action  = %(banaction)s[name=%(__name__)s-tcp, port="%(tcpport)s", protocol="tcp", chain="%(chain)s", actname=%(banaction)s-tcp]
-           %(banaction)s[name=%(__name__)s-udp, port="%(udpport)s", protocol="udp", chain="%(chain)s", actname=%(banaction)s-udp]
+action_  = %(default/action_)s[name=%(__name__)s-tcp, port="%(tcpport)s", protocol="tcp"]
+           %(default/action_)s[name=%(__name__)s-udp, port="%(udpport)s", protocol="udp"]
+
+[softethervpn]
+port     = 500,4500
+protocol = udp
+logpath  = /usr/local/vpnserver/security_log/*/sec.log
+
+[gitlab]
+port    = http,https
+logpath = /var/log/gitlab/gitlab-rails/application.log
+
+[grafana]
+port    = http,https
+logpath = /var/log/grafana/grafana.log

 [bitwarden]
 port    = http,https
@@ -871,8 +868,8 @@ findtime     = 1
 [murmur]
 # AKA mumble-server
 port     = 64738
-action   = %(banaction)s[name=%(__name__)s-tcp, port="%(port)s", protocol=tcp, chain="%(chain)s", actname=%(banaction)s-tcp]
-           %(banaction)s[name=%(__name__)s-udp, port="%(port)s", protocol=udp, chain="%(chain)s", actname=%(banaction)s-udp]
+action_  = %(default/action_)s[name=%(__name__)s-tcp, protocol="tcp"]
+           %(default/action_)s[name=%(__name__)s-udp, protocol="udp"]
 logpath  = /var/log/mumble-server/mumble-server.log


@@ -38,27 +38,31 @@ class ActionReader(DefinitionInitConfigReader):

 	_configOpts = {
 		"actionstart": ["string", None],
-		"actionstart_on_demand": ["string", None],
+		"actionstart_on_demand": ["bool", None],
 		"actionstop": ["string", None],
 		"actionflush": ["string", None],
 		"actionreload": ["string", None],
 		"actioncheck": ["string", None],
 		"actionrepair": ["string", None],
-		"actionrepair_on_unban": ["string", None],
+		"actionrepair_on_unban": ["bool", None],
 		"actionban": ["string", None],
 		"actionreban": ["string", None],
 		"actionunban": ["string", None],
-		"norestored": ["string", None],
+		"norestored": ["bool", None],
 	}

 	def __init__(self, file_, jailName, initOpts, **kwargs):
+		# always supply jail name as name parameter if not specified in options:
+		n = initOpts.get("name")
+		if n is None:
+			initOpts["name"] = n = jailName
 		actname = initOpts.get("actname")
 		if actname is None:
 			actname = file_
+			# ensure we've unique action name per jail:
+			if n != jailName:
+				actname += n[len(jailName):] if n.startswith(jailName) else '-' + n
 			initOpts["actname"] = actname
-		# always supply jail name as name parameter if not specified in options:
-		if initOpts.get("name") is None:
-			initOpts["name"] = jailName
 		self._name = actname
 		DefinitionInitConfigReader.__init__(
 			self, file_, jailName, initOpts, **kwargs)
@@ -79,11 +83,6 @@ class ActionReader(DefinitionInitConfigReader):
 	def convert(self):
 		opts = self.getCombined(
 			ignore=CommandAction._escapedTags | set(('timeout', 'bantime')))
-		# type-convert only after combined (otherwise boolean converting prevents substitution):
-		for o in ('norestored', 'actionstart_on_demand', 'actionrepair_on_unban'):
-			if opts.get(o):
-				opts[o] = self._convert_to_boolean(opts[o])
-		
 		# stream-convert:
 		head = ["set", self._jailName]
 		stream = list()
@@ -29,7 +29,7 @@ import re
 import sys
 from ..helpers import getLogger

-if sys.version_info >= (3,2):
+if sys.version_info >= (3,): # pragma: 2.x no cover

 	# SafeConfigParser deprecated from Python 3.2 (renamed to ConfigParser)
 	from configparser import ConfigParser as SafeConfigParser, BasicInterpolation, \
@@ -61,7 +61,7 @@ if sys.version_info >= (3,2):
 				return super(BasicInterpolationWithName, self)._interpolate_some(
 					parser, option, accum, rest, section, map, *args, **kwargs)

-else: # pragma: no cover
+else: # pragma: 3.x no cover
 	from ConfigParser import SafeConfigParser, \
 		InterpolationMissingOptionError, NoOptionError, NoSectionError

@@ -372,7 +372,8 @@ after = 1.conf
 						s2 = alls.get(n)
 						if isinstance(s2, dict):
 							# save previous known values, for possible using in local interpolations later:
-							self.merge_section('KNOWN/'+n, s2, '')
+							self.merge_section('KNOWN/'+n, 
+								dict(filter(lambda i: i[0] in s, s2.iteritems())), '')
 							# merge section
 							s2.update(s)
 						else:
@@ -34,6 +34,30 @@ from ..helpers import getLogger, _as_bool, _merge_dicts, substituteRecursiveTags
 # Gets the instance of the logger.
 logSys = getLogger(__name__)

+CONVERTER = {
+	"bool": _as_bool,
+	"int": int,
+}
+def _OptionsTemplateGen(options):
+	"""Iterator over the options template with default options.
+	
+	Each options entry is composed of an array or tuple with:
+		[[type, name, ?default?], ...]
+	Or it is a dict:
+		{name: [type, default], ...}
+	"""
+	if isinstance(options, (list,tuple)):
+		for optname in options:
+			if len(optname) > 2:
+				opttype, optname, optvalue = optname
+			else:
+				(opttype, optname), optvalue = optname, None
+			yield opttype, optname, optvalue
+	else:
+		for optname in options:
+			opttype, optvalue = options[optname]
+			yield opttype, optname, optvalue
+

 class ConfigReader():
 	"""Generic config reader class.
@@ -120,6 +144,10 @@ class ConfigReader():
 		except AttributeError:
 			return False

+	def has_option(self, sec, opt, withDefault=True):
+		return self._cfg.has_option(sec, opt) if withDefault \
+			else opt in self._cfg._sections.get(sec, {})
+
 	def merge_defaults(self, d):
 		self._cfg.get_defaults().update(d)

@@ -224,31 +252,22 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes):
 	# Or it is a dict:
 	#  {name: [type, default], ...}
 	
-	def getOptions(self, sec, options, pOptions=None, shouldExist=False):
+	def getOptions(self, sec, options, pOptions=None, shouldExist=False, convert=True):
 		values = dict()
 		if pOptions is None:
 			pOptions = {}
 		# Get only specified options:
-		for optname in options:
-			if isinstance(options, (list,tuple)):
-				if len(optname) > 2:
-					opttype, optname, optvalue = optname
-				else:
-					(opttype, optname), optvalue = optname, None
-			else:
-				opttype, optvalue = options[optname]
+		for opttype, optname, optvalue in _OptionsTemplateGen(options):
 			if optname in pOptions:
 				continue
 			try:
-				if opttype == "bool":
-					v = self.getboolean(sec, optname)
-					if v is None: continue
-				elif opttype == "int":
-					v = self.getint(sec, optname)
-					if v is None: continue
-				else:
-					v = self.get(sec, optname, vars=pOptions)
+				v = self.get(sec, optname, vars=pOptions)
 				values[optname] = v
+				if convert:
+					conv = CONVERTER.get(opttype)
+					if conv:
+						if v is None: continue
+						values[optname] = conv(v)
 			except NoSectionError as e:
 				if shouldExist:
 					raise
@@ -261,8 +280,8 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes):
 					logSys.warning("'%s' not defined in '%s'. Using default one: %r"
 								% (optname, sec, optvalue))
 					values[optname] = optvalue
-				elif logSys.getEffectiveLevel() <= logLevel:
-					logSys.log(logLevel, "Non essential option '%s' not defined in '%s'.", optname, sec)
+				# elif logSys.getEffectiveLevel() <= logLevel:
+				# 	logSys.log(logLevel, "Non essential option '%s' not defined in '%s'.", optname, sec)
 			except ValueError:
 				logSys.warning("Wrong value for '" + optname + "' in '" + sec +
 							"'. Using default one: '" + repr(optvalue) + "'")
@@ -320,8 +339,9 @@ class DefinitionInitConfigReader(ConfigReader):
 			pOpts = dict()
 		if self._initOpts:
 			pOpts = _merge_dicts(pOpts, self._initOpts)
+		# type-convert only in combined (otherwise int/bool converting prevents substitution):
 		self._opts = ConfigReader.getOptions(
-			self, "Definition", self._configOpts, pOpts)
+			self, "Definition", self._configOpts, pOpts, convert=False)
 		self._pOpts = pOpts
 		if self.has_section("Init"):
 			# get only own options (without options from default):
@@ -342,10 +362,21 @@ class DefinitionInitConfigReader(ConfigReader):
 				if opt == '__name__' or opt in self._opts: continue
 				self._opts[opt] = self.get("Definition", opt)

+	def convertOptions(self, opts, configOpts):
+		"""Convert interpolated combined options to expected type.
+		"""
+		for opttype, optname, optvalue in _OptionsTemplateGen(configOpts):
+			conv = CONVERTER.get(opttype)
+			if conv:
+				v = opts.get(optname)
+				if v is None: continue
+				try:
+					opts[optname] = conv(v)
+				except ValueError:
+					logSys.warning("Wrong %s value %r for %r. Using default one: %r",
+						opttype, v, optname, optvalue)
+					opts[optname] = optvalue

-	def _convert_to_boolean(self, value):
-		return _as_bool(value)
-	
 	def getCombOption(self, optname):
 		"""Get combined definition option (as string) using pre-set and init
 		options as preselection (values with higher precedence as specified in section).
@@ -380,6 +411,8 @@ class DefinitionInitConfigReader(ConfigReader):
 			ignore=ignore, addrepl=self.getCombOption)
 		if not opts:
 			raise ValueError('recursive tag definitions unable to be resolved')
+		# convert options after all interpolations:
+		self.convertOptions(opts, self._configOpts)
 		return opts
 	
 	def convert(self):
@@ -48,7 +48,8 @@ class CSocket:
 	def send(self, msg, nonblocking=False, timeout=None):
 		# Convert every list member to string
 		obj = dumps(map(CSocket.convert, msg), HIGHEST_PROTOCOL)
-		self.__csock.send(obj + CSPROTO.END)
+		self.__csock.send(obj)
+		self.__csock.send(CSPROTO.END)
 		return self.receive(self.__csock, nonblocking, timeout)

 	def settimeout(self, timeout):
@@ -81,9 +82,12 @@ class CSocket:
 		msg = CSPROTO.EMPTY
 		if nonblocking: sock.setblocking(0)
 		if timeout: sock.settimeout(timeout)
-		while msg.rfind(CSPROTO.END) == -1:
-			chunk = sock.recv(512)
-			if chunk in ('', b''): # python 3.x may return b'' instead of ''
-				raise RuntimeError("socket connection broken")
+		bufsize = 1024
+		while msg.rfind(CSPROTO.END, -32) == -1:
+			chunk = sock.recv(bufsize)
+			if not len(chunk):
+				raise socket.error(104, 'Connection reset by peer')
+			if chunk == CSPROTO.END: break
 			msg = msg + chunk
+			if bufsize < 32768: bufsize <<= 1
 		return loads(msg)
@@ -168,19 +168,6 @@ class Fail2banClient(Fail2banCmdLine, Thread):
 		if not ret:
 			return None

-		# verify that directory for the socket file exists
-		socket_dir = os.path.dirname(self._conf["socket"])
-		if not os.path.exists(socket_dir):
-			logSys.error(
-				"There is no directory %s to contain the socket file %s."
-				% (socket_dir, self._conf["socket"]))
-			return None
-		if not os.access(socket_dir, os.W_OK | os.X_OK): # pragma: no cover
-			logSys.error(
-				"Directory %s exists but not accessible for writing"
-				% (socket_dir,))
-			return None
-
 		# Check already running
 		if not self._conf["force"] and os.path.exists(self._conf["socket"]):
 			logSys.error("Fail2ban seems to be in unexpected state (not running but the socket exists)")
@@ -27,15 +27,20 @@ import sys

 from ..version import version, normVersion
 from ..protocol import printFormatted
-from ..helpers import getLogger, str2LogLevel, getVerbosityFormat
+from ..helpers import getLogger, str2LogLevel, getVerbosityFormat, BrokenPipeError

 # Gets the instance of the logger.
 logSys = getLogger("fail2ban")

 def output(s): # pragma: no cover
-	print(s)
+	try:
+		print(s)
+	except (BrokenPipeError, IOError) as e: # pragma: no cover
+		if e.errno != 32: # closed / broken pipe
+			raise

-CONFIG_PARAMS = ("socket", "pidfile", "logtarget", "loglevel", "syslogsocket",)
+# Config parameters required to start fail2ban which can be also set via command line (overwrite fail2ban.conf),
+CONFIG_PARAMS = ("socket", "pidfile", "logtarget", "loglevel", "syslogsocket")
 # Used to signal - we are in test cases (ex: prevents change logging params, log capturing, etc)
 PRODUCTION = True

@@ -94,9 +99,10 @@ class Fail2banCmdLine():
 		output("and bans the corresponding IP addresses using firewall rules.")
 		output("")
 		output("Options:")
-		output("    -c <DIR>                configuration directory")
-		output("    -s <FILE>               socket path")
-		output("    -p <FILE>               pidfile path")
+		output("    -c, --conf <DIR>        configuration directory")
+		output("    -s, --socket <FILE>     socket path")
+		output("    -p, --pidfile <FILE>    pidfile path")
+		output("    --pname <NAME>          name of the process (main thread) to identify instance (default fail2ban-server)")
 		output("    --loglevel <LEVEL>      logging level")
 		output("    --logtarget <TARGET>    logging target, use file-name or stdout, stderr, syslog or sysout.")
 		output("    --syslogsocket auto|<FILE>")
@@ -129,17 +135,15 @@ class Fail2banCmdLine():
 		"""
 		for opt in optList:
 			o = opt[0]
-			if o == "-c":
+			if o in ("-c", "--conf"):
 				self._conf["conf"] = opt[1]
-			elif o == "-s":
+			elif o in ("-s", "--socket"):
 				self._conf["socket"] = opt[1]
-			elif o == "-p":
+			elif o in ("-p", "--pidfile"):
 				self._conf["pidfile"] = opt[1]
-			elif o.startswith("--log") or o.startswith("--sys"):
-				self._conf[ o[2:] ] = opt[1]
-			elif o in ["-d", "--dp", "--dump-pretty"]:
+			elif o in ("-d", "--dp", "--dump-pretty"):
 				self._conf["dump"] = True if o == "-d" else 2
-			elif o == "-t" or o == "--test":
+			elif o in ("-t", "--test"):
 				self.cleanConfOnly = True
 				self._conf["test"] = True
 			elif o == "-v":
@@ -163,12 +167,14 @@ class Fail2banCmdLine():
 				from ..server.mytime import MyTime
 				output(MyTime.str2seconds(opt[1]))
 				return True
-			elif o in ["-h", "--help"]:
+			elif o in ("-h", "--help"):
 				self.dispUsage()
 				return True
-			elif o in ["-V", "--version"]:
+			elif o in ("-V", "--version"):
 				self.dispVersion(o == "-V")
 				return True
+			elif o.startswith("--"): # other long named params (see also resetConf)
+				self._conf[ o[2:] ] = opt[1]
 		return None

 	def initCmdLine(self, argv):
@@ -185,7 +191,8 @@ class Fail2banCmdLine():
 			try:
 				cmdOpts = 'hc:s:p:xfbdtviqV'
 				cmdLongOpts = ['loglevel=', 'logtarget=', 'syslogsocket=', 'test', 'async',
-					'timeout=', 'str2sec=', 'help', 'version', 'dp', '--dump-pretty']
+					'conf=', 'pidfile=', 'pname=', 'socket=',
+					'timeout=', 'str2sec=', 'help', 'version', 'dp', 'dump-pretty']
 				optList, self._args = getopt.getopt(self._argv[1:], cmdOpts, cmdLongOpts)
 			except getopt.GetoptError:
 				self.dispUsage()
@@ -227,7 +234,8 @@ class Fail2banCmdLine():
 					if not conf:
 						self.configurator.readEarly()
 						conf = self.configurator.getEarlyOptions()
-					self._conf[o] = conf[o]
+					if o in conf:
+						self._conf[o] = conf[o]

 			logSys.info("Using socket file %s", self._conf["socket"])

@@ -304,18 +312,24 @@ class Fail2banCmdLine():
 	# since method is also exposed in API via globally bound variable
 	@staticmethod
 	def _exit(code=0):
-		if hasattr(os, '_exit') and os._exit:
-			os._exit(code)
-		else:
-			sys.exit(code)
+		# implicit flush without to produce broken pipe error (32):
+		sys.stderr.close()
+		try:
+			sys.stdout.flush()
+			# exit:
+			if hasattr(sys, 'exit') and sys.exit:
+				sys.exit(code)
+			else:
+				os._exit(code)
+		except (BrokenPipeError, IOError) as e: # pragma: no cover
+			if e.errno != 32: # closed / broken pipe
+				raise

 	@staticmethod
 	def exit(code=0):
 		logSys.debug("Exit with code %s", code)
 		# because of possible buffered output in python, we should flush it before exit:
 		logging.shutdown()
-		sys.stdout.flush()
-		sys.stderr.flush()
 		# exit
 		Fail2banCmdLine._exit(code)

@@ -53,6 +53,7 @@ class Fail2banReader(ConfigReader):
 		opts = [["string", "loglevel", "INFO" ],
 				["string", "logtarget", "STDERR"],
 				["string", "syslogsocket", "auto"],
+				["string", "allowipv6", "auto"],
 				["string", "dbfile", "/var/lib/fail2ban/fail2ban.sqlite3"],
 				["int",    "dbmaxmatches", None],
 				["string", "dbpurgeage", "1d"]]
@@ -74,6 +75,7 @@ class Fail2banReader(ConfigReader):
 		# Also dbfile should be set before all other database options.
 		# So adding order indices into items, to be stripped after sorting, upon return
 		order = {"thread":0, "syslogsocket":11, "loglevel":12, "logtarget":13,
+			"allowipv6": 14,
 			"dbfile":50, "dbmaxmatches":51, "dbpurgeage":51}
 		stream = list()
 		for opt in self.__opts:
@@ -21,7 +21,6 @@ Fail2Ban  reads log file that contains password failure report
 and bans the corresponding IP addresses using firewall rules.

 This tools can test regular expressions for "fail2ban".
-
 """

 __author__ = "Fail2Ban Developers"
@@ -36,11 +35,11 @@ __license__ = "GPL"

 import getopt
 import logging
+import re
 import os
 import shlex
 import sys
 import time
-import time
 import urllib
 from optparse import OptionParser, Option

@@ -53,7 +52,7 @@ except ImportError:

 from ..version import version, normVersion
 from .filterreader import FilterReader
-from ..server.filter import Filter, FileContainer
+from ..server.filter import Filter, FileContainer, MyTime
 from ..server.failregex import Regex, RegexException

 from ..helpers import str2LogLevel, getVerbosityFormat, FormatterWithTraceBack, getLogger, \
@@ -109,19 +108,22 @@ class _f2bOptParser(OptionParser):
 	def format_help(self, *args, **kwargs):
 		""" Overwritten format helper with full ussage."""
 		self.usage = ''
-		return "Usage: " + usage() + __doc__ + """
+		return "Usage: " + usage() + "\n" + __doc__ + """
 LOG:
-    string                  a string representing a log line
-    filename                path to a log file (/var/log/auth.log)
-    "systemd-journal"       search systemd journal (systemd-python required)
+  string                a string representing a log line
+  filename              path to a log file (/var/log/auth.log)
+  systemd-journal       search systemd journal (systemd-python required),
+                        optionally with backend parameters, see `man jail.conf`
+                        for usage and examples (systemd-journal[journalflags=1]).

 REGEX:
-    string                  a string representing a 'failregex'
-    filename                path to a filter file (filter.d/sshd.conf)
+  string                a string representing a 'failregex'
+  filter                name of filter, optionally with options (sshd[mode=aggressive])
+  filename              path to a filter file (filter.d/sshd.conf)

 IGNOREREGEX:
-    string                  a string representing an 'ignoreregex'
-    filename                path to a filter file (filter.d/sshd.conf)
+  string                a string representing an 'ignoreregex'
+  filename              path to a filter file (filter.d/sshd.conf)
 \n""" + OptionParser.format_help(self, *args, **kwargs) + """\n
 Report bugs to https://github.com/fail2ban/fail2ban/issues\n
 """ + __copyright__ + "\n"
@@ -252,6 +254,8 @@ class Fail2banRegex(object):

 		self.share_config=dict()
 		self._filter = Filter(None)
+		self._prefREMatched = 0
+		self._prefREGroups = list()
 		self._ignoreregex = list()
 		self._failregex = list()
 		self._time_elapsed = None
@@ -265,13 +269,21 @@ class Fail2banRegex(object):
 			self.setJournalMatch(shlex.split(opts.journalmatch))
 		if opts.timezone:
 			self._filter.setLogTimeZone(opts.timezone)
+		self._filter.checkFindTime = False
+		if True: # not opts.out:
+			MyTime.setAlternateNow(0); # accept every date (years from 19xx up to end of current century, '%ExY' and 'Exy' patterns)
+			from ..server.strptime import _updateTimeRE
+			_updateTimeRE()
 		if opts.datepattern:
 			self.setDatePattern(opts.datepattern)
 		if opts.usedns:
 			self._filter.setUseDns(opts.usedns)
 		self._filter.returnRawHost = opts.raw
-		self._filter.checkFindTime = False
 		self._filter.checkAllRegex = opts.checkAllRegex and not opts.out
+		# ignore pending (without ID/IP), added to matches if it hits later (if ID/IP can be retreved)
+		self._filter.ignorePending = bool(opts.out)
+		# callback to increment ignored RE's by index (during process):
+		self._filter.onIgnoreRegex = self._onIgnoreRegex
 		self._backend = 'auto'

 	def output(self, line):
@@ -288,8 +300,8 @@ class Fail2banRegex(object):
 			self._filter.setDatePattern(pattern)
 			self._datepattern_set = True
 			if pattern is not None:
-				self.output( "Use      datepattern : %s" % (
-					self._filter.getDatePattern()[1], ) )
+				self.output( "Use      datepattern : %s : %s" % (
+					pattern, self._filter.getDatePattern()[1], ) )

 	def setMaxLines(self, v):
 		if not self._maxlines_set:
@@ -318,26 +330,33 @@ class Fail2banRegex(object):
 		regex = regextype + 'regex'
 		# try to check - we've case filter?[options...]?:
 		basedir = self._opts.config
+		fltName = value
 		fltFile = None
 		fltOpt = {}
 		if regextype == 'fail':
-			fltName, fltOpt = extractOptions(value)
-			if fltName is not None:
-				if "." in fltName[~5:]:
-					tryNames = (fltName,)
-				else:
-					tryNames = (fltName, fltName + '.conf', fltName + '.local')
-				for fltFile in tryNames:
-					if not "/" in fltFile:
-						if os.path.basename(basedir) == 'filter.d':
-							fltFile = os.path.join(basedir, fltFile)
-						else:
-							fltFile = os.path.join(basedir, 'filter.d', fltFile)
+			if re.search(r'^/{0,3}[\w/_\-.]+(?:\[.*\])?$', value):
+				try:
+					fltName, fltOpt = extractOptions(value)
+					if "." in fltName[~5:]:
+						tryNames = (fltName,)
 					else:
-						basedir = os.path.dirname(fltFile)
-					if os.path.isfile(fltFile):
-						break
-					fltFile = None
+						tryNames = (fltName, fltName + '.conf', fltName + '.local')
+					for fltFile in tryNames:
+						if not "/" in fltFile:
+							if os.path.basename(basedir) == 'filter.d':
+								fltFile = os.path.join(basedir, fltFile)
+							else:
+								fltFile = os.path.join(basedir, 'filter.d', fltFile)
+						else:
+							basedir = os.path.dirname(fltFile)
+						if os.path.isfile(fltFile):
+							break
+						fltFile = None
+				except Exception as e:
+					output("ERROR: Wrong filter name or options: %s" % (str(e),))
+					output("       while parsing: %s" % (value,))
+					if self._verbose: raise(e)
+					return False
 		# if it is filter file:
 		if fltFile is not None:
 			if (basedir == self._opts.config
@@ -372,11 +391,8 @@ class Fail2banRegex(object):
 			if not ret:
 				output( "ERROR: failed to load filter %s" % value )
 				return False
-			# overwrite default logtype (considering that the filter could specify this too in Definition/Init sections):
-			if not fltOpt.get('logtype'):
-				reader.merge_defaults({
-					'logtype': ['file','journal'][int(self._backend.startswith("systemd"))]
-				})
+			# set backend-related options (logtype):
+			reader.applyAutoOptions(self._backend)
 			# get, interpolate and convert options:
 			reader.getOptions(None)
 			# show real options if expected:
@@ -436,71 +452,140 @@ class Fail2banRegex(object):
 					'add%sRegex' % regextype.title())(regex.getFailRegex())
 		return True

-	def testIgnoreRegex(self, line):
-		found = False
-		try:
-			ret = self._filter.ignoreLine([(line, "", "")])
-			if ret is not None:
-				found = True
-				regex = self._ignoreregex[ret].inc()
-		except RegexException as e: # pragma: no cover
-			output( 'ERROR: %s' % e )
-			return False
-		return found
+	def _onIgnoreRegex(self, idx, ignoreRegex):
+		self._lineIgnored = True
+		self._ignoreregex[idx].inc()

 	def testRegex(self, line, date=None):
 		orgLineBuffer = self._filter._Filter__lineBuffer
+		# duplicate line buffer (list can be changed inplace during processLine):
+		if self._filter.getMaxLines() > 1:
+			orgLineBuffer = orgLineBuffer[:]
 		fullBuffer = len(orgLineBuffer) >= self._filter.getMaxLines()
-		is_ignored = False
+		is_ignored = self._lineIgnored = False
 		try:
 			found = self._filter.processLine(line, date)
 			lines = []
-			line = self._filter.processedLine()
 			ret = []
 			for match in found:
-				# Append True/False flag depending if line was matched by
-				# more than one regex
-				match.append(len(ret)>1)
-				regex = self._failregex[match[0]]
-				regex.inc()
-				regex.appendIP(match)
+				if not self._opts.out:
+					# Append True/False flag depending if line was matched by
+					# more than one regex
+					match.append(len(ret)>1)
+					regex = self._failregex[match[0]]
+					regex.inc()
+					regex.appendIP(match)
 				if not match[3].get('nofail'):
 					ret.append(match)
 				else:
 					is_ignored = True
+			if self._opts.out: # (formated) output - don't need stats:
+				return None, ret, None
+			# prefregex stats:
+			if self._filter.prefRegex:
+				pre = self._filter.prefRegex
+				if pre.hasMatched():
+					self._prefREMatched += 1
+					if self._verbose:
+						if len(self._prefREGroups) < self._maxlines:
+							self._prefREGroups.append(pre.getGroups())
+						else:
+							if len(self._prefREGroups) == self._maxlines:
+								self._prefREGroups.append('...')
 		except RegexException as e: # pragma: no cover
 			output( 'ERROR: %s' % e )
-			return False
-		for bufLine in orgLineBuffer[int(fullBuffer):]:
-			if bufLine not in self._filter._Filter__lineBuffer:
-				try:
-					self._line_stats.missed_lines.pop(
-						self._line_stats.missed_lines.index("".join(bufLine)))
-					if self._debuggex:
-						self._line_stats.missed_lines_timeextracted.pop(
-							self._line_stats.missed_lines_timeextracted.index(
-								"".join(bufLine[::2])))
-				except ValueError:
-					pass
-				# if buffering - add also another lines from match:
-				if self._print_all_matched:
-					if not self._debuggex:
-						self._line_stats.matched_lines.append("".join(bufLine))
-					else:
-						lines.append(bufLine[0] + bufLine[2])
-				self._line_stats.matched += 1
-				self._line_stats.missed -= 1
+			return None, 0, None
+		if self._filter.getMaxLines() > 1:
+			for bufLine in orgLineBuffer[int(fullBuffer):]:
+				if bufLine not in self._filter._Filter__lineBuffer:
+					try:
+						self._line_stats.missed_lines.pop(
+							self._line_stats.missed_lines.index("".join(bufLine)))
+						if self._debuggex:
+							self._line_stats.missed_lines_timeextracted.pop(
+								self._line_stats.missed_lines_timeextracted.index(
+									"".join(bufLine[::2])))
+					except ValueError:
+						pass
+					# if buffering - add also another lines from match:
+					if self._print_all_matched:
+						if not self._debuggex:
+							self._line_stats.matched_lines.append("".join(bufLine))
+						else:
+							lines.append(bufLine[0] + bufLine[2])
+					self._line_stats.matched += 1
+					self._line_stats.missed -= 1
 		if lines: # pre-lines parsed in multiline mode (buffering)
-			lines.append(line)
+			lines.append(self._filter.processedLine())
 			line = "\n".join(lines)
-		return line, ret, is_ignored
+		return line, ret, (is_ignored or self._lineIgnored)
+
+	def _prepaireOutput(self):
+		"""Prepares output- and fetch-function corresponding given '--out' option (format)"""
+		ofmt = self._opts.out
+		if ofmt in ('id', 'ip'):
+			def _out(ret):
+				for r in ret:
+					output(r[1])
+		elif ofmt == 'msg':
+			def _out(ret):
+				for r in ret:
+					for r in r[3].get('matches'):
+						if not isinstance(r, basestring):
+							r = ''.join(r for r in r)
+						output(r)
+		elif ofmt == 'row':
+			def _out(ret):
+				for r in ret:
+					output('[%r,\t%r,\t%r],' % (r[1],r[2],dict((k,v) for k, v in r[3].iteritems() if k != 'matches')))
+		elif '<' not in ofmt:
+			def _out(ret):
+				for r in ret:
+					output(r[3].get(ofmt))
+		else: # extended format with tags substitution:
+			from ..server.actions import Actions, CommandAction, BanTicket
+			def _escOut(t, v):
+				# use safe escape (avoid inject on pseudo tag "\x00msg\x00"):
+				if t not in ('msg',):
+					return v.replace('\x00', '\\x00')
+				return v
+			def _out(ret):
+				rows = []
+				wrap = {'NL':0}
+				for r in ret:
+					ticket = BanTicket(r[1], time=r[2], data=r[3])
+					aInfo = Actions.ActionInfo(ticket)
+					# if msg tag is used - output if single line (otherwise let it as is to wrap multilines later):
+					def _get_msg(self):
+						if not wrap['NL'] and len(r[3].get('matches', [])) <= 1:
+							return self['matches']
+						else: # pseudo tag for future replacement:
+							wrap['NL'] = 1
+							return "\x00msg\x00"
+					aInfo['msg'] = _get_msg
+					# not recursive interpolation (use safe escape):
+					v = CommandAction.replaceDynamicTags(ofmt, aInfo, escapeVal=_escOut)
+					if wrap['NL']: # contains multiline tags (msg):
+						rows.append((r, v))
+						continue
+					output(v)
+				# wrap multiline tag (msg) interpolations to single line:
+				for r, v in rows:
+					for r in r[3].get('matches'):
+						if not isinstance(r, basestring):
+							r = ''.join(r for r in r)
+						r = v.replace("\x00msg\x00", r)
+						output(r)
+		return _out
+

 	def process(self, test_lines):
 		t0 = time.time()
+		if self._opts.out: # get out function
+			out = self._prepaireOutput()
 		for line in test_lines:
 			if isinstance(line, tuple):
-				line_datetimestripped, ret, is_ignored = self.testRegex(
-					line[0], line[1])
+				line_datetimestripped, ret, is_ignored = self.testRegex(line[0], line[1])
 				line = "".join(line[0])
 			else:
 				line = line.rstrip('\r\n')
@@ -508,8 +593,10 @@ class Fail2banRegex(object):
 					# skip comment and empty lines
 					continue
 				line_datetimestripped, ret, is_ignored = self.testRegex(line)
-			if not is_ignored:
-				is_ignored = self.testIgnoreRegex(line_datetimestripped)
+
+			if self._opts.out: # (formated) output:
+				if len(ret) > 0 and not is_ignored: out(ret)
+				continue

 			if is_ignored:
 				self._line_stats.ignored += 1
@@ -517,42 +604,25 @@ class Fail2banRegex(object):
 					self._line_stats.ignored_lines.append(line)
 					if self._debuggex:
 						self._line_stats.ignored_lines_timeextracted.append(line_datetimestripped)
-
-			if len(ret) > 0:
-				assert(not is_ignored)
-				if self._opts.out:
-					if self._opts.out in ('id', 'ip'):
-						for ret in ret:
-							output(ret[1])
-					elif self._opts.out == 'msg':
-						for ret in ret:
-							output('\n'.join(map(lambda v:''.join(v for v in v), ret[3].get('matches'))))
-					elif self._opts.out == 'row':
-						for ret in ret:
-							output('[%r,\t%r,\t%r],' % (ret[1],ret[2],dict((k,v) for k, v in ret[3].iteritems() if k != 'matches')))
-					else:
-						for ret in ret:
-							output(ret[3].get(self._opts.out))
-					continue
+			elif len(ret) > 0:
 				self._line_stats.matched += 1
 				if self._print_all_matched:
 					self._line_stats.matched_lines.append(line)
 					if self._debuggex:
 						self._line_stats.matched_lines_timeextracted.append(line_datetimestripped)
 			else:
-				if not is_ignored:
-					self._line_stats.missed += 1
-					if not self._print_no_missed and (self._print_all_missed or self._line_stats.missed <= self._maxlines + 1):
-						self._line_stats.missed_lines.append(line)
-						if self._debuggex:
-							self._line_stats.missed_lines_timeextracted.append(line_datetimestripped)
+				self._line_stats.missed += 1
+				if not self._print_no_missed and (self._print_all_missed or self._line_stats.missed <= self._maxlines + 1):
+					self._line_stats.missed_lines.append(line)
+					if self._debuggex:
+						self._line_stats.missed_lines_timeextracted.append(line_datetimestripped)
 			self._line_stats.tested += 1

 		self._time_elapsed = time.time() - t0

 	def printLines(self, ltype):
 		lstats = self._line_stats
-		assert(self._line_stats.missed == lstats.tested - (lstats.matched + lstats.ignored))
+		assert(lstats.missed == lstats.tested - (lstats.matched + lstats.ignored))
 		lines = lstats[ltype]
 		l = lstats[ltype + '_lines']
 		multiline = self._filter.getMaxLines() > 1
@@ -610,7 +680,18 @@ class Fail2banRegex(object):
 			pprint_list(out, " #) [# of hits] regular expression")
 			return total

-		# Print title
+		# Print prefregex:
+		if self._filter.prefRegex:
+			#self._filter.prefRegex.hasMatched()
+			pre = self._filter.prefRegex 
+			out = [pre.getRegex()]
+			if self._verbose:
+				for grp in self._prefREGroups:
+					out.append("    %s" % (grp,))
+			output( "\n%s: %d total" % ("Prefregex", self._prefREMatched) )
+			pprint_list(out)
+
+		# Print regex's:
 		total = print_failregexes("Failregex", self._failregex)
 		_ = print_failregexes("Ignoreregex", self._ignoreregex)

@@ -689,10 +770,10 @@ class Fail2banRegex(object):
 			test_lines = journal_lines_gen(flt, myjournal)
 		else:
 			# if single line parsing (without buffering)
-			if self._filter.getMaxLines() <= 1:
+			if self._filter.getMaxLines() <= 1 and '\n' not in cmd_log:
 				self.output( "Use      single line : %s" % shortstr(cmd_log.replace("\n", r"\n")) )
 				test_lines = [ cmd_log ]
-			else: # multi line parsing (with buffering)
+			else: # multi line parsing (with and without buffering)
 				test_lines = cmd_log.split("\n")
 				self.output( "Use      multi line : %s line(s)" % len(test_lines) )
 				for i, l in enumerate(test_lines):
@@ -712,6 +793,7 @@ class Fail2banRegex(object):


 def exec_command_line(*args):
+	logging.exitOnIOError = True
 	parser = get_opt_parser()
 	(opts, args) = parser.parse_args(*args)
 	errors = []
@@ -53,6 +53,14 @@ class FilterReader(DefinitionInitConfigReader):
 	def getFile(self):
 		return self.__file

+	def applyAutoOptions(self, backend):
+		# set init option to backend-related logtype, considering
+		# that the filter settings may be overwritten in its local:
+		if (not self._initOpts.get('logtype') and 
+		    not self.has_option('Definition', 'logtype', False)
+		  ):
+			self._initOpts['logtype'] = ['file','journal'][int(backend.startswith("systemd"))]
+
 	def convert(self):
 		stream = list()
 		opts = self.getCombined()
@@ -133,20 +133,18 @@ class JailReader(ConfigReader):
 			# Read filter
 			flt = self.__opts["filter"]
 			if flt:
-				filterName, filterOpt = extractOptions(flt)
-				if not filterName:
-					raise JailDefError("Invalid filter definition %r" % flt)
+				try:
+					filterName, filterOpt = extractOptions(flt)
+				except ValueError as e:
+					raise JailDefError("Invalid filter definition %r: %s" % (flt, e))
 				self.__filter = FilterReader(
 					filterName, self.__name, filterOpt, 
 					share_config=self.share_config, basedir=self.getBaseDir())
 				ret = self.__filter.read()
 				if not ret:
 					raise JailDefError("Unable to read the filter %r" % filterName)
-				if not filterOpt.get('logtype'):
-					# overwrite default logtype backend-related (considering that the filter settings may be overwritten):
-					self.__filter.merge_defaults({
-						'logtype': ['file','journal'][int(self.__opts.get('backend', '').startswith("systemd"))]
-					})
+				# set backend-related options (logtype):
+				self.__filter.applyAutoOptions(self.__opts.get('backend', ''))
 				# merge options from filter as 'known/...' (all options unfiltered):
 				self.__filter.getOptions(self.__opts, all=True)
 				ConfigReader.merge_section(self, self.__name, self.__filter.getCombined(), 'known/')
@@ -170,10 +168,10 @@ class JailReader(ConfigReader):
 					if not act:			  # skip empty actions
 						continue
 					# join with previous line if needed (consider possible new-line):
-					actName, actOpt = extractOptions(act)
-					prevln = ''
-					if not actName:
-						raise JailDefError("Invalid action definition %r" % act)
+					try:
+						actName, actOpt = extractOptions(act)
+					except ValueError as e:
+						raise JailDefError("Invalid action definition %r: %s" % (act, e))
 					if actName.endswith(".py"):
 						self.__actions.append([
 							"set",
@@ -208,6 +208,26 @@ class FormatterWithTraceBack(logging.Formatter):
 		return logging.Formatter.format(self, record)


+logging.exitOnIOError = False
+def __stopOnIOError(logSys=None, logHndlr=None): # pragma: no cover
+	if logSys and len(logSys.handlers):
+		logSys.removeHandler(logSys.handlers[0])
+	if logHndlr:
+		logHndlr.close = lambda: None
+	logging.StreamHandler.flush = lambda self: None
+	#sys.excepthook = lambda *args: None
+	if logging.exitOnIOError:
+		try:
+			sys.stderr.close()
+		except:
+			pass
+		sys.exit(0)
+
+try:
+	BrokenPipeError = BrokenPipeError
+except NameError: # pragma: 3.x no cover
+	BrokenPipeError = IOError
+
 __origLog = logging.Logger._log
 def __safeLog(self, level, msg, args, **kwargs):
 	"""Safe log inject to avoid possible errors by unsafe log-handlers, 
@@ -223,6 +243,10 @@ def __safeLog(self, level, msg, args, **kwargs):
 	try:
 		# if isEnabledFor(level) already called...
 		__origLog(self, level, msg, args, **kwargs)
+	except (BrokenPipeError, IOError) as e: # pragma: no cover
+		if e.errno == 32: # closed / broken pipe
+			__stopOnIOError(self)
+		raise
 	except Exception as e: # pragma: no cover - unreachable if log-handler safe in this python-version
 		try:
 			for args in (
@@ -237,6 +261,18 @@ def __safeLog(self, level, msg, args, **kwargs):
 			pass
 logging.Logger._log = __safeLog

+__origLogFlush = logging.StreamHandler.flush
+def __safeLogFlush(self):
+	"""Safe flush inject stopping endless logging on closed streams (redirected pipe).
+	"""
+	try:
+		__origLogFlush(self)
+	except (BrokenPipeError, IOError) as e: # pragma: no cover
+		if e.errno == 32: # closed / broken pipe
+			__stopOnIOError(None, self)
+		raise
+logging.StreamHandler.flush = __safeLogFlush
+
 def getLogger(name):
 	"""Get logging.Logger instance with Fail2Ban logger name convention
 	"""
@@ -267,7 +303,7 @@ def getVerbosityFormat(verbosity, fmt=' %(message)s', addtime=True, padding=True
 			if addtime:
 				fmt = ' %(asctime)-15s' + fmt
 	else: # default (not verbose):
-		fmt = "%(name)-23.23s [%(process)d]: %(levelname)-7s" + fmt
+		fmt = "%(name)-24s[%(process)d]: %(levelname)-7s" + fmt
 		if addtime:
 			fmt = "%(asctime)s " + fmt
 	# remove padding if not needed:
@@ -291,7 +327,7 @@ def splitwords(s):
 	"""
 	if not s:
 		return []
-	return filter(bool, map(str.strip, re.split('[ ,\n]+', s)))
+	return filter(bool, map(lambda v: v.strip(), re.split('[ ,\n]+', s)))

 if sys.version_info >= (3,5):
 	eval(compile(r'''if 1:
@@ -335,21 +371,27 @@ OPTION_CRE = re.compile(r"^([^\[]+)(?:\[(.*)\])?\s*$", re.DOTALL)
 # since v0.10 separator extended with `]\s*[` for support of multiple option groups, syntax 
 # `action = act[p1=...][p2=...]`
 OPTION_EXTRACT_CRE = re.compile(
-	r'([\w\-_\.]+)=(?:"([^"]*)"|\'([^\']*)\'|([^,\]]*))(?:,|\]\s*\[|$)', re.DOTALL)
+	r'\s*([\w\-_\.]+)=(?:"([^"]*)"|\'([^\']*)\'|([^,\]]*))(?:,|\]\s*\[|$|(?P<wrngA>.+))|,?\s*$|(?P<wrngB>.+)', re.DOTALL)
 # split by new-line considering possible new-lines within options [...]:
 OPTION_SPLIT_CRE = re.compile(
-	r'(?:[^\[\n]+(?:\s*\[\s*(?:[\w\-_\.]+=(?:"[^"]*"|\'[^\']*\'|[^,\]]*)\s*(?:,|\]\s*\[)?\s*)*\])?\s*|[^\n]+)(?=\n\s*|$)', re.DOTALL)
+	r'(?:[^\[\s]+(?:\s*\[\s*(?:[\w\-_\.]+=(?:"[^"]*"|\'[^\']*\'|[^,\]]*)\s*(?:,|\]\s*\[)?\s*)*\])?\s*|\S+)(?=\n\s*|\s+|$)', re.DOTALL)

 def extractOptions(option):
 	match = OPTION_CRE.match(option)
 	if not match:
-		# TODO proper error handling
-		return None, None
+		raise ValueError("unexpected option syntax")
 	option_name, optstr = match.groups()
 	option_opts = dict()
 	if optstr:
 		for optmatch in OPTION_EXTRACT_CRE.finditer(optstr):
+			if optmatch.group("wrngA"):
+				raise ValueError("unexpected syntax at %d after option %r: %s" % (
+					optmatch.start("wrngA"), optmatch.group(1), optmatch.group("wrngA")[0:25]))
+			if optmatch.group("wrngB"):
+				raise ValueError("expected option, wrong syntax at %d: %s" % (
+					optmatch.start("wrngB"), optmatch.group("wrngB")[0:25]))
 			opt = optmatch.group(1)
+			if not opt: continue
 			value = [
 				val for val in optmatch.group(2,3,4) if val is not None][0]
 			option_opts[opt.strip()] = value.strip()
@@ -363,8 +405,8 @@ def splitWithOptions(option):
 # tags (<tag>) in tagged options.
 #

-# max tag replacement count:
-MAX_TAG_REPLACE_COUNT = 10
+# max tag replacement count (considering tag X in tag Y repeat):
+MAX_TAG_REPLACE_COUNT = 25

 # compiled RE for tag name (replacement name) 
 TAG_CRE = re.compile(r'<([^ <>]+)>')
@@ -398,6 +440,7 @@ def substituteRecursiveTags(inptags, conditional='',
 	done = set()
 	noRecRepl = hasattr(tags, "getRawItem")
 	# repeat substitution while embedded-recursive (repFlag is True)
+	repCounts = {}
 	while True:
 		repFlag = False
 		# substitute each value:
@@ -409,7 +452,7 @@ def substituteRecursiveTags(inptags, conditional='',
 			value = orgval = uni_string(tags[tag])
 			# search and replace all tags within value, that can be interpolated using other tags:
 			m = tre_search(value)
-			refCounts = {}
+			rplc = repCounts.get(tag, {})
 			#logSys.log(5, 'TAG: %s, value: %s' % (tag, value))
 			while m:
 				# found replacement tag:
@@ -419,13 +462,13 @@ def substituteRecursiveTags(inptags, conditional='',
 					m = tre_search(value, m.end())
 					continue
 				#logSys.log(5, 'found: %s' % rtag)
-				if rtag == tag or refCounts.get(rtag, 1) > MAX_TAG_REPLACE_COUNT:
+				if rtag == tag or rplc.get(rtag, 1) > MAX_TAG_REPLACE_COUNT:
 					# recursive definitions are bad
 					#logSys.log(5, 'recursion fail tag: %s value: %s' % (tag, value) )
 					raise ValueError(
 						"properties contain self referencing definitions "
 						"and cannot be resolved, fail tag: %s, found: %s in %s, value: %s" % 
-						(tag, rtag, refCounts, value))
+						(tag, rtag, rplc, value))
 				repl = None
 				if conditional:
 					repl = tags.get(rtag + '?' + conditional)
@@ -445,7 +488,7 @@ def substituteRecursiveTags(inptags, conditional='',
 				value = value.replace('<%s>' % rtag, repl)
 				#logSys.log(5, 'value now: %s' % value)
 				# increment reference count:
-				refCounts[rtag] = refCounts.get(rtag, 0) + 1
+				rplc[rtag] = rplc.get(rtag, 0) + 1
 				# the next match for replace:
 				m = tre_search(value, m.start())
 			#logSys.log(5, 'TAG: %s, newvalue: %s' % (tag, value))
@@ -453,6 +496,7 @@ def substituteRecursiveTags(inptags, conditional='',
 			if orgval != value:
 				# check still contains any tag - should be repeated (possible embedded-recursive substitution):
 				if tre_search(value):
+					repCounts[tag] = rplc
 					repFlag = True
 				# copy return tags dict to prevent modifying of inptags:
 				if id(tags) == id(inptags):
@@ -55,6 +55,8 @@ protocol = [
 ["stop", "stops all jails and terminate the server"], 
 ["unban --all", "unbans all IP addresses (in all jails and database)"],
 ["unban <IP> ... <IP>", "unbans <IP> (in all jails and database)"],
+["banned", "return jails with banned IPs as dictionary"],
+["banned <IP> ... <IP>]", "return list(s) of jails where given IP(s) are banned"],
 ["status", "gets the current status of the server"], 
 ["ping", "tests if the server is alive"],
 ["echo", "for internal usage, returns back and outputs a given string"],
@@ -120,6 +122,8 @@ protocol = [
 ["set <JAIL> action <ACT> <PROPERTY> <VALUE>", "sets the <VALUE> of <PROPERTY> for the action <ACT> for <JAIL>"],
 ["set <JAIL> action <ACT> <METHOD>[ <JSONKWARGS>]", "calls the <METHOD> with <JSONKWARGS> for the action <ACT> for <JAIL>"],
 ['', "JAIL INFORMATION", ""],
+["get <JAIL> banned", "return banned IPs of <JAIL>"],
+["get <JAIL> banned <IP> ... <IP>]", "return 1 if IP is banned in <JAIL> otherwise 0, or a list of 1/0 for multiple IPs"],
 ["get <JAIL> logpath", "gets the list of the monitored files for <JAIL>"],
 ["get <JAIL> logencoding", "gets the encoding of the log files for <JAIL>"],
 ["get <JAIL> journalmatch", "gets the journal filter match for <JAIL>"],
@@ -30,7 +30,10 @@ import tempfile
 import threading
 import time
 from abc import ABCMeta
-from collections import MutableMapping
+try:
+	from collections.abc import MutableMapping
+except ImportError:
+	from collections import MutableMapping

 from .failregex import mapTag2Opt
 from .ipdns import DNSUtils
@@ -392,10 +395,13 @@ class CommandAction(ActionBase):
 	def _getOperation(self, tag, family):
 		# replace operation tag (interpolate all values), be sure family is enclosed as conditional value
 		# (as lambda in addrepl so only if not overwritten in action):
-		return self.replaceTag(tag, self._properties,
+		cmd = self.replaceTag(tag, self._properties,
 			conditional=('family='+family if family else ''),
-			addrepl=(lambda tag:family if tag == 'family' else None),
 			cache=self.__substCache)
+		if '<' not in cmd or not family: return cmd
+		# replace family as dynamic tags, important - don't cache, no recursion and auto-escape here:
+		cmd = self.replaceDynamicTags(cmd, {'family':family})
+		return cmd

 	def _operationExecuted(self, tag, family, *args):
 		""" Get, set or delete command of operation considering family.
@@ -769,7 +775,7 @@ class CommandAction(ActionBase):
 	ESCAPE_VN_CRE = re.compile(r"\W")

 	@classmethod
-	def replaceDynamicTags(cls, realCmd, aInfo):
+	def replaceDynamicTags(cls, realCmd, aInfo, escapeVal=None):
 		"""Replaces dynamical tags in `query` with property values.

 		**Important**
@@ -794,16 +800,17 @@ class CommandAction(ActionBase):
 		# array for escaped vars:
 		varsDict = dict()

-		def escapeVal(tag, value):
-			# if the value should be escaped:
-			if cls.ESCAPE_CRE.search(value):
-				# That one needs to be escaped since its content is
-				# out of our control
-				tag = 'f2bV_%s' % cls.ESCAPE_VN_CRE.sub('_', tag)
-				varsDict[tag] = value # add variable
-				value = '$'+tag	# replacement as variable
-			# replacement for tag:
-			return value
+		if not escapeVal:
+			def escapeVal(tag, value):
+				# if the value should be escaped:
+				if cls.ESCAPE_CRE.search(value):
+					# That one needs to be escaped since its content is
+					# out of our control
+					tag = 'f2bV_%s' % cls.ESCAPE_VN_CRE.sub('_', tag)
+					varsDict[tag] = value # add variable
+					value = '$'+tag	# replacement as variable
+				# replacement for tag:
+				return value

 		# additional replacement as calling map:
 		ADD_REPL_TAGS_CM = CallingMap(ADD_REPL_TAGS)
@@ -28,7 +28,10 @@ import logging
 import os
 import sys
 import time
-from collections import Mapping
+try:
+	from collections.abc import Mapping
+except ImportError:
+	from collections import Mapping
 try:
 	from collections import OrderedDict
 except ImportError:
@@ -210,6 +213,14 @@ class Actions(JailThread, Mapping):
 	def getBanTime(self):
 		return self.__banManager.getBanTime()

+	def getBanned(self, ids):
+		lst = self.__banManager.getBanList()
+		if not ids:
+			return lst
+		if len(ids) == 1:
+			return 1 if ids[0] in lst else 0
+		return map(lambda ip: 1 if ip in lst else 0, ids)
+
 	def addBannedIP(self, ip):
 		"""Ban an IP or list of IPs."""
 		unixTime = MyTime.time()
@@ -243,7 +254,7 @@ class Actions(JailThread, Mapping):
 		if ip is None:
 			return self.__flushBan(db)
 		# Multiple IPs:
-		if isinstance(ip, list):
+		if isinstance(ip, (list, tuple)):
 			missed = []
 			cnt = 0
 			for i in ip:
@@ -265,6 +276,14 @@ class Actions(JailThread, Mapping):
 			# Unban the IP.
 			self.__unBan(ticket)
 		else:
+			# Multiple IPs by subnet or dns:
+			if not isinstance(ip, IPAddr):
+				ipa = IPAddr(ip)
+				if not ipa.isSingle: # subnet (mask/cidr) or raw (may be dns/hostname):
+					ips = filter(ipa.contains, self.__banManager.getBanList())
+					if ips:
+						return self.removeBannedIP(ips, db, ifexists)
+			# not found:
 			msg = "%s is not banned" % ip
 			logSys.log(logging.MSG, msg)
 			if ifexists:
@@ -311,23 +330,33 @@ class Actions(JailThread, Mapping):
 					self._jail.name, name, e,
 					exc_info=logSys.getEffectiveLevel()<=logging.DEBUG)
 		while self.active:
-			if self.idle:
-				logSys.debug("Actions: enter idle mode")
-				Utils.wait_for(lambda: not self.active or not self.idle,
-					lambda: False, self.sleeptime)
-				logSys.debug("Actions: leave idle mode")
-				continue
-			# wait for ban (stop if gets inactive):
-			bancnt = Utils.wait_for(lambda: not self.active or self.__checkBan(), self.sleeptime)
-			cnt += bancnt
-			# unban if nothing is banned not later than banned tickets >= banPrecedence
-			if not bancnt or cnt >= self.banPrecedence:
-				if self.active:
-					# let shrink the ban list faster
-					bancnt *= 2
-					self.__checkUnBan(bancnt if bancnt and bancnt < self.unbanMaxCount else self.unbanMaxCount)
-				cnt = 0
-		
+			try:
+				if self.idle:
+					logSys.debug("Actions: enter idle mode")
+					Utils.wait_for(lambda: not self.active or not self.idle,
+						lambda: False, self.sleeptime)
+					logSys.debug("Actions: leave idle mode")
+					continue
+				# wait for ban (stop if gets inactive, pending ban or unban):
+				bancnt = 0
+				wt = min(self.sleeptime, self.__banManager._nextUnbanTime - MyTime.time())
+				logSys.log(5, "Actions: wait for pending tickets %s (default %s)", wt, self.sleeptime)
+				if Utils.wait_for(lambda: not self.active or self._jail.hasFailTickets, wt):
+					bancnt = self.__checkBan()
+					cnt += bancnt
+				# unban if nothing is banned not later than banned tickets >= banPrecedence
+				if not bancnt or cnt >= self.banPrecedence:
+					if self.active:
+						# let shrink the ban list faster
+						bancnt *= 2
+						logSys.log(5, "Actions: check-unban %s, bancnt %s, max: %s", bancnt if bancnt and bancnt < self.unbanMaxCount else self.unbanMaxCount, bancnt, self.unbanMaxCount)
+						self.__checkUnBan(bancnt if bancnt and bancnt < self.unbanMaxCount else self.unbanMaxCount)
+					cnt = 0
+			except Exception as e: # pragma: no cover
+				logSys.error("[%s] unhandled error in actions thread: %s",
+					self._jail.name, e,
+					exc_info=logSys.getEffectiveLevel()<=logging.DEBUG)
+
 		self.__flushBan(stop=True)
 		self.stopActions()
 		return True
@@ -452,7 +481,7 @@ class Actions(JailThread, Mapping):
 				logSys.notice("[%s] %sBan %s", self._jail.name, ('' if not bTicket.restored else 'Restore '), ip)
 				for name, action in self._actions.iteritems():
 					try:
-						if ticket.restored and getattr(action, 'norestored', False):
+						if bTicket.restored and getattr(action, 'norestored', False):
 							continue
 						if not aInfo.immutable: aInfo.reset()
 						action.ban(aInfo)
@@ -495,6 +524,12 @@ class Actions(JailThread, Mapping):
 						cnt += self.__reBan(bTicket, actions=rebanacts)
 				else: # pragma: no cover - unexpected: ticket is not banned for some reasons - reban using all actions:
 					cnt += self.__reBan(bTicket)
+			# add ban to database:
+			if not bTicket.restored and self._jail.database is not None:
+				# ignore too old (repeated and ignored) tickets,
+				# [todo] replace it with inOperation later (once it gets back-ported):
+				if not reason and bTicket.getTime() >= MyTime.time() - 60:
+					self._jail.database.addBan(self._jail, bTicket)
 		if cnt:
 			logSys.debug("Banned %s / %s, %s ticket(s) in %r", cnt, 
 				self.__banManager.getBanTotal(), self.__banManager.size(), self._jail.name)
@@ -637,13 +672,19 @@ class Actions(JailThread, Mapping):
 		"""Status of current and total ban counts and current banned IP list.
 		"""
 		# TODO: Allow this list to be printed as 'status' output
-		supported_flavors = ["basic", "cymru"]
+		supported_flavors = ["short", "basic", "cymru"]
 		if flavor is None or flavor not in supported_flavors:
 			logSys.warning("Unsupported extended jail status flavor %r. Supported: %s" % (flavor, supported_flavors))
 		# Always print this information (basic)
-		ret = [("Currently banned", self.__banManager.size()),
-			   ("Total banned", self.__banManager.getBanTotal()),
-			   ("Banned IP list", self.__banManager.getBanList())]
+		if flavor != "short":
+			banned = self.__banManager.getBanList()
+			cnt = len(banned)
+		else:
+			cnt = self.__banManager.size()
+		ret = [("Currently banned", cnt),
+			   ("Total banned", self.__banManager.getBanTotal())]
+		if flavor != "short":
+			ret += [("Banned IP list", banned)]
 		if flavor == "cymru":
 			cymru_info = self.__banManager.getBanListExtendedCymruInfo()
 			ret += \
@@ -57,7 +57,7 @@ class BanManager:
 		## Total number of banned IP address
 		self.__banTotal = 0
 		## The time for next unban process (for performance and load reasons):
-		self.__nextUnbanTime = BanTicket.MAX_TIME
+		self._nextUnbanTime = BanTicket.MAX_TIME
 	
 	##
 	# Set the ban time.
@@ -66,8 +66,7 @@ class BanManager:
 	# @param value the time
 	
 	def setBanTime(self, value):
-		with self.__lock:
-			self.__banTime = int(value)
+		self.__banTime = int(value)
 	
 	##
 	# Get the ban time.
@@ -76,8 +75,7 @@ class BanManager:
 	# @return the time
 	
 	def getBanTime(self):
-		with self.__lock:
-			return self.__banTime
+		return self.__banTime
 	
 	##
 	# Set the total number of banned address.
@@ -85,8 +83,7 @@ class BanManager:
 	# @param value total number
 	
 	def setBanTotal(self, value):
-		with self.__lock:
-			self.__banTotal = value
+		self.__banTotal = value
 	
 	##
 	# Get the total number of banned address.
@@ -94,8 +91,7 @@ class BanManager:
 	# @return the total number
 	
 	def getBanTotal(self):
-		with self.__lock:
-			return self.__banTotal
+		return self.__banTotal

 	##
 	# Returns a copy of the IP list.
@@ -103,8 +99,7 @@ class BanManager:
 	# @return IP list
 	
 	def getBanList(self):
-		with self.__lock:
-			return self.__banList.keys()
+		return list(self.__banList.keys())

 	##
 	# Returns a iterator to ban list (used in reload, so idle).
@@ -112,8 +107,8 @@ class BanManager:
 	# @return ban list iterator
 	
 	def __iter__(self):
-		with self.__lock:
-			return self.__banList.itervalues()
+		# ensure iterator is safe - traverse over the list in snapshot created within lock (GIL):
+		return iter(list(self.__banList.values()))

 	##
 	# Returns normalized value
@@ -295,8 +290,8 @@ class BanManager:
 			self.__banList[fid] = ticket
 			self.__banTotal += 1
 			# correct next unban time:
-			if self.__nextUnbanTime > eob:
-				self.__nextUnbanTime = eob
+			if self._nextUnbanTime > eob:
+				self._nextUnbanTime = eob
 			return True

 	##
@@ -327,12 +322,8 @@ class BanManager:
 	
 	def unBanList(self, time, maxCount=0x7fffffff):
 		with self.__lock:
-			# Permanent banning
-			if self.__banTime < 0:
-				return list()
-
 			# Check next unban time:
-			nextUnbanTime = self.__nextUnbanTime
+			nextUnbanTime = self._nextUnbanTime
 			if nextUnbanTime > time:
 				return list()

@@ -345,12 +336,12 @@ class BanManager:
 				if time > eob:
 					unBanList[fid] = ticket
 					if len(unBanList) >= maxCount: # stop search cycle, so reset back the next check time
-						nextUnbanTime = self.__nextUnbanTime
+						nextUnbanTime = self._nextUnbanTime
 						break
 				elif nextUnbanTime > eob:
 					nextUnbanTime = eob

-			self.__nextUnbanTime = nextUnbanTime
+			self._nextUnbanTime = nextUnbanTime
 			# Removes tickets.
 			if len(unBanList):
 				if len(unBanList) / 2.0 <= len(self.__banList) / 3.0:
@@ -455,22 +455,24 @@ class Fail2BanDb(object):
 			If log was already present in database, value of last position
 			in the log file; else `None`
 		"""
+		return self._addLog(cur, jail, container.getFileName(), container.getPos(), container.getHash())
+
+	def _addLog(self, cur, jail, name, pos=0, md5=None):
 		lastLinePos = None
 		cur.execute(
 			"SELECT firstlinemd5, lastfilepos FROM logs "
 				"WHERE jail=? AND path=?",
-			(jail.name, container.getFileName()))
+			(jail.name, name))
 		try:
 			firstLineMD5, lastLinePos = cur.fetchone()
 		except TypeError:
-			firstLineMD5 = False
+			firstLineMD5 = None

-		cur.execute(
-				"INSERT OR REPLACE INTO logs(jail, path, firstlinemd5, lastfilepos) "
-					"VALUES(?, ?, ?, ?)",
-				(jail.name, container.getFileName(),
-					container.getHash(), container.getPos()))
-		if container.getHash() != firstLineMD5:
+		if not firstLineMD5 and (pos or md5):
+			cur.execute(
+					"INSERT OR REPLACE INTO logs(jail, path, firstlinemd5, lastfilepos) "
+						"VALUES(?, ?, ?, ?)", (jail.name, name, md5, pos))
+		if md5 is not None and md5 != firstLineMD5:
 			lastLinePos = None
 		return lastLinePos

@@ -499,7 +501,7 @@ class Fail2BanDb(object):
 		return set(row[0] for row in cur.fetchmany())

 	@commitandrollback
-	def updateLog(self, cur, *args, **kwargs):
+	def updateLog(self, cur, jail, container):
 		"""Updates hash and last position in log file.

 		Parameters
@@ -509,14 +511,48 @@ class Fail2BanDb(object):
 		container : FileContainer
 			File container of the log file being updated.
 		"""
-		self._updateLog(cur, *args, **kwargs)
+		self._updateLog(cur, jail, container.getFileName(), container.getPos(), container.getHash())

-	def _updateLog(self, cur, jail, container):
+	def _updateLog(self, cur, jail, name, pos, md5):
 		cur.execute(
 			"UPDATE logs SET firstlinemd5=?, lastfilepos=? "
-				"WHERE jail=? AND path=?",
-			(container.getHash(), container.getPos(),
-				jail.name, container.getFileName()))
+				"WHERE jail=? AND path=?", (md5, pos, jail.name, name))
+		# be sure it is set (if not available):
+		if not cur.rowcount:
+			cur.execute(
+					"INSERT OR REPLACE INTO logs(jail, path, firstlinemd5, lastfilepos) "
+						"VALUES(?, ?, ?, ?)", (jail.name, name, md5, pos))
+
+	@commitandrollback
+	def getJournalPos(self, cur, jail, name, time=0, iso=None):
+		"""Get journal position from database.
+
+		Parameters
+		----------
+		jail : Jail
+			Jail of which the journal belongs to.
+		name, time, iso :
+			Journal name (typically systemd-journal) and last known time.
+
+		Returns
+		-------
+		int (or float)
+			Last position (as time) if it was already present in database; else `None`
+		"""
+		return self._addLog(cur, jail, name, time, iso); # no hash, just time as iso
+
+	@commitandrollback
+	def updateJournal(self, cur, jail, name, time, iso):
+		"""Updates last position (as time) of journal.
+
+		Parameters
+		----------
+		jail : Jail
+			Jail of which the journal belongs to.
+		name, time, iso :
+			Journal name (typically systemd-journal) and last known time.
+		"""
+		self._updateLog(cur, jail, name, time, iso); # no hash, just time as iso

 	@commitandrollback
 	def addBan(self, cur, jail, ticket):
@@ -35,7 +35,7 @@ from ..helpers import getLogger
 # Gets the instance of the logger.
 logSys = getLogger(__name__)

-logLevel = 6
+logLevel = 5

 RE_DATE_PREMATCH = re.compile(r"(?<!\\)\{DATE\}", re.IGNORECASE)
 DD_patternCache = Utils.Cache(maxCount=1000, maxTime=60*60)
@@ -282,6 +282,8 @@ class DateDetector(object):
 				elif "{DATE}" in key:
 					self.addDefaultTemplate(preMatch=pattern, allDefaults=False)
 					return
+				elif key == "{NONE}":
+					template = _getPatternTemplate('{UNB}^', key)
 				else:
 					template = _getPatternTemplate(pattern, key)

@@ -337,65 +339,76 @@ class DateDetector(object):
 		# if no templates specified - default templates should be used:
 		if not len(self.__templates):
 			self.addDefaultTemplate()
-		logSys.log(logLevel-1, "try to match time for line: %.120s", line)
-		match = None
+		log = logSys.log if logSys.getEffectiveLevel() <= logLevel else lambda *args: None
+		log(logLevel-1, "try to match time for line: %.120s", line)
+
 		# first try to use last template with same start/end position:
+		match = None
+		found = None, 0x7fffffff, 0x7fffffff, -1
 		ignoreBySearch = 0x7fffffff
 		i = self.__lastTemplIdx
 		if i < len(self.__templates):
 			ddtempl = self.__templates[i]
 			template = ddtempl.template
 			if template.flags & (DateTemplate.LINE_BEGIN|DateTemplate.LINE_END):
-				if logSys.getEffectiveLevel() <= logLevel-1: # pragma: no cover - very-heavy debug
-					logSys.log(logLevel-1, "  try to match last anchored template #%02i ...", i)
+				log(logLevel-1, "  try to match last anchored template #%02i ...", i)
 				match = template.matchDate(line)
 				ignoreBySearch = i
 			else:
 				distance, endpos = self.__lastPos[0], self.__lastEndPos[0]
-				if logSys.getEffectiveLevel() <= logLevel-1:
-					logSys.log(logLevel-1, "  try to match last template #%02i (from %r to %r): ...%r==%r %s %r==%r...",
-						i, distance, endpos, 
-						line[distance-1:distance], self.__lastPos[1],
-						line[distance:endpos],
-						line[endpos:endpos+1], self.__lastEndPos[1])
-				# check same boundaries left/right, otherwise possible collision/pattern switch:
-				if (line[distance-1:distance] == self.__lastPos[1] and 
-						line[endpos:endpos+1] == self.__lastEndPos[1]
-				):
+				log(logLevel-1, "  try to match last template #%02i (from %r to %r): ...%r==%r %s %r==%r...",
+					i, distance, endpos,
+					line[distance-1:distance], self.__lastPos[1],
+					line[distance:endpos],
+					line[endpos:endpos+1], self.__lastEndPos[2])
+				# check same boundaries left/right, outside fully equal, inside only if not alnum (e. g. bound RE
+				# with space or some special char), otherwise possible collision/pattern switch:
+				if ((
+					line[distance-1:distance] == self.__lastPos[1] or
+						(line[distance:distance+1] == self.__lastPos[2] and not self.__lastPos[2].isalnum())
+					) and (
+					line[endpos:endpos+1] == self.__lastEndPos[2] or
+						(line[endpos-1:endpos] == self.__lastEndPos[1] and not self.__lastEndPos[1].isalnum())
+				)):
+					# search in line part only:
+					log(logLevel-1, "  boundaries are correct, search in part %r", line[distance:endpos])
 					match = template.matchDate(line, distance, endpos)
+				else:
+					log(logLevel-1, "  boundaries show conflict, try whole search")
+					match = template.matchDate(line)
+					ignoreBySearch = i
 			if match:
 				distance = match.start()
 				endpos = match.end()
 				# if different position, possible collision/pattern switch:
 				if (
+					len(self.__templates) == 1 or # single template:
 					template.flags & (DateTemplate.LINE_BEGIN|DateTemplate.LINE_END) or 
 					(distance == self.__lastPos[0] and endpos == self.__lastEndPos[0])
 				):
-					logSys.log(logLevel, "  matched last time template #%02i", i)
+					log(logLevel, "  matched last time template #%02i", i)
 				else:
-					logSys.log(logLevel, "  ** last pattern collision - pattern change, search ...")
+					log(logLevel, "  ** last pattern collision - pattern change, reserve & search ...")
+					found = match, distance, endpos, i; # save current best alternative
 					match = None
 			else:
-				logSys.log(logLevel, "  ** last pattern not found - pattern change, search ...")
+				log(logLevel, "  ** last pattern not found - pattern change, search ...")
 		# search template and better match:
 		if not match:
-			logSys.log(logLevel, " search template (%i) ...", len(self.__templates))
-			found = None, 0x7fffffff, 0x7fffffff, -1
+			log(logLevel, " search template (%i) ...", len(self.__templates))
 			i = 0
 			for ddtempl in self.__templates:
-				if logSys.getEffectiveLevel() <= logLevel-1:
-					logSys.log(logLevel-1, "  try template #%02i: %s", i, ddtempl.name)
 				if i == ignoreBySearch:
 					i += 1
 					continue
+				log(logLevel-1, "  try template #%02i: %s", i, ddtempl.name)
 				template = ddtempl.template
 				match = template.matchDate(line)
 				if match:
 					distance = match.start()
 					endpos = match.end()
-					if logSys.getEffectiveLevel() <= logLevel:
-						logSys.log(logLevel, "  matched time template #%02i (at %r <= %r, %r) %s",
-							i, distance, ddtempl.distance, self.__lastPos[0], template.name)
+					log(logLevel, "  matched time template #%02i (at %r <= %r, %r) %s",
+						i, distance, ddtempl.distance, self.__lastPos[0], template.name)
 					## last (or single) template - fast stop:
 					if i+1 >= len(self.__templates):
 						break
@@ -408,7 +421,7 @@ class DateDetector(object):
 					## [grave] if distance changed, possible date-match was found somewhere 
 					## in body of message, so save this template, and search further:
 					if distance > ddtempl.distance or distance > self.__lastPos[0]:
-						logSys.log(logLevel, "  ** distance collision - pattern change, reserve")
+						log(logLevel, "  ** distance collision - pattern change, reserve")
 						## shortest of both:
 						if distance < found[1]:
 							found = match, distance, endpos, i
@@ -422,7 +435,7 @@ class DateDetector(object):
 			# check other template was found (use this one with shortest distance):
 			if not match and found[0]:
 				match, distance, endpos, i = found
-				logSys.log(logLevel, "  use best time template #%02i", i)
+				log(logLevel, "  use best time template #%02i", i)
 				ddtempl = self.__templates[i]
 				template = ddtempl.template
 		# we've winner, incr hits, set distance, usage, reorder, etc:
@@ -432,8 +445,8 @@ class DateDetector(object):
 			ddtempl.distance = distance
 			if self.__firstUnused == i:
 				self.__firstUnused += 1
-			self.__lastPos = distance, line[distance-1:distance]
-			self.__lastEndPos = endpos, line[endpos:endpos+1]
+			self.__lastPos = distance, line[distance-1:distance], line[distance]
+			self.__lastEndPos = endpos, line[endpos-1], line[endpos:endpos+1]
 			# if not first - try to reorder current template (bubble up), they will be not sorted anymore:
 			if i and i != self.__lastTemplIdx:
 				i = self._reorderTemplate(i)
@@ -442,7 +455,7 @@ class DateDetector(object):
 			return (match, template)

 		# not found:
-		logSys.log(logLevel, " no template.")
+		log(logLevel, " no template.")
 		return (None, None)

 	@property
@@ -36,15 +36,16 @@ logSys = getLogger(__name__)
 RE_GROUPED = re.compile(r'(?<!(?:\(\?))(?<!\\)\((?!\?)')
 RE_GROUP = ( re.compile(r'^((?:\(\?\w+\))?\^?(?:\(\?\w+\))?)(.*?)(\$?)$'), r"\1(\2)\3" )

+RE_EXLINE_NO_BOUNDS = re.compile(r'^\{UNB\}')
 RE_EXLINE_BOUND_BEG = re.compile(r'^\{\^LN-BEG\}')
-RE_EXSANC_BOUND_BEG = re.compile(r'^\(\?:\^\|\\b\|\\W\)')
+RE_EXSANC_BOUND_BEG = re.compile(r'^\((?:\?:)?\^\|\\b\|\\W\)')
 RE_EXEANC_BOUND_BEG = re.compile(r'\(\?=\\b\|\\W\|\$\)$')
-RE_NO_WRD_BOUND_BEG = re.compile(r'^\(*(?:\(\?\w+\))?(?:\^|\(*\*\*|\(\?:\^)')
+RE_NO_WRD_BOUND_BEG = re.compile(r'^\(*(?:\(\?\w+\))?(?:\^|\(*\*\*|\((?:\?:)?\^)')
 RE_NO_WRD_BOUND_END = re.compile(r'(?<!\\)(?:\$\)?|\\b|\\s|\*\*\)*)$')
 RE_DEL_WRD_BOUNDS = ( re.compile(r'^\(*(?:\(\?\w+\))?\(*\*\*|(?<!\\)\*\*\)*$'), 
 	                    lambda m: m.group().replace('**', '') )

-RE_LINE_BOUND_BEG = re.compile(r'^(?:\(\?\w+\))?(?:\^|\(\?:\^(?!\|))')
+RE_LINE_BOUND_BEG = re.compile(r'^(?:\(\?\w+\))?(?:\^|\((?:\?:)?\^(?!\|))')
 RE_LINE_BOUND_END = re.compile(r'(?<![\\\|])(?:\$\)?)$')

 RE_ALPHA_PATTERN = re.compile(r'(?<!\%)\%[aAbBpc]')
@@ -119,7 +120,7 @@ class DateTemplate(object):
 		if boundBegin:
 			self.flags |= DateTemplate.WORD_BEGIN if wordBegin != 'start' else DateTemplate.LINE_BEGIN
 			if wordBegin != 'start':
-				regex = r'(?:^|\b|\W)' + regex
+				regex = r'(?=^|\b|\W)' + regex
 			else:
 				regex = r"^(?:\W{0,2})?" + regex
 				if not self.name.startswith('{^LN-BEG}'):
@@ -128,12 +129,14 @@ class DateTemplate(object):
 		if boundEnd:
 			self.flags |= DateTemplate.WORD_END
 			regex += r'(?=\b|\W|$)'
-		if RE_LINE_BOUND_BEG.search(regex): self.flags |= DateTemplate.LINE_BEGIN
-		if RE_LINE_BOUND_END.search(regex): self.flags |= DateTemplate.LINE_END
+		if not (self.flags & DateTemplate.LINE_BEGIN) and RE_LINE_BOUND_BEG.search(regex):
+			self.flags |= DateTemplate.LINE_BEGIN
+		if not (self.flags & DateTemplate.LINE_END) and RE_LINE_BOUND_END.search(regex):
+			self.flags |= DateTemplate.LINE_END
 		# remove possible special pattern "**" in front and end of regex:
 		regex = RE_DEL_WRD_BOUNDS[0].sub(RE_DEL_WRD_BOUNDS[1], regex)
 		self._regex = regex
-		logSys.log(7, '  constructed regex %s', regex)
+		logSys.log(4, '  constructed regex %s', regex)
 		self._cRegex = None

 	regex = property(getRegex, setRegex, doc=
@@ -156,6 +159,7 @@ class DateTemplate(object):
 		"""
 		if not self._cRegex:
 			self._compileRegex()
+		logSys.log(4, "   search %s", self.regex)
 		dateMatch = self._cRegex.search(line, *args); # pos, endpos
 		if dateMatch:
 			self.hits += 1
@@ -188,7 +192,7 @@ class DateTemplate(object):
 	def unboundPattern(pattern):
 		return RE_EXEANC_BOUND_BEG.sub('',
 			RE_EXSANC_BOUND_BEG.sub('',
-				RE_EXLINE_BOUND_BEG.sub('', pattern)
+				RE_EXLINE_BOUND_BEG.sub('', RE_EXLINE_NO_BOUNDS.sub('', pattern))
 			)
 		)

@@ -297,6 +301,10 @@ class DatePatternRegex(DateTemplate):
 	def setRegex(self, pattern, wordBegin=True, wordEnd=True):
 		# original pattern:
 		self._pattern = pattern
+		# if unbound signalled - reset boundaries left and right:
+		if RE_EXLINE_NO_BOUNDS.search(pattern):
+			pattern = RE_EXLINE_NO_BOUNDS.sub('', pattern)
+			wordBegin = wordEnd = False
 		# if explicit given {^LN-BEG} - remove it from pattern and set 'start' in wordBegin:
 		if wordBegin and RE_EXLINE_BOUND_BEG.search(pattern):
 			pattern = RE_EXLINE_BOUND_BEG.sub('', pattern)
@@ -43,26 +43,20 @@ class FailManager:
 		self.__maxRetry = 3
 		self.__maxTime = 600
 		self.__failTotal = 0
-		self.maxMatches = 50
+		self.maxMatches = 5
 		self.__bgSvc = BgService()
 	
 	def setFailTotal(self, value):
-		with self.__lock:
-			self.__failTotal = value
+		self.__failTotal = value
 		
 	def getFailTotal(self):
-		with self.__lock:
-			return self.__failTotal
+		return self.__failTotal
 	
 	def getFailCount(self):
 		# may be slow on large list of failures, should be used for test purposes only...
 		with self.__lock:
 			return len(self.__failList), sum([f.getRetry() for f in self.__failList.values()])

-	def getFailTotal(self):
-		with self.__lock:
-			return self.__failTotal
-
 	def setMaxRetry(self, value):
 		self.__maxRetry = value
 	
@@ -92,10 +86,7 @@ class FailManager:
 					if attempt <= 0:
 						attempt += 1
 				unixTime = ticket.getTime()
-				fData.setLastTime(unixTime)
-				if fData.getLastReset() < unixTime - self.__maxTime:
-					fData.setLastReset(unixTime)
-					fData.setRetry(0)
+				fData.adjustTime(unixTime, self.__maxTime)
 				fData.inc(matches, attempt, count)
 				# truncate to maxMatches:
 				if self.maxMatches:
@@ -130,13 +121,13 @@ class FailManager:
 		return attempts
 	
 	def size(self):
-		with self.__lock:
-			return len(self.__failList)
+		return len(self.__failList)
 	
 	def cleanup(self, time):
+		time -= self.__maxTime
 		with self.__lock:
 			todelete = [fid for fid,item in self.__failList.iteritems() \
-				if item.getLastTime() + self.__maxTime <= time]
+				if item.getTime() <= time]
 			if len(todelete) == len(self.__failList):
 				# remove all:
 				self.__failList = dict()
@@ -150,7 +141,7 @@ class FailManager:
 			else:
 				# create new dictionary without items to be deleted:
 				self.__failList = dict((fid,item) for fid,item in self.__failList.iteritems() \
-					if item.getLastTime() + self.__maxTime > time)
+					if item.getTime() > time)
 		self.__bgSvc.service()
 	
 	def delFailure(self, fid):
@@ -138,6 +138,8 @@ class Regex:
 		except sre_constants.error:
 			raise RegexException("Unable to compile regular expression '%s'" %
 								 regex)
+		# set fetch handler depending on presence of alternate tags:
+		self.getGroups = self._getGroupsWithAlt if self._altValues else self._getGroups

 	def __str__(self):
 		return "%s(%r)" % (self.__class__.__name__, self._regex)
@@ -277,11 +279,12 @@ class Regex:
 	# Returns all matched groups.
 	#

-	def getGroups(self):
-		if not self._altValues:
-			return self._matchCache.groupdict()
-		# merge alternate values (e. g. 'alt_user_1' -> 'user' or 'alt_host' -> 'host'):
+	def _getGroups(self):
+		return self._matchCache.groupdict()
+
+	def _getGroupsWithAlt(self):
 		fail = self._matchCache.groupdict()
+		# merge alternate values (e. g. 'alt_user_1' -> 'user' or 'alt_host' -> 'host'):
 		#fail = fail.copy()
 		for k,n in self._altValues:
 			v = fail.get(k)
@@ -289,6 +292,9 @@ class Regex:
 				fail[n] = v
 		return fail

+	def getGroups(self): # pragma: no cover - abstract function (replaced in __init__)
+		pass
+
 	##
 	# Returns skipped lines.
 	#
@@ -80,6 +80,7 @@ class Filter(JailThread):
 		## Ignore own IPs flag:
 		self.__ignoreSelf = True
 		## The ignore IP list.
+		self.__ignoreIpSet = set()
 		self.__ignoreIpList = []
 		## External command
 		self.__ignoreCommand = False
@@ -92,6 +93,8 @@ class Filter(JailThread):
 		## Store last time stamp, applicable for multi-line
 		self.__lastTimeText = ""
 		self.__lastDate = None
+		## Next service (cleanup) time
+		self.__nextSvcTime = -(1<<63)
 		## if set, treat log lines without explicit time zone to be in this time zone
 		self.__logtimezone = None
 		## Default or preferred encoding (to decode bytes from file or journal):
@@ -101,14 +104,26 @@ class Filter(JailThread):
 		## Error counter (protected, so can be used in filter implementations)
 		## if it reached 100 (at once), run-cycle will go idle
 		self._errors = 0
+		## Next time to update log or journal position in database:
+		self._nextUpdateTM = 0
+		## Pending updates (must be executed at next update time or during stop):
+		self._pendDBUpdates = {}
 		## return raw host (host is not dns):
 		self.returnRawHost = False
 		## check each regex (used for test purposes):
 		self.checkAllRegex = False
+		## avoid finding of pending failures (without ID/IP, used in fail2ban-regex):
+		self.ignorePending = True
+		## callback called on ignoreregex match :
+		self.onIgnoreRegex = None
 		## if true ignores obsolete failures (failure time < now - findTime):
 		self.checkFindTime = True
+		## shows that filter is in operation mode (processing new messages):
+		self.inOperation = True
 		## Ticks counter
 		self.ticks = 0
+		## Processed lines counter
+		self.procLines = 0
 		## Thread name:
 		self.name="f2b/f."+self.jailName

@@ -168,7 +183,7 @@ class Filter(JailThread):
 	# @param value the regular expression

 	def addFailRegex(self, value):
-		multiLine = self.getMaxLines() > 1
+		multiLine = self.__lineBufferSize > 1
 		try:
 			regex = FailRegex(value, prefRegex=self.__prefRegex, multiline=multiLine,
 				useDns=self.__useDns)
@@ -432,12 +447,23 @@ class Filter(JailThread):

 	def performBan(self, ip=None):
 		"""Performs a ban for IPs (or given ip) that are reached maxretry of the jail."""
-		try: # pragma: no branch - exception is the only way out
-			while True:
+		while True:
+			try:
 				ticket = self.failManager.toBan(ip)
-				self.jail.putFailTicket(ticket)
-		except FailManagerEmpty:
-			self.failManager.cleanup(MyTime.time())
+			except FailManagerEmpty:
+				break
+			self.jail.putFailTicket(ticket)
+			if ip: break
+		self.performSvc()
+
+	def performSvc(self, force=False):
+		"""Performs a service tasks (clean failure list)."""
+		tm = MyTime.time()
+		# avoid too early clean up:
+		if force or tm >= self.__nextSvcTime:
+			self.__nextSvcTime = tm + 5
+			# clean up failure list:
+			self.failManager.cleanup(tm)

 	def addAttempt(self, ip, *matches):
 		"""Generate a failed attempt for ip"""
@@ -451,10 +477,10 @@ class Filter(JailThread):
 		logSys.info(
 			"[%s] Attempt %s - %s", self.jailName, ip, datetime.datetime.fromtimestamp(unixTime).strftime("%Y-%m-%d %H:%M:%S")
 		)
-		self.failManager.addFailure(ticket, len(matches) or 1)
-
+		attempts = self.failManager.addFailure(ticket, len(matches) or 1)
 		# Perform the ban if this attempt is resulted to:
-		self.performBan(ip)
+		if attempts >= self.failManager.getMaxRetry():
+			self.performBan(ip)

 		return 1

@@ -483,28 +509,36 @@ class Filter(JailThread):
 		# Create IP address object
 		ip = IPAddr(ipstr)
 		# Avoid exact duplicates
-		if ip in self.__ignoreIpList:
-			logSys.warn("  Ignore duplicate %r (%r), already in ignore list", ip, ipstr)
+		if ip in self.__ignoreIpSet or ip in self.__ignoreIpList:
+			logSys.log(logging.MSG, "  Ignore duplicate %r (%r), already in ignore list", ip, ipstr)
 			return
 		# log and append to ignore list
 		logSys.debug("  Add %r to ignore list (%r)", ip, ipstr)
-		self.__ignoreIpList.append(ip)
+		# if single IP (not DNS or a subnet) add to set, otherwise to list:
+		if ip.isSingle:
+			self.__ignoreIpSet.add(ip)
+		else:
+			self.__ignoreIpList.append(ip)

 	def delIgnoreIP(self, ip=None):
 		# clear all:
 		if ip is None:
+			self.__ignoreIpSet.clear()
 			del self.__ignoreIpList[:]
 			return
 		# delete by ip:
 		logSys.debug("  Remove %r from ignore list", ip)
-		self.__ignoreIpList.remove(ip)
+		if ip in self.__ignoreIpSet:
+			self.__ignoreIpSet.remove(ip)
+		else:
+			self.__ignoreIpList.remove(ip)

 	def logIgnoreIp(self, ip, log_ignore, ignore_source="unknown source"):
 		if log_ignore:
 			logSys.info("[%s] Ignore %s by %s", self.jailName, ip, ignore_source)

 	def getIgnoreIP(self):
-		return self.__ignoreIpList
+		return self.__ignoreIpList + list(self.__ignoreIpSet)

 	##
 	# Check if IP address/DNS is in the ignore list.
@@ -544,8 +578,11 @@ class Filter(JailThread):
 			if self.__ignoreCache: c.set(key, True)
 			return True

+		# check if the IP is covered by ignore IP (in set or in subnet/dns):
+		if ip in self.__ignoreIpSet:
+			self.logIgnoreIp(ip, log_ignore, ignore_source="ip")
+			return True
 		for net in self.__ignoreIpList:
-			# check if the IP is covered by ignore IP
 			if ip.isInNet(net):
 				self.logIgnoreIp(ip, log_ignore, ignore_source=("ip" if net.isValid else "dns"))
 				if self.__ignoreCache: c.set(key, True)
@@ -568,29 +605,97 @@ class Filter(JailThread):
 		if self.__ignoreCache: c.set(key, False)
 		return False

+	def _logWarnOnce(self, nextLTM, *args):
+		"""Log some issue as warning once per day, otherwise level 7"""
+		if MyTime.time() < getattr(self, nextLTM, 0):
+			if logSys.getEffectiveLevel() <= 7: logSys.log(7, *(args[0]))
+		else:
+			setattr(self, nextLTM, MyTime.time() + 24*60*60)
+			for args in args:
+				logSys.warning('[%s] ' + args[0], self.jailName, *args[1:])
+
 	def processLine(self, line, date=None):
 		"""Split the time portion from log msg and return findFailures on them
 		"""
+		logSys.log(7, "Working on line %r", line)
+
+		noDate = False
 		if date:
 			tupleLine = line
+			line = "".join(line)
+			self.__lastTimeText = tupleLine[1]
+			self.__lastDate = date
 		else:
-			l = line.rstrip('\r\n')
-			logSys.log(7, "Working on line %r", line)
-
-			(timeMatch, template) = self.dateDetector.matchTime(l)
-			if timeMatch:
-				tupleLine  = (
-					l[:timeMatch.start(1)],
-					l[timeMatch.start(1):timeMatch.end(1)],
-					l[timeMatch.end(1):],
-					(timeMatch, template)
-				)
+			# try to parse date:
+			timeMatch = self.dateDetector.matchTime(line)
+			m = timeMatch[0]
+			if m:
+				s = m.start(1)
+				e = m.end(1)
+				m = line[s:e]
+				tupleLine = (line[:s], m, line[e:])
+				if m: # found and not empty - retrive date:
+					date = self.dateDetector.getTime(m, timeMatch)
+					if date is not None:
+						# Lets get the time part
+						date = date[0]
+						self.__lastTimeText = m
+						self.__lastDate = date
+					else:
+						logSys.error("findFailure failed to parse timeText: %s", m)
+				# matched empty value - date is optional or not available - set it to last known or now:
+				elif self.__lastDate and self.__lastDate > MyTime.time() - 60:
+					# set it to last known:
+					tupleLine = ("", self.__lastTimeText, line)
+					date = self.__lastDate
+				else:
+					# set it to now:
+					date = MyTime.time()
 			else:
-				tupleLine = (l, "", "", None)
+				tupleLine = ("", "", line)
+			# still no date - try to use last known:
+			if date is None:
+				noDate = True
+				if self.__lastDate and self.__lastDate > MyTime.time() - 60:
+					tupleLine = ("", self.__lastTimeText, line)
+					date = self.__lastDate
+				elif self.checkFindTime and self.inOperation:
+					date = MyTime.time()
+		
+		if self.checkFindTime and date is not None:
+			# if in operation (modifications have been really found):
+			if self.inOperation:
+				# if weird date - we'd simulate now for timeing issue (too large deviation from now):
+				delta = int(date - MyTime.time())
+				if abs(delta) > 60:
+					# log timing issue as warning once per day:
+					self._logWarnOnce("_next_simByTimeWarn",
+						("Detected a log entry %s %s the current time in operation mode. "
+						 "This looks like a %s problem. Treating such entries as if they just happened.",
+						 MyTime.seconds2str(abs(delta)), "before" if delta < 0 else "after",
+						 "latency" if -3300 <= delta < 0 else "timezone"
+						 ),
+						("Please check a jail for a timing issue. Line with odd timestamp: %s",
+						 line))
+					# simulate now as date:
+					date = MyTime.time()
+					self.__lastDate = date
+			else:
+				# in initialization (restore) phase, if too old - ignore:
+				if date < MyTime.time() - self.getFindTime():
+					# log time zone issue as warning once per day:
+					self._logWarnOnce("_next_ignByTimeWarn",
+						("Ignoring all log entries older than %ss; these are probably" +
+						 " messages generated while fail2ban was not running.",
+							self.getFindTime()),
+						("Please check a jail for a timing issue. Line with odd timestamp: %s",
+						 line))
+					# ignore - too old (obsolete) entry:
+					return []

 		# save last line (lazy convert of process line tuple to string on demand):
 		self.processedLine = lambda: "".join(tupleLine[::2])
-		return self.findFailure(tupleLine, date)
+		return self.findFailure(tupleLine, date, noDate=noDate)

 	def processLineAndAdd(self, line, date=None):
 		"""Processes the line for failures and populates failManager
@@ -602,13 +707,24 @@ class Filter(JailThread):
 				fail = element[3]
 				logSys.debug("Processing line with time:%s and ip:%s", 
 						unixTime, ip)
+				# ensure the time is not in the future, e. g. by some estimated (assumed) time:
+				if self.checkFindTime and unixTime > MyTime.time():
+					unixTime = MyTime.time()
 				tick = FailTicket(ip, unixTime, data=fail)
 				if self._inIgnoreIPList(ip, tick):
 					continue
 				logSys.info(
 					"[%s] Found %s - %s", self.jailName, ip, datetime.datetime.fromtimestamp(unixTime).strftime("%Y-%m-%d %H:%M:%S")
 				)
-				self.failManager.addFailure(tick)
+				attempts = self.failManager.addFailure(tick)
+				# avoid RC on busy filter (too many failures) - if attempts for IP/ID reached maxretry,
+				# we can speedup ban, so do it as soon as possible:
+				if attempts >= self.failManager.getMaxRetry():
+					self.performBan(ip)
+			self.procLines += 1
+			# every 100 lines check need to perform service tasks:
+			if self.procLines % 100 == 0:
+				self.performSvc()
 			# reset (halve) error counter (successfully processed line):
 			if self._errors:
 				self._errors //= 2
@@ -618,7 +734,7 @@ class Filter(JailThread):
 			# incr common error counter:
 			self.commonError()

-	def commonError(self):
+	def commonError(self, reason="common", exc=None):
 		# incr error counter, stop processing (going idle) after 100th error :
 		self._errors += 1
 		# sleep a little bit (to get around time-related errors):
@@ -628,20 +744,26 @@ class Filter(JailThread):
 			self._errors //= 2
 			self.idle = True

-	##
-	# Returns true if the line should be ignored.
-	#
-	# Uses ignoreregex.
-	# @param line: the line
-	# @return: a boolean
-
-	def ignoreLine(self, tupleLines):
-		buf = Regex._tupleLinesBuf(tupleLines)
+	def _ignoreLine(self, buf, orgBuffer, failRegex=None):
+		# if multi-line buffer - use matched only, otherwise (single line) - original buf:
+		if failRegex and self.__lineBufferSize > 1:
+			orgBuffer = failRegex.getMatchedTupleLines()
+			buf = Regex._tupleLinesBuf(orgBuffer)
+		# search ignored:
+		fnd = None
 		for ignoreRegexIndex, ignoreRegex in enumerate(self.__ignoreRegex):
-			ignoreRegex.search(buf, tupleLines)
+			ignoreRegex.search(buf, orgBuffer)
 			if ignoreRegex.hasMatched():
-				return ignoreRegexIndex
-		return None
+				fnd = ignoreRegexIndex
+				logSys.log(7, "  Matched ignoreregex %d and was ignored", fnd)
+				if self.onIgnoreRegex: self.onIgnoreRegex(fnd, ignoreRegex)
+				# remove ignored match:
+				if not self.checkAllRegex or self.__lineBufferSize > 1:
+					# todo: check ignoreRegex.getUnmatchedTupleLines() would be better (fix testGetFailuresMultiLineIgnoreRegex):
+					if failRegex:
+						self.__lineBuffer = failRegex.getUnmatchedTupleLines()
+				if not self.checkAllRegex: break
+		return fnd

 	def _updateUsers(self, fail, user=()):
 		users = fail.get('users')
@@ -651,54 +773,31 @@ class Filter(JailThread):
 				fail['users'] = users = set()
 			users.add(user)
 			return users
-		return None
-
-	# # ATM incremental (non-empty only) merge deactivated ...
-	# @staticmethod
-	# def _updateFailure(self, mlfidGroups, fail):
-	# 	# reset old failure-ids when new types of id available in this failure:
-	# 	fids = set()
-	# 	for k in ('fid', 'ip4', 'ip6', 'dns'):
-	# 		if fail.get(k):
-	# 			fids.add(k)
-	# 	if fids:
-	# 		for k in ('fid', 'ip4', 'ip6', 'dns'):
-	# 			if k not in fids:
-	# 				try:
-	# 					del mlfidGroups[k]
-	# 				except:
-	# 					pass
-	# 	# update not empty values:
-	# 	mlfidGroups.update(((k,v) for k,v in fail.iteritems() if v))
+		return users

 	def _mergeFailure(self, mlfid, fail, failRegex):
 		mlfidFail = self.mlfidCache.get(mlfid) if self.__mlfidCache else None
 		users = None
 		nfflgs = 0
 		if fail.get("mlfgained"):
-			nfflgs |= 9
+			nfflgs |= (8|1)
 			if not fail.get('nofail'):
 				fail['nofail'] = fail["mlfgained"]
 		elif fail.get('nofail'): nfflgs |= 1
-		if fail.get('mlfforget'): nfflgs |= 2
+		if fail.pop('mlfforget', None): nfflgs |= 2
 		# if multi-line failure id (connection id) known:
 		if mlfidFail:
 			mlfidGroups = mlfidFail[1]
 			# update users set (hold all users of connect):
 			users = self._updateUsers(mlfidGroups, fail.get('user'))
 			# be sure we've correct current state ('nofail' and 'mlfgained' only from last failure)
-			try:
-				del mlfidGroups['nofail']
-				del mlfidGroups['mlfgained']
-			except KeyError:
-				pass
-			# # ATM incremental (non-empty only) merge deactivated (for future version only),
-			# # it can be simulated using alternate value tags, like <F-ALT_VAL>...</F-ALT_VAL>,
-			# # so previous value 'val' will be overwritten only if 'alt_val' is not empty...		
-			# _updateFailure(mlfidGroups, fail)
-			#
+			if mlfidGroups.pop('nofail', None): nfflgs |= 4
+			if mlfidGroups.pop('mlfgained', None): nfflgs |= 4
+			# if we had no pending failures then clear the matches (they are already provided):
+			if (nfflgs & 4) == 0 and not mlfidGroups.get('mlfpending', 0):
+				mlfidGroups.pop("matches", None)
 			# overwrite multi-line failure with all values, available in fail:
-			mlfidGroups.update(fail)
+			mlfidGroups.update(((k,v) for k,v in fail.iteritems() if v is not None))
 			# new merged failure data:
 			fail = mlfidGroups
 			# if forget (disconnect/reset) - remove cached entry:
@@ -709,24 +808,19 @@ class Filter(JailThread):
 			mlfidFail = [self.__lastDate, fail]
 			self.mlfidCache.set(mlfid, mlfidFail)
 		# check users in order to avoid reset failure by multiple logon-attempts:
-		if users and len(users) > 1:
-			# we've new user, reset 'nofail' because of multiple users attempts:
-			try:
-				del fail['nofail']
-				nfflgs &= ~1 # reset nofail
-			except KeyError:
-				pass
+		if fail.pop('mlfpending', 0) or users and len(users) > 1:
+			# we've pending failures or new user, reset 'nofail' because of failures or multiple users attempts:
+			fail.pop('nofail', None)
+			fail.pop('mlfgained', None)
+			nfflgs &= ~(8|1) # reset nofail and gained
 		# merge matches:
-		if not (nfflgs & 1): # current nofail state (corresponding users)
-			try:
-				m = fail.pop("nofail-matches")
-				m += fail.get("matches", [])
-			except KeyError:
-				m = fail.get("matches", [])
-			if not (nfflgs & 8): # no gain signaled
+		if (nfflgs & 1) == 0: # current nofail state (corresponding users)
+			m = fail.pop("nofail-matches", [])
+			m += fail.get("matches", [])
+			if (nfflgs & 8) == 0: # no gain signaled
 				m += failRegex.getMatchedTupleLines()
 			fail["matches"] = m
-		elif not (nfflgs & 2) and (nfflgs & 1): # not mlfforget and nofail:
+		elif (nfflgs & 3) == 1: # not mlfforget and nofail:
 			fail["nofail-matches"] = fail.get("nofail-matches", []) + failRegex.getMatchedTupleLines()
 		# return merged:
 		return fail
@@ -739,7 +833,7 @@ class Filter(JailThread):
 	# to find the logging time.
 	# @return a dict with IP and timestamp.

-	def findFailure(self, tupleLine, date=None):
+	def findFailure(self, tupleLine, date, noDate=False):
 		failList = list()

 		ll = logSys.getEffectiveLevel()
@@ -749,62 +843,33 @@ class Filter(JailThread):
 			returnRawHost = True
 			cidr = IPAddr.CIDR_RAW

-		# Checks if we mut ignore this line.
-		if self.ignoreLine([tupleLine[::2]]) is not None:
-			# The ignoreregex matched. Return.
-			if ll <= 7: logSys.log(7, "Matched ignoreregex and was \"%s\" ignored",
-				"".join(tupleLine[::2]))
-			return failList
-
-		timeText = tupleLine[1]
-		if date:
-			self.__lastTimeText = timeText
-			self.__lastDate = date
-		elif timeText:
-
-			dateTimeMatch = self.dateDetector.getTime(timeText, tupleLine[3])
-
-			if dateTimeMatch is None:
-				logSys.error("findFailure failed to parse timeText: %s", timeText)
-				date = self.__lastDate
-
-			else:
-				# Lets get the time part
-				date = dateTimeMatch[0]
-
-				self.__lastTimeText = timeText
-				self.__lastDate = date
-		else:
-			timeText = self.__lastTimeText or "".join(tupleLine[::2])
-			date = self.__lastDate
-
-		if self.checkFindTime and date is not None and date < MyTime.time() - self.getFindTime():
-			if ll <= 5: logSys.log(5, "Ignore line since time %s < %s - %s", 
-				date, MyTime.time(), self.getFindTime())
-			return failList
-
 		if self.__lineBufferSize > 1:
-			orgBuffer = self.__lineBuffer = (
-				self.__lineBuffer + [tupleLine[:3]])[-self.__lineBufferSize:]
+			self.__lineBuffer.append(tupleLine)
+			orgBuffer = self.__lineBuffer = self.__lineBuffer[-self.__lineBufferSize:]
 		else:
-			orgBuffer = self.__lineBuffer = [tupleLine[:3]]
-		if ll <= 5: logSys.log(5, "Looking for match of %r", self.__lineBuffer)
-		buf = Regex._tupleLinesBuf(self.__lineBuffer)
+			orgBuffer = self.__lineBuffer = [tupleLine]
+		if ll <= 5: logSys.log(5, "Looking for match of %r", orgBuffer)
+		buf = Regex._tupleLinesBuf(orgBuffer)
+
+		# Checks if we must ignore this line (only if fewer ignoreregex than failregex).
+		if self.__ignoreRegex and len(self.__ignoreRegex) < len(self.__failRegex) - 2:
+			if self._ignoreLine(buf, orgBuffer) is not None:
+				# The ignoreregex matched. Return.
+				return failList

 		# Pre-filter fail regex (if available):
 		preGroups = {}
 		if self.__prefRegex:
 			if ll <= 5: logSys.log(5, "  Looking for prefregex %r", self.__prefRegex.getRegex())
-			self.__prefRegex.search(buf, self.__lineBuffer)
+			self.__prefRegex.search(buf, orgBuffer)
 			if not self.__prefRegex.hasMatched():
 				if ll <= 5: logSys.log(5, "  Prefregex not matched")
 				return failList
 			preGroups = self.__prefRegex.getGroups()
 			if ll <= 7: logSys.log(7, "  Pre-filter matched %s", preGroups)
-			repl = preGroups.get('content')
+			repl = preGroups.pop('content', None)
 			# Content replacement:
 			if repl:
-				del preGroups['content']
 				self.__lineBuffer, buf = [('', '', repl)], None

 		# Iterates over all the regular expressions.
@@ -822,28 +887,21 @@ class Filter(JailThread):
 				# The failregex matched.
 				if ll <= 7: logSys.log(7, "  Matched failregex %d: %s", failRegexIndex, fail)
 				# Checks if we must ignore this match.
-				if self.ignoreLine(failRegex.getMatchedTupleLines()) \
-						is not None:
+				if self.__ignoreRegex and self._ignoreLine(buf, orgBuffer, failRegex) is not None:
 					# The ignoreregex matched. Remove ignored match.
-					self.__lineBuffer, buf = failRegex.getUnmatchedTupleLines(), None
-					if ll <= 7: logSys.log(7, "  Matched ignoreregex and was ignored")
+					buf = None
 					if not self.checkAllRegex:
 						break
-					else:
-						continue
-				if date is None:
-					logSys.warning(
-						"Found a match for %r but no valid date/time "
-						"found for %r. Please try setting a custom "
-						"date pattern (see man page jail.conf(5)). "
-						"If format is complex, please "
-						"file a detailed issue on"
-						" https://github.com/fail2ban/fail2ban/issues "
-						"in order to get support for this format.",
-						 "\n".join(failRegex.getMatchedLines()), timeText)
 					continue
+				if noDate:
+					self._logWarnOnce("_next_noTimeWarn",
+						("Found a match but no valid date/time found for %r.", tupleLine[1]),
+						("Match without a timestamp: %s", "\n".join(failRegex.getMatchedLines())),
+						("Please try setting a custom date pattern (see man page jail.conf(5)).",)
+					)
+					if date is None and self.checkFindTime: continue
 				# we should check all regex (bypass on multi-line, otherwise too complex):
-				if not self.checkAllRegex or self.getMaxLines() > 1:
+				if not self.checkAllRegex or self.__lineBufferSize > 1:
 					self.__lineBuffer, buf = failRegex.getUnmatchedTupleLines(), None
 				# merge data if multi-line failure:
 				raw = returnRawHost
@@ -888,7 +946,8 @@ class Filter(JailThread):
 				if host is None:
 					if ll <= 7: logSys.log(7, "No failure-id by mlfid %r in regex %s: %s",
 						mlfid, failRegexIndex, fail.get('mlfforget', "waiting for identifier"))
-					if not self.checkAllRegex: return failList
+					fail['mlfpending'] = 1; # mark failure is pending
+					if not self.checkAllRegex and self.ignorePending: return failList
 					ips = [None]
 				# if raw - add single ip or failure-id,
 				# otherwise expand host to multiple ips using dns (or ignore it if not valid):
@@ -901,6 +960,9 @@ class Filter(JailThread):
 				# otherwise, try to use dns conversion:
 				else:
 					ips = DNSUtils.textToIp(host, self.__useDns)
+				# if checkAllRegex we must make a copy (to be sure next RE doesn't change merged/cached failure):
+				if self.checkAllRegex and mlfid is not None:
+					fail = fail.copy()
 				# append failure with match to the list:
 				for ip in ips:
 					failList.append([failRegexIndex, ip, date, fail])
@@ -946,7 +1008,7 @@ class FileFilter(Filter):
 					log.setPos(lastpos)
 			self.__logs[path] = log
 			logSys.info("Added logfile: %r (pos = %s, hash = %s)" , path, log.getPos(), log.getHash())
-			if autoSeek:
+			if autoSeek and not tail:
 				self.__autoSeek[path] = autoSeek
 			self._addLogPath(path)			# backend specific

@@ -965,9 +1027,6 @@ class FileFilter(Filter):
 			log = self.__logs.pop(path)
 		except KeyError:
 			return
-		db = self.jail.database
-		if db is not None:
-			db.updateLog(self.jail, log)
 		logSys.info("Removed logfile: %r", path)
 		self._delLogPath(path)
 		return
@@ -1030,7 +1089,8 @@ class FileFilter(Filter):
 	# MyTime.time()-self.findTime. When a failure is detected, a FailTicket
 	# is created and is added to the FailManager.

-	def getFailures(self, filename):
+	def getFailures(self, filename, inOperation=None):
+		if self.idle: return False
 		log = self.getLog(filename)
 		if log is None:
 			logSys.error("Unable to get failures in %s", filename)
@@ -1075,15 +1135,26 @@ class FileFilter(Filter):
 			if has_content:
 				while not self.idle:
 					line = log.readline()
-					if not line or not self.active:
-						# The jail reached the bottom or has been stopped
+					if not self.active: break; # jail has been stopped
+					if not line:
+						# The jail reached the bottom, simply set in operation for this log
+						# (since we are first time at end of file, growing is only possible after modifications):
+						log.inOperation = True
 						break
-					self.processLineAndAdd(line)
+					# acquire in operation from log and process:
+					self.inOperation = inOperation if inOperation is not None else log.inOperation
+					self.processLineAndAdd(line.rstrip('\r\n'))
 		finally:
 			log.close()
-		db = self.jail.database
-		if db is not None:
-			db.updateLog(self.jail, log)
+		if self.jail.database is not None:
+			self._pendDBUpdates[log] = 1
+			if (
+				self.ticks % 100 == 0
+				or MyTime.time() >= self._nextUpdateTM
+				or not self.active
+			):
+				self._updateDBPending()
+				self._nextUpdateTM = MyTime.time() + Utils.DEFAULT_SLEEP_TIME * 5
 		return True

 	##
@@ -1183,12 +1254,33 @@ class FileFilter(Filter):
 		ret.append(("File list", path))
 		return ret

-	def stop(self):
-		"""Stop monitoring of log-file(s)
+	def _updateDBPending(self):
+		"""Apply pending updates (log position) to database.
 		"""
+		db = self.jail.database
+		while True:
+			try:
+				log, args = self._pendDBUpdates.popitem()
+			except KeyError:
+				break
+			db.updateLog(self.jail, log)
+			
+	def onStop(self):
+		"""Stop monitoring of log-file(s). Invoked after run method.
+		"""
+		# ensure positions of pending logs are up-to-date:
+		if self._pendDBUpdates and self.jail.database:
+			self._updateDBPending()
 		# stop files monitoring:
 		for path in self.__logs.keys():
 			self.delLogPath(path)
+
+	def stop(self):
+		"""Stop filter
+		"""
+		# normally onStop will be called automatically in thread after its run ends, 
+		# but for backwards compatibilities we'll invoke it in caller of stop method.
+		self.onStop()
 		# stop thread:
 		super(Filter, self).stop()

@@ -1216,7 +1308,7 @@ except ImportError: # pragma: no cover

 class FileContainer:

-	def __init__(self, filename, encoding, tail = False):
+	def __init__(self, filename, encoding, tail=False):
 		self.__filename = filename
 		self.setEncoding(encoding)
 		self.__tail = tail
@@ -1237,6 +1329,17 @@ class FileContainer:
 				self.__pos = 0
 		finally:
 			handler.close()
+		## shows that log is in operation mode (expecting new messages only from here):
+		self.inOperation = tail
+
+	def __hash__(self):
+		return hash(self.__filename)
+	def __eq__(self, other):
+		return (id(self) == id(other) or
+			self.__filename == (other.__filename if isinstance(other, FileContainer) else other)
+		)
+	def __repr__(self):
+		return 'file-log:'+self.__filename

 	def getFileName(self):
 		return self.__filename
@@ -1310,16 +1413,17 @@ class FileContainer:
 			return line.decode(enc, 'strict')
 		except (UnicodeDecodeError, UnicodeEncodeError) as e:
 			global _decode_line_warn
-			lev = logging.DEBUG
-			if _decode_line_warn.get(filename, 0) <= MyTime.time():
+			lev = 7
+			if not _decode_line_warn.get(filename, 0):
 				lev = logging.WARNING
-				_decode_line_warn[filename] = MyTime.time() + 24*60*60
+				_decode_line_warn.set(filename, 1)
 			logSys.log(lev,
-				"Error decoding line from '%s' with '%s'."
-				" Consider setting logencoding=utf-8 (or another appropriate"
-				" encoding) for this jail. Continuing"
-				" to process line ignoring invalid characters: %r",
-				filename, enc, line)
+				"Error decoding line from '%s' with '%s'.", filename, enc)
+			if logSys.getEffectiveLevel() <= lev:
+				logSys.log(lev, "Consider setting logencoding=utf-8 (or another appropriate"
+					" encoding) for this jail. Continuing"
+					" to process line ignoring invalid characters: %r",
+					line)
 			# decode with replacing error chars:
 			line = line.decode(enc, 'replace')
 		return line
@@ -1340,7 +1444,7 @@ class FileContainer:
 		## print "D: Closed %s with pos %d" % (handler, self.__pos)
 		## sys.stdout.flush()

-_decode_line_warn = {}
+_decode_line_warn = Utils.Cache(maxCount=1000, maxTime=24*60*60);


 ##
@@ -55,7 +55,6 @@ class FilterGamin(FileFilter):

 	def __init__(self, jail):
 		FileFilter.__init__(self, jail)
-		self.__modified = False
 		# Gamin monitor
 		self.monitor = gamin.WatchMonitor()
 		fd = self.monitor.get_fd()
@@ -67,20 +66,9 @@ class FilterGamin(FileFilter):
 		logSys.log(4, "Got event: " + repr(event) + " for " + path)
 		if event in (gamin.GAMCreated, gamin.GAMChanged, gamin.GAMExists):
 			logSys.debug("File changed: " + path)
-			self.__modified = True

 		self.ticks += 1
-		self._process_file(path)
-
-	def _process_file(self, path):
-		"""Process a given file
-
-		TODO -- RF:
-		this is a common logic and must be shared/provided by FileFilter
-		"""
 		self.getFailures(path)
-		self.performBan()
-		self.__modified = False

 	##
 	# Add a log file path
@@ -127,6 +115,9 @@ class FilterGamin(FileFilter):
 			Utils.wait_for(lambda: not self.active or self._handleEvents(),
 				self.sleeptime)
 			self.ticks += 1
+			if self.ticks % 10 == 0:
+				self.performSvc()
+
 		logSys.debug("[%s] filter terminated", self.jailName)
 		return True

@@ -27,9 +27,7 @@ __license__ = "GPL"
 import os
 import time

-from .failmanager import FailManagerEmpty
 from .filter import FileFilter
-from .mytime import MyTime
 from .utils import Utils
 from ..helpers import getLogger, logging

@@ -55,7 +53,6 @@ class FilterPoll(FileFilter):

 	def __init__(self, jail):
 		FileFilter.__init__(self, jail)
-		self.__modified = False
 		## The time of the last modification of the file.
 		self.__prevStats = dict()
 		self.__file404Cnt = dict()
@@ -111,21 +108,21 @@ class FilterPoll(FileFilter):
 				modlst = []
 				Utils.wait_for(lambda: not self.active or self.getModified(modlst),
 					self.sleeptime)
+				if not self.active: # pragma: no cover - timing
+					break
 				for filename in modlst:
 					self.getFailures(filename)
-					self.__modified = True

 				self.ticks += 1
-				if self.__modified:
-					self.performBan()
-					self.__modified = False
+				if self.ticks % 10 == 0:
+					self.performSvc()
 			except Exception as e: # pragma: no cover
 				if not self.active: # if not active - error by stop...
 					break
 				logSys.error("Caught unhandled exception in main cycle: %r", e,
 					exc_info=logSys.getEffectiveLevel()<=logging.DEBUG)
 				# incr common error counter:
-				self.commonError()
+				self.commonError("unhandled", e)
 		logSys.debug("[%s] filter terminated", self.jailName)
 		return True

@@ -139,7 +136,7 @@ class FilterPoll(FileFilter):
 		try:
 			logStats = os.stat(filename)
 			stats = logStats.st_mtime, logStats.st_ino, logStats.st_size
-			pstats = self.__prevStats.get(filename, (0))
+			pstats = self.__prevStats.get(filename, (0,))
 			if logSys.getEffectiveLevel() <= 4:
 				# we do not want to waste time on strftime etc if not necessary
 				dt = logStats.st_mtime - pstats[0]
@@ -75,7 +75,6 @@ class FilterPyinotify(FileFilter):

 	def __init__(self, jail):
 		FileFilter.__init__(self, jail)
-		self.__modified = False
 		# Pyinotify watch manager
 		self.__monitor = pyinotify.WatchManager()
 		self.__notifier = None
@@ -140,8 +139,6 @@ class FilterPyinotify(FileFilter):
 		"""
 		if not self.idle:
 			self.getFailures(path)
-			self.performBan()
-			self.__modified = False

 	def _addPending(self, path, reason, isDir=False):
 		if path not in self.__pending:
@@ -168,7 +165,7 @@ class FilterPyinotify(FileFilter):
 			return
 		found = {}
 		minTime = 60
-		for path, (retardTM, isDir) in self.__pending.iteritems():
+		for path, (retardTM, isDir) in list(self.__pending.items()):
 			if ntm - self.__pendingChkTime < retardTM:
 				if minTime > retardTM: minTime = retardTM
 				continue
@@ -187,10 +184,11 @@ class FilterPyinotify(FileFilter):
 		for path, isDir in found.iteritems():
 			self._delPending(path)
 			# refresh monitoring of this:
-			self._refreshWatcher(path, isDir=isDir)
+			if isDir is not None:
+				self._refreshWatcher(path, isDir=isDir)
 			if isDir:
 				# check all files belong to this dir:
-				for logpath in self.__watchFiles:
+				for logpath in list(self.__watchFiles):
 					if logpath.startswith(path + pathsep):
 						# if still no file - add to pending, otherwise refresh and process:
 						if not os.path.isfile(logpath):
@@ -270,28 +268,32 @@ class FilterPyinotify(FileFilter):

 	def _addLogPath(self, path):
 		self._addFileWatcher(path)
-		self._process_file(path)
+		# notify (wake up if in waiting):
+		if self.active:
+			self.__pendingMinTime = 0
+		# retard until filter gets started, isDir=None signals special case: process file only (don't need to refresh monitor):
+		self._addPending(path, ('INITIAL', path), isDir=None)

-    ##
+	##
 	# Delete a log path
 	#
 	# @param path the log file to delete

 	def _delLogPath(self, path):
+		self._delPending(path)
 		if not self._delFileWatcher(path): # pragma: no cover
 			logSys.error("Failed to remove watch on path: %s", path)
-		self._delPending(path)

 		path_dir = dirname(path)
-		for k in self.__watchFiles:
+		for k in list(self.__watchFiles):
 			if k.startswith(path_dir + pathsep):
 				path_dir = None
 				break
 		if path_dir:
 			# Remove watches for the directory
 			# since there is no other monitored file under this directory
-			self._delDirWatcher(path_dir)
 			self._delPending(path_dir)
+			self._delDirWatcher(path_dir)

 	# pyinotify.ProcessEvent default handler:
 	def __process_default(self, event):
@@ -337,16 +339,26 @@ class FilterPyinotify(FileFilter):
 				self.__notifier.process_events()

 				# wait for events / timeout:
-				notify_maxtout = self.__notify_maxtout
 				def __check_events():
-					return not self.active or self.__notifier.check_events(timeout=notify_maxtout)
-				if Utils.wait_for(__check_events, min(self.sleeptime, self.__pendingMinTime)):
+					return (
+						not self.active
+						or bool(self.__notifier.check_events(timeout=self.__notify_maxtout))
+						or (self.__pendingMinTime and self.__pending)
+					)
+				wres = Utils.wait_for(__check_events, min(self.sleeptime, self.__pendingMinTime))
+				if wres:
 					if not self.active: break
-					self.__notifier.read_events()
+					if not isinstance(wres, dict):
+						self.__notifier.read_events()
+
+				self.ticks += 1

 				# check pending files/dirs (logrotate ready):
-				if not self.idle:
-					self._checkPending()
+				if self.idle:
+					continue
+				self._checkPending()
+				if self.ticks % 10 == 0:
+					self.performSvc()

 			except Exception as e: # pragma: no cover
 				if not self.active: # if not active - error by stop...
@@ -354,10 +366,8 @@ class FilterPyinotify(FileFilter):
 				logSys.error("Caught unhandled exception in main cycle: %r", e,
 					exc_info=logSys.getEffectiveLevel()<=logging.DEBUG)
 				# incr common error counter:
-				self.commonError()
+				self.commonError("unhandled", e)
 			
-			self.ticks += 1
-
 		logSys.debug("[%s] filter exited (pyinotifier)", self.jailName)
 		self.__notifier = None

@@ -23,6 +23,7 @@ __copyright__ = "Copyright (c) 2013 Steven Hiscocks"
 __license__ = "GPL"

 import datetime
+import os
 import time
 from distutils.version import LooseVersion

@@ -91,9 +92,14 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover
 		try:
 			args['flags'] = int(kwargs.pop('journalflags'))
 		except KeyError:
-			# be sure all journal types will be opened if files specified (don't set flags):
-			if 'files' not in args or not len(args['files']):
-				args['flags'] = 4
+			# be sure all journal types will be opened if files/path specified (don't set flags):
+			if ('files' not in args or not len(args['files'])) and ('path' not in args or not args['path']):
+				args['flags'] = int(os.getenv("F2B_SYSTEMD_DEFAULT_FLAGS", 4))
+				
+		try:
+			args['namespace'] = kwargs.pop('namespace')
+		except KeyError:
+			pass

 		return args

@@ -190,6 +196,13 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover
 	def getJournalReader(self):
 		return self.__journal

+	def getJrnEntTime(self, logentry):
+		""" Returns time of entry as tuple (ISO-str, Posix)."""
+		date = logentry.get('_SOURCE_REALTIME_TIMESTAMP')
+		if date is None:
+				date = logentry.get('__REALTIME_TIMESTAMP')
+		return (date.isoformat(), time.mktime(date.timetuple()) + date.microsecond/1.0E6)
+
 	##
 	# Format journal log entry into syslog style
 	#
@@ -222,9 +235,8 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover
 				logelements[-1] += v
 			logelements[-1] += ":"
 			if logelements[-1] == "kernel:":
-				if '_SOURCE_MONOTONIC_TIMESTAMP' in logentry:
-					monotonic = logentry.get('_SOURCE_MONOTONIC_TIMESTAMP')
-				else:
+				monotonic = logentry.get('_SOURCE_MONOTONIC_TIMESTAMP')
+				if monotonic is None:
 					monotonic = logentry.get('__MONOTONIC_TIMESTAMP')[0]
 				logelements.append("[%12.6f]" % monotonic.total_seconds())
 		msg = logentry.get('MESSAGE','')
@@ -235,19 +247,21 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover

 		logline = " ".join(logelements)

-		date = logentry.get('_SOURCE_REALTIME_TIMESTAMP',
-				logentry.get('__REALTIME_TIMESTAMP'))
+		date = self.getJrnEntTime(logentry)
 		logSys.log(5, "[%s] Read systemd journal entry: %s %s", self.jailName,
-			date.isoformat(), logline)
+			date[0], logline)
 		## use the same type for 1st argument:
-		return ((logline[:0], date.isoformat(), logline.replace('\n', '\\n')),
-			time.mktime(date.timetuple()) + date.microsecond/1.0E6)
+		return ((logline[:0], date[0], logline.replace('\n', '\\n')), date[1])

 	def seekToTime(self, date):
 		if not isinstance(date, datetime.datetime):
 			date = datetime.datetime.fromtimestamp(date)
 		self.__journal.seek_realtime(date)

+	def inOperationMode(self):
+		self.inOperation = True
+		logSys.info("[%s] Jail is in operation now (process new journal entries)", self.jailName)
+
 	##
 	# Main loop.
 	#
@@ -258,21 +272,46 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover

 		if not self.getJournalMatch():
 			logSys.notice(
-				"Jail started without 'journalmatch' set. "
+				"[%s] Jail started without 'journalmatch' set. "
 				"Jail regexs will be checked against all journal entries, "
-				"which is not advised for performance reasons.")
+				"which is not advised for performance reasons.", self.jailName)

-		# Seek to now - findtime in journal
-		start_time = datetime.datetime.now() - \
-				datetime.timedelta(seconds=int(self.getFindTime()))
-		self.seekToTime(start_time)
-		# Move back one entry to ensure do not end up in dead space
-		# if start time beyond end of journal
+		# Save current cursor position (to recognize in operation mode):
+		logentry = None
 		try:
-			self.__journal.get_previous()
+			self.__journal.seek_tail()
+			logentry = self.__journal.get_previous()
+			self.__journal.get_next()
 		except OSError:
-			pass # Reading failure, so safe to ignore
+			logentry = None # Reading failure, so safe to ignore
+		if logentry:
+			# Try to obtain the last known time (position of journal)
+			startTime = 0
+			if self.jail.database is not None:
+				startTime = self.jail.database.getJournalPos(self.jail, 'systemd-journal') or 0
+			# Seek to max(last_known_time, now - findtime) in journal
+			startTime = max( startTime, MyTime.time() - int(self.getFindTime()) )
+			self.seekToTime(startTime)
+			# Not in operation while we'll read old messages ...
+			self.inOperation = False
+			# Save current time in order to check time to switch "in operation" mode
+			startTime = (1, MyTime.time(), logentry.get('__CURSOR'))
+			# Move back one entry to ensure do not end up in dead space
+			# if start time beyond end of journal
+			try:
+				self.__journal.get_previous()
+			except OSError:
+				pass # Reading failure, so safe to ignore
+		else:
+			# empty journal or no entries for current filter:
+			self.inOperationMode()
+			# seek_tail() seems to have a bug by no entries (could bypass some entries hereafter), so seek to now instead:
+			startTime = MyTime.time()
+			self.seekToTime(startTime)
+			# for possible future switches of in-operation mode:
+			startTime = (0, startTime)

+		line = None
 		while self.active:
 			# wait for records (or for timeout in sleeptime seconds):
 			try:
@@ -282,9 +321,10 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover
 				#self.__journal.wait(self.sleeptime) != journal.NOP
 				## 
 				## wait for entries without sleep in intervals, because "sleeping" in journal.wait:
-				Utils.wait_for(lambda: not self.active or \
-					self.__journal.wait(Utils.DEFAULT_SLEEP_INTERVAL) != journal.NOP,
-					self.sleeptime, 0.00001)
+				if not logentry:
+					Utils.wait_for(lambda: not self.active or \
+						self.__journal.wait(Utils.DEFAULT_SLEEP_INTERVAL) != journal.NOP,
+						self.sleeptime, 0.00001)
 				if self.idle:
 					# because journal.wait will returns immediatelly if we have records in journal,
 					# just wait a little bit here for not idle, to prevent hi-load:
@@ -303,23 +343,50 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover
 							e, exc_info=logSys.getEffectiveLevel() <= logging.DEBUG)
 					self.ticks += 1
 					if logentry:
-						self.processLineAndAdd(
-							*self.formatJournalEntry(logentry))
+						line, tm = self.formatJournalEntry(logentry)
+						# switch "in operation" mode if we'll find start entry (+ some delta):
+						if not self.inOperation:
+							if tm >= MyTime.time() - 1: # reached now (approximated):
+								self.inOperationMode()
+							elif startTime[0] == 1:
+								# if it reached start entry (or get read time larger than start time)
+								if logentry.get('__CURSOR') == startTime[2] or tm > startTime[1]:
+									# give the filter same time it needed to reach the start entry:
+									startTime = (0, MyTime.time()*2 - startTime[1])
+							elif tm > startTime[1]: # reached start time (approximated):
+								self.inOperationMode()
+						# process line
+						self.processLineAndAdd(line, tm)
 						self.__modified += 1
 						if self.__modified >= 100: # todo: should be configurable
 							break
 					else:
+						# "in operation" mode since we don't have messages anymore (reached end of journal):
+						if not self.inOperation:
+							self.inOperationMode()
 						break
-				if self.__modified:
-					self.performBan()
-					self.__modified = 0
+				self.__modified = 0
+				if self.ticks % 10 == 0:
+					self.performSvc()
+				# update position in log (time and iso string):
+				if self.jail.database:
+					if line:
+						self._pendDBUpdates['systemd-journal'] = (tm, line[1])
+						line = None
+					if self._pendDBUpdates and (
+				    self.ticks % 100 == 0
+				    or MyTime.time() >= self._nextUpdateTM
+				    or not self.active
+				  ):
+						self._updateDBPending()
+						self._nextUpdateTM = MyTime.time() + Utils.DEFAULT_SLEEP_TIME * 5
 			except Exception as e: # pragma: no cover
 				if not self.active: # if not active - error by stop...
 					break
 				logSys.error("Caught unhandled exception in main cycle: %r", e,
 					exc_info=logSys.getEffectiveLevel()<=logging.DEBUG)
 				# incr common error counter:
-				self.commonError()
+				self.commonError("unhandled", e)

 		logSys.debug("[%s] filter terminated", self.jailName)

@@ -339,3 +406,22 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover
 		ret.append(("Journal matches",
 			[" + ".join(" ".join(match) for match in self.__matches)]))
 		return ret
+
+	def _updateDBPending(self):
+		"""Apply pending updates (jornal position) to database.
+		"""
+		db = self.jail.database
+		while True:
+			try:
+				log, args = self._pendDBUpdates.popitem()
+			except KeyError:
+				break
+			db.updateJournal(self.jail, log, *args)
+
+	def onStop(self):
+		"""Stop monitoring of journal. Invoked after run method.
+		"""
+		# ensure positions of pending logs are up-to-date:
+		if self._pendDBUpdates and self.jail.database:
+			self._updateDBPending()
+
@@ -169,27 +169,31 @@ class DNSUtils:
 		DNSUtils.CACHE_ipToName.set(key, name)
 		return name

+	# key find cached own hostnames (this tuple-key cannot be used elsewhere):
+	_getSelfNames_key = ('self','dns')
+
 	@staticmethod
 	def getSelfNames():
 		"""Get own host names of self"""
-		# try find cached own hostnames (this tuple-key cannot be used elsewhere):
-		key = ('self','dns')
-		names = DNSUtils.CACHE_ipToName.get(key)
+		# try find cached own hostnames:
+		names = DNSUtils.CACHE_ipToName.get(DNSUtils._getSelfNames_key)
 		# get it using different ways (a set with names of localhost, hostname, fully qualified):
 		if names is None:
 			names = set([
 				'localhost', DNSUtils.getHostname(False), DNSUtils.getHostname(True)
 			]) - set(['']) # getHostname can return ''
 		# cache and return :
-		DNSUtils.CACHE_ipToName.set(key, names)
+		DNSUtils.CACHE_ipToName.set(DNSUtils._getSelfNames_key, names)
 		return names

+	# key to find cached own IPs (this tuple-key cannot be used elsewhere):
+	_getSelfIPs_key = ('self','ips')
+
 	@staticmethod
 	def getSelfIPs():
 		"""Get own IP addresses of self"""
-		# try find cached own IPs (this tuple-key cannot be used elsewhere):
-		key = ('self','ips')
-		ips = DNSUtils.CACHE_nameToIp.get(key)
+		# to find cached own IPs:
+		ips = DNSUtils.CACHE_nameToIp.get(DNSUtils._getSelfIPs_key)
 		# get it using different ways (a set with IPs of localhost, hostname, fully qualified):
 		if ips is None:
 			ips = set()
@@ -199,13 +203,30 @@ class DNSUtils:
 				except Exception as e: # pragma: no cover
 					logSys.warning("Retrieving own IPs of %s failed: %s", hostname, e)
 		# cache and return :
-		DNSUtils.CACHE_nameToIp.set(key, ips)
+		DNSUtils.CACHE_nameToIp.set(DNSUtils._getSelfIPs_key, ips)
 		return ips

+	_IPv6IsAllowed = None
+
+	@staticmethod
+	def setIPv6IsAllowed(value):
+		DNSUtils._IPv6IsAllowed = value
+		logSys.debug("IPv6 is %s", ('on' if value else 'off') if value is not None else 'auto')
+		return value
+
+	# key to find cached value of IPv6 allowance (this tuple-key cannot be used elsewhere):
+	_IPv6IsAllowed_key = ('self','ipv6-allowed')
+
 	@staticmethod
 	def IPv6IsAllowed():
-		# return os.path.exists("/proc/net/if_inet6") || any((':' in ip) for ip in DNSUtils.getSelfIPs())
-		return any((':' in ip.ntoa) for ip in DNSUtils.getSelfIPs())
+		if DNSUtils._IPv6IsAllowed is not None:
+			return DNSUtils._IPv6IsAllowed
+		v = DNSUtils.CACHE_nameToIp.get(DNSUtils._IPv6IsAllowed_key)
+		if v is not None:
+			return v
+		v = any((':' in ip.ntoa) for ip in DNSUtils.getSelfIPs())
+		DNSUtils.CACHE_nameToIp.set(DNSUtils._IPv6IsAllowed_key, v)
+		return v


 ##
@@ -379,6 +400,12 @@ class IPAddr(object):
 		"""
 		return self._family != socket.AF_UNSPEC

+	@property
+	def isSingle(self):
+		"""Returns whether the object is a single IP address (not DNS and subnet)
+		"""
+		return self._plen == {socket.AF_INET: 32, socket.AF_INET6: 128}.get(self._family, -1000)
+
 	def __eq__(self, other):
 		if self._family == IPAddr.CIDR_RAW and not isinstance(other, IPAddr):
 			return self._raw == other
@@ -511,6 +538,11 @@ class IPAddr(object):
 		
 		return (self.addr & mask) == net.addr

+	def contains(self, ip):
+		"""Return whether the object (as network) contains given IP
+		"""
+		return isinstance(ip, IPAddr) and (ip == self or ip.isInNet(self))
+
 	# Pre-calculated map: addr to maskplen
 	def __getMaskMap():
 		m6 = (1 << 128)-1
@@ -156,6 +156,10 @@ class Jail(object):
 		"""
 		return self.__db

+	@database.setter
+	def database(self, value):
+		self.__db = value;
+
 	@property
 	def filter(self):
 		"""The filter which the jail is using to monitor log files.
@@ -187,14 +191,18 @@ class Jail(object):
 			("Actions", self.actions.status(flavor=flavor)),
 			]

+	@property
+	def hasFailTickets(self):
+		"""Retrieve whether queue has tickets to ban.
+		"""
+		return not self.__queue.empty()
+
 	def putFailTicket(self, ticket):
 		"""Add a fail ticket to the jail.

 		Used by filter to add a failure for banning.
 		"""
 		self.__queue.put(ticket)
-		if not ticket.restored and self.database is not None:
-			self.database.addBan(self, ticket)

 	def getFailTicket(self):
 		"""Get a fail ticket from the jail.
@@ -22,7 +22,10 @@ __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2013- Yaroslav Halchenko"
 __license__ = "GPL"

 from threading import Lock
-from collections import Mapping
+try:
+	from collections.abc import Mapping
+except ImportError:
+	from collections import Mapping

 from ..exceptions import DuplicateJailException, UnknownJailException
 from .jail import Jail
@@ -67,6 +67,8 @@ class JailThread(Thread):
 		def run_with_except_hook(*args, **kwargs):
 			try:
 				run(*args, **kwargs)
+				# call on stop callback to do some finalizations:
+				self.onStop()
 			except Exception as e:
 				# avoid very sporadic error "'NoneType' object has no attribute 'exc_info'" (https://bugs.python.org/issue7336)
 				# only extremely fast systems are affected ATM (2.7 / 3.x), if thread ends nothing is available here.
@@ -97,6 +99,12 @@ class JailThread(Thread):
 		self.active = True
 		super(JailThread, self).start()

+	@abstractmethod
+	def onStop(self): # pragma: no cover - absract
+		"""Abstract - Called when thread ends (after run).
+		"""
+		pass
+
 	def stop(self):
 		"""Sets `active` property to False, to flag run method to return.
 		"""
@@ -120,3 +128,6 @@ class JailThread(Thread):
 ## python 2.x replace binding of private __bootstrap method:
 if sys.version_info < (3,): # pragma: 3.x no cover
 	JailThread._Thread__bootstrap = JailThread._JailThread__bootstrap
+## python 3.9, restore isAlive method:
+elif not hasattr(JailThread, 'isAlive'): # pragma: 2.x no cover
+	 JailThread.isAlive = JailThread.is_alive
@@ -161,3 +161,62 @@ class MyTime:
 			val = rexp.sub(rpl, val)
 		val = MyTime._str2sec_fini.sub(r"\1+\2", val)
 		return eval(val)
+
+	class seconds2str():
+		"""Converts seconds to string on demand (if string representation needed).
+		Ex: seconds2str(86400*390)            = 1y 3w 4d
+		    seconds2str(86400*368)            = 1y 3d
+		    seconds2str(86400*365.5)          = 1y
+		    seconds2str(86400*2+3600*7+60*15) = 2d 7h 15m
+		    seconds2str(86400*2+3599)         = 2d 1h
+		    seconds2str(3600-5)               = 1h
+		    seconds2str(3600-10)              = 59m 50s
+		    seconds2str(59)                   = 59s
+		"""
+		def __init__(self, sec):
+			self.sec = sec
+		def __str__(self):
+			# s = str(datetime.timedelta(seconds=int(self.sec)))
+			# return s if s[-3:] != ":00" else s[:-3]
+			s = self.sec; c = 3
+			# automatic accuracy: round by large values (and maximally 3 groups)
+			if s >= 31536000: # a year as 365*24*60*60 (don't need to consider leap year by this accuracy)
+				s = int(round(float(s)/86400)) # round by a day
+				r = str(s//365) + 'y '; s %= 365
+				if s >= 7:
+					r += str(s//7) + 'w '; s %= 7
+				if s:
+					r += str(s) + 'd '
+				return r[:-1]
+			if s >= 604800: # a week as 24*60*60*7
+				s = int(round(float(s)/3600)) # round by a hour
+				r = str(s//168) + 'w '; s %= 168
+				if s >= 24:
+					r += str(s//24) + 'd '; s %= 24
+				if s:
+					r += str(s) + 'h '
+				return r[:-1]
+			if s >= 86400: # a day as 24*60*60
+				s = int(round(float(s)/60)) # round by a minute
+				r = str(s//1440) + 'd '; s %= 1440
+				if s >= 60:
+					r += str(s//60) + 'h '; s %= 60
+				if s:
+					r += str(s) + 'm '
+				return r[:-1]
+			if s >= 3595: # a hour as 60*60 (- 5 seconds)
+				s = int(round(float(s)/10)) # round by 10 seconds
+				r = str(s//360) + 'h '; s %= 360
+				if s >= 6: # a minute
+					r += str(s//6) + 'm '; s %= 6
+				return r[:-1]
+			r = ''
+			if s >= 60: # a minute
+				r += str(s//60) + 'm '; s %= 60
+			if s: # remaining seconds
+				r += str(s) + 's '
+			elif not self.sec: # 0s
+				r = '0 '
+			return r[:-1]
+		def __repr__(self):
+			return self.__str__()
@@ -33,7 +33,7 @@ import stat
 import sys

 from .jails import Jails
-from .filter import FileFilter, JournalFilter
+from .filter import DNSUtils, FileFilter, JournalFilter
 from .transmitter import Transmitter
 from .asyncserver import AsyncServer, AsyncServerException
 from .. import version
@@ -57,6 +57,23 @@ except ImportError: # pragma: no cover
 def _thread_name():
 	return threading.current_thread().__class__.__name__

+try:
+	FileExistsError
+except NameError: # pragma: 3.x no cover
+	FileExistsError = OSError
+
+def _make_file_path(name):
+	"""Creates path of file (last level only) on demand"""
+	name = os.path.dirname(name)
+	# only if it is absolute (e. g. important for socket, so if unix path):
+	if os.path.isabs(name):
+		# be sure path exists (create last level of directory on demand):
+		try:
+			os.mkdir(name)
+		except (OSError, FileExistsError) as e:
+			if e.errno != 17: # pragma: no cover - not EEXIST is not covered
+				raise
+

 class Server:
 	
@@ -80,8 +97,6 @@ class Server:
 			'Linux': '/dev/log',
 		}
 		self.__prev_signals = {}
-		# replace real thread name with short process name (for top/ps/pstree or diagnostic):
-		prctl_set_th_name('f2b/server')

 	def __sigTERMhandler(self, signum, frame): # pragma: no cover - indirect tested
 		logSys.debug("Caught signal %d. Exiting", signum)
@@ -98,7 +113,7 @@ class Server:

 	def start(self, sock, pidfile, force=False, conf={}):
 		# First set the mask to only allow access to owner
-		os.umask(0077)
+		os.umask(0o077)
 		# Second daemonize before logging etc, because it will close all handles:
 		if self.__daemon: # pragma: no cover
 			logSys.info("Starting in daemon mode")
@@ -112,6 +127,9 @@ class Server:
 				logSys.error(err)
 				raise ServerInitializationError(err)
 			# We are daemon.
+
+		# replace main thread (and process) name to identify server (for top/ps/pstree or diagnostic):
+		prctl_set_th_name(conf.get("pname", "fail2ban-server"))
 		
 		# Set all logging parameters (or use default if not specified):
 		self.__verbose = conf.get("verbose", None)
@@ -140,6 +158,7 @@ class Server:
 		# Creates a PID file.
 		try:
 			logSys.debug("Creating PID file %s", pidfile)
+			_make_file_path(pidfile)
 			pidFile = open(pidfile, 'w')
 			pidFile.write("%s\n" % os.getpid())
 			pidFile.close()
@@ -149,6 +168,7 @@ class Server:
 		# Start the communication
 		logSys.debug("Starting communication")
 		try:
+			_make_file_path(sock)
 			self.__asyncServer = AsyncServer(self.__transm)
 			self.__asyncServer.onstart = conf.get('onstart')
 			self.__asyncServer.start(sock, force)
@@ -254,6 +274,11 @@ class Server:
 			for name in self.__jails.keys():
 				self.delJail(name, stop=False, join=True)

+	def clearCaches(self):
+		# we need to clear caches, to be able to recognize new IPs/families etc:
+		DNSUtils.CACHE_nameToIp.clear()
+		DNSUtils.CACHE_ipToName.clear()	
+
 	def reloadJails(self, name, opts, begin):
 		if begin:
 			# begin reload:
@@ -275,6 +300,8 @@ class Server:
 						if "--restart" in opts:
 							self.stopJail(name)
 				else:
+					# invalidate caches by reload
+					self.clearCaches()
 					# first unban all ips (will be not restored after (re)start):
 					if "--unban" in opts:
 						self.setUnbanIP()
@@ -345,7 +372,7 @@ class Server:
 		if isinstance(filter_, FileFilter):
 			return filter_.getLogPaths()
 		else: # pragma: systemd no cover
-			logSys.info("Jail %s is not a FileFilter instance" % name)
+			logSys.debug("Jail %s is not a FileFilter instance" % name)
 			return []
 	
 	def addJournalMatch(self, name, match): # pragma: systemd no cover
@@ -363,7 +390,7 @@ class Server:
 		if isinstance(filter_, JournalFilter):
 			return filter_.getJournalMatch()
 		else:
-			logSys.info("Jail %s is not a JournalFilter instance" % name)
+			logSys.debug("Jail %s is not a JournalFilter instance" % name)
 			return []
 	
 	def setLogEncoding(self, name, encoding):
@@ -501,6 +528,32 @@ class Server:
 			cnt += jail.actions.removeBannedIP(value, ifexists=ifexists)
 		return cnt
 		
+	def banned(self, name=None, ids=None):
+		if name is not None:
+			# single jail:
+			jails = [self.__jails[name]]
+		else:
+			# in all jails:
+			jails = self.__jails.values()
+		# check banned ids:
+		res = []
+		if name is None and ids:
+			for ip in ids:
+				ret = []
+				for jail in jails:
+					if jail.actions.getBanned([ip]):
+						ret.append(jail.name)
+				res.append(ret)
+		else:
+			for jail in jails:
+				ret = jail.actions.getBanned(ids)
+				if name is not None:
+					return ret
+					res.append(ret)
+				else:
+					res.append({jail.name: ret})
+		return res
+		
 	def getBanTime(self, name):
 		return self.__jails[name].actions.getBanTime()
 	
@@ -717,6 +770,11 @@ class Server:
 				logSys.info("flush performed on %s" % self.__logTarget)
 			return "flushed"
 			
+	@staticmethod
+	def setIPv6IsAllowed(value):
+		value = _as_bool(value) if value != 'auto' else None
+		return DNSUtils.setIPv6IsAllowed(value)
+
 	def setThreadOptions(self, value):
 		for o, v in value.iteritems():
 			if o == 'stacksize':
@@ -740,6 +798,7 @@ class Server:
 			self.__db = None
 		else:
 			if Fail2BanDb is not None:
+				_make_file_path(filename)
 				self.__db = Fail2BanDb(filename)
 				self.__db.delAllJails()
 			else: # pragma: no cover
@@ -30,17 +30,6 @@ locale_time = LocaleTime()
 TZ_ABBR_RE = r"[A-Z](?:[A-Z]{2,4})?"
 FIXED_OFFSET_TZ_RE = re.compile(r"(%s)?([+-][01]\d(?::?\d{2})?)?$" % (TZ_ABBR_RE,))

-def _getYearCentRE(cent=(0,3), distance=3, now=(MyTime.now(), MyTime.alternateNow)):
-	""" Build century regex for last year and the next years (distance).
-		
-	Thereby respect possible run in the test-cases (alternate date used there)
-	"""
-	cent = lambda year, f=cent[0], t=cent[1]: str(year)[f:t]
-	exprset = set( cent(now[0].year + i) for i in (-1, distance) )
-	if len(now) and now[1]:
-		exprset |= set( cent(now[1].year + i) for i in (-1, distance) )
-	return "(?:%s)" % "|".join(exprset) if len(exprset) > 1 else "".join(exprset)
-
 timeRE = TimeRE()

 # %k - one- or two-digit number giving the hour of the day (0-23) on a 24-hour clock,
@@ -63,20 +52,68 @@ timeRE['z'] = r"(?P<z>Z|UTC|GMT|[+-][01]\d(?::?\d{2})?)"
 timeRE['ExZ'] = r"(?P<Z>%s)" % (TZ_ABBR_RE,)
 timeRE['Exz'] = r"(?P<z>(?:%s)?[+-][01]\d(?::?\d{2})?|%s)" % (TZ_ABBR_RE, TZ_ABBR_RE)

+# overwrite default patterns, since they can be non-optimal:
+timeRE['d'] = r"(?P<d>[1-2]\d|[0 ]?[1-9]|3[0-1])"
+timeRE['m'] = r"(?P<m>0?[1-9]|1[0-2])"
+timeRE['Y'] = r"(?P<Y>\d{4})"
+timeRE['H'] = r"(?P<H>[0-1]?\d|2[0-3])"
+timeRE['M'] = r"(?P<M>[0-5]?\d)"
+timeRE['S'] = r"(?P<S>[0-5]?\d|6[0-1])"
+
 # Extend build-in TimeRE with some exact patterns
 # exact two-digit patterns:
-timeRE['Exd'] = r"(?P<d>3[0-1]|[1-2]\d|0[1-9])"
-timeRE['Exm'] = r"(?P<m>1[0-2]|0[1-9])"
-timeRE['ExH'] = r"(?P<H>2[0-3]|[0-1]\d)"
-timeRE['Exk'] = r" ?(?P<H>2[0-3]|[0-1]\d|\d)"
+timeRE['Exd'] = r"(?P<d>[1-2]\d|0[1-9]|3[0-1])"
+timeRE['Exm'] = r"(?P<m>0[1-9]|1[0-2])"
+timeRE['ExH'] = r"(?P<H>[0-1]\d|2[0-3])"
+timeRE['Exk'] = r" ?(?P<H>[0-1]?\d|2[0-3])"
 timeRE['Exl'] = r" ?(?P<I>1[0-2]|\d)"
 timeRE['ExM'] = r"(?P<M>[0-5]\d)"
-timeRE['ExS'] = r"(?P<S>6[0-1]|[0-5]\d)"
-# more precise year patterns, within same century of last year and
-# the next 3 years (for possible long uptime of fail2ban); thereby
-# respect possible run in the test-cases (alternate date used there):
-timeRE['ExY'] = r"(?P<Y>%s\d)" % _getYearCentRE(cent=(0,3), distance=3)
-timeRE['Exy'] = r"(?P<y>%s\d)" % _getYearCentRE(cent=(2,3), distance=3)
+timeRE['ExS'] = r"(?P<S>[0-5]\d|6[0-1])"
+
+def _updateTimeRE():
+	def _getYearCentRE(cent=(0,3), distance=3, now=(MyTime.now(), MyTime.alternateNow)):
+		""" Build century regex for last year and the next years (distance).
+			
+		Thereby respect possible run in the test-cases (alternate date used there)
+		"""
+		cent = lambda year, f=cent[0], t=cent[1]: str(year)[f:t]
+		def grp(exprset):
+			c = None
+			if len(exprset) > 1:
+				for i in exprset:
+					if c is None or i[0:-1] == c:
+						c = i[0:-1]
+					else:
+						c = None
+						break
+				if not c:
+					for i in exprset:
+						if c is None or i[0] == c:
+							c = i[0]
+						else:
+							c = None
+							break
+				if c:
+					return "%s%s" % (c, grp([i[len(c):] for i in exprset]))
+			return ("(?:%s)" % "|".join(exprset) if len(exprset[0]) > 1 else "[%s]" % "".join(exprset)) \
+				if len(exprset) > 1 else "".join(exprset)
+		exprset = set( cent(now[0].year + i) for i in (-1, distance) )
+		if len(now) > 1 and now[1]:
+			exprset |= set( cent(now[1].year + i) for i in xrange(-1, now[0].year-now[1].year+1, distance) )
+		return grp(sorted(list(exprset)))
+
+	# more precise year patterns, within same century of last year and
+	# the next 3 years (for possible long uptime of fail2ban); thereby
+	# consider possible run in the test-cases (alternate date used there),
+	# so accept years: 20xx (from test-date or 2001 up to current century)
+	timeRE['ExY'] = r"(?P<Y>%s\d)" % _getYearCentRE(cent=(0,3), distance=3, 
+		now=(datetime.datetime.now(), datetime.datetime.fromtimestamp(
+					min(MyTime.alternateNowTime or 978393600, 978393600))
+				)
+		)
+	timeRE['Exy'] = r"(?P<y>\d{2})"
+
+_updateTimeRE()

 def getTimePatternRE():
 	keys = timeRE.keys()
@@ -168,9 +205,9 @@ def reGroupDictStrptime(found_dict, msec=False, default_tz=None):
 	"""

 	now = \
-	year = month = day = hour = minute = tzoffset = \
+	year = month = day = tzoffset = \
 	weekday = julian = week_of_year = None
-	second = fraction = 0
+	hour = minute = second = fraction = 0
 	for key, val in found_dict.iteritems():
 		if val is None: continue
 		# Directives not explicitly handled below:
@@ -291,9 +328,8 @@ def reGroupDictStrptime(found_dict, msec=False, default_tz=None):
 			date_result -= datetime.timedelta(days=1)
 	if assume_year:
 		if not now: now = MyTime.now()
-		if date_result > now:
-			# Could be last year?
-			# also reset month and day as it's not yesterday...
+		if date_result > now + datetime.timedelta(days=1): # ignore by timezone issues (+24h)
+			# assume last year - also reset month and day as it's not yesterday...
 			date_result = date_result.replace(
 				year=year-1, month=month, day=day)

@@ -218,21 +218,20 @@ class FailTicket(Ticket):

 	def __init__(self, ip=None, time=None, matches=None, data={}, ticket=None):
 		# this class variables:
-		self.__retry = 0
-		self.__lastReset = None
+		self._firstTime = None
+		self._retry = 1
 		# create/copy using default ticket constructor:
 		Ticket.__init__(self, ip, time, matches, data, ticket)
 		# init:
-		if ticket is None:
-			self.__lastReset = time if time is not None else self.getTime()
-		if not self.__retry:
-			self.__retry = self._data['failures'];
+		if not isinstance(ticket, FailTicket):
+			self._firstTime = time if time is not None else self.getTime()
+			self._retry = self._data.get('failures', 1)

 	def setRetry(self, value):
 		""" Set artificial retry count, normally equal failures / attempt,
 		used in incremental features (BanTimeIncr) to increase retry count for bad IPs
 		"""
-		self.__retry = value
+		self._retry = value
 		if not self._data['failures']:
 			self._data['failures'] = 1
 		if not value:
@@ -243,10 +242,23 @@ class FailTicket(Ticket):
 		""" Returns failures / attempt count or
 		artificial retry count increased for bad IPs
 		"""
-		return max(self.__retry, self._data['failures'])
+		return self._retry
+
+	def adjustTime(self, time, maxTime):
+		""" Adjust time of ticket and current attempts count considering given maxTime
+		as estimation from rate by previous known interval (if it exceeds the findTime)
+		"""
+		if time > self._time:
+			# expand current interval and attemps count (considering maxTime):
+			if self._firstTime < time - maxTime:
+				# adjust retry calculated as estimation from rate by previous known interval:
+				self._retry = int(round(self._retry / float(time - self._firstTime) * maxTime))
+				self._firstTime = time - maxTime
+			# last time of failure:
+			self._time = time

 	def inc(self, matches=None, attempt=1, count=1):
-		self.__retry += count
+		self._retry += count
 		self._data['failures'] += attempt
 		if matches:
 			# we should duplicate "matches", because possibly referenced to multiple tickets:
@@ -255,19 +267,6 @@ class FailTicket(Ticket):
 			else:
 				self._data['matches'] = matches

-	def setLastTime(self, value):
-		if value > self._time:
-			self._time = value
-	
-	def getLastTime(self):
-		return self._time
-
-	def getLastReset(self):
-		return self.__lastReset
-
-	def setLastReset(self, value):
-		self.__lastReset = value
-
 ##
 # Ban Ticket.
 #
@@ -118,6 +118,9 @@ class Transmitter:
 			if len(value) == 1 and value[0] == "--all":
 				return self.__server.setUnbanIP()
 			return self.__server.setUnbanIP(None, value)
+		elif name == "banned":
+			# check IP is banned in all jails:
+			return self.__server.banned(None, command[1:])
 		elif name == "echo":
 			return command[1:]
 		elif name == "server-status":
@@ -170,6 +173,11 @@ class Transmitter:
 				return self.__server.getSyslogSocket()
 			else:
 				raise Exception("Failed to change syslog socket")
+		elif name == "allowipv6":
+			value = command[1]
+			self.__server.setIPv6IsAllowed(value)
+			if self.__quiet: return
+			return value
 		#Thread
 		elif name == "thread":
 			value = command[1]
@@ -274,7 +282,8 @@ class Transmitter:
 			value = command[2]
 			self.__server.setPrefRegex(name, value)
 			if self.__quiet: return
-			return self.__server.getPrefRegex(name)
+			v = self.__server.getPrefRegex(name)
+			return v.getRegex() if v else ""
 		elif command[1] == "addfailregex":
 			value = command[2]
 			self.__server.addFailRegex(name, value, multiple=multiple)
@@ -424,7 +433,10 @@ class Transmitter:
 				return None
 			else:
 				return db.purgeage
-		# Filter
+		# Jail, Filter
+		elif command[1] == "banned":
+			# check IP is banned in all jails:
+			return self.__server.banned(name, command[2:])
 		elif command[1] == "logpath":
 			return self.__server.getLogPath(name)
 		elif command[1] == "logencoding":
@@ -440,7 +452,8 @@ class Transmitter:
 		elif command[1] == "ignorecache":
 			return self.__server.getIgnoreCache(name)
 		elif command[1] == "prefregex":
-			return self.__server.getPrefRegex(name)
+			v = self.__server.getPrefRegex(name)
+			return v.getRegex() if v else ""
 		elif command[1] == "failregex":
 			return self.__server.getFailRegex(name)
 		elif command[1] == "ignoreregex":
@@ -125,6 +125,10 @@ class Utils():
 			with self.__lock:
 				self._cache.pop(k, None)

+		def clear(self):
+			with self.__lock:
+				self._cache.clear()
+

 	@staticmethod
 	def setFBlockMode(fhandle, value):
@@ -260,7 +264,6 @@ class Utils():
 				if stdout is not None and stdout != '' and std_level >= logSys.getEffectiveLevel():
 					for l in stdout.splitlines():
 						logSys.log(std_level, "%x -- stdout: %r", realCmdId, uni_decode(l))
-				popen.stdout.close()
 			if popen.stderr:
 				try:
 					if retcode is None or retcode < 0:
@@ -271,7 +274,9 @@ class Utils():
 				if stderr is not None and stderr != '' and std_level >= logSys.getEffectiveLevel():
 					for l in stderr.splitlines():
 						logSys.log(std_level, "%x -- stderr: %r", realCmdId, uni_decode(l))
-				popen.stderr.close()
+
+		if popen.stdout: popen.stdout.close()
+		if popen.stderr: popen.stderr.close()

 		success = False
 		if retcode in success_codes:
@@ -1,157 +0,0 @@
-# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*-
-# vi: set ft=python sts=4 ts=4 sw=4 noet :
-
-# This file is part of Fail2Ban.
-#
-# Fail2Ban is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# Fail2Ban is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Fail2Ban; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
-
-import os
-import unittest
-import sys
-from functools import wraps
-from socket import timeout
-from ssl import SSLError
-
-from ..actiontestcase import CallingMap
-from ..dummyjail import DummyJail
-from ..servertestcase import IPAddr
-from ..utils import LogCaptureTestCase, CONFIG_DIR
-
-if sys.version_info >= (3, ): # pragma: 2.x no cover
-	from urllib.error import HTTPError, URLError
-else: # pragma: 3.x no cover
-	from urllib2 import HTTPError, URLError
-
-def skip_if_not_available(f):
-	"""Helper to decorate tests to skip in case of timeout/http-errors like "502 bad gateway".
-	"""
-	@wraps(f)
-	def wrapper(self, *args):
-		try:
-			return f(self, *args)
-		except (SSLError, HTTPError, URLError, timeout) as e: # pragma: no cover - timeout/availability issues
-			if not isinstance(e, timeout) and 'timed out' not in str(e):
-				if not hasattr(e, 'code') or e.code > 200 and e.code <= 404:
-					raise
-			raise unittest.SkipTest('Skip test because of %s' % e)
-	return wrapper
-
-if sys.version_info >= (2,7): # pragma: no cover - may be unavailable
-	class BadIPsActionTest(LogCaptureTestCase):
-
-		available = True, None
-		pythonModule = None
-		modAction = None
-
-		@skip_if_not_available
-		def setUp(self):
-			"""Call before every test case."""
-			super(BadIPsActionTest, self).setUp()
-			unittest.F2B.SkipIfNoNetwork()
-
-			self.jail = DummyJail()
-
-			self.jail.actions.add("test")
-
-			pythonModuleName = os.path.join(CONFIG_DIR, "action.d", "badips.py")
-
-			# check availability (once if not alive, used shorter timeout as in test cases):
-			if BadIPsActionTest.available[0]:
-				if not BadIPsActionTest.modAction:
-					if not BadIPsActionTest.pythonModule:
-						BadIPsActionTest.pythonModule = self.jail.actions._load_python_module(pythonModuleName)
-					BadIPsActionTest.modAction = BadIPsActionTest.pythonModule.Action
-					self.jail.actions._load_python_module(pythonModuleName)
-				BadIPsActionTest.available = BadIPsActionTest.modAction.isAvailable(timeout=2 if unittest.F2B.fast else 30)
-			if not BadIPsActionTest.available[0]:
-				raise unittest.SkipTest('Skip test because service is not available: %s' % BadIPsActionTest.available[1])
-
-			self.jail.actions.add("badips", pythonModuleName, initOpts={
-				'category': "ssh",
-				'banaction': "test",
-				'age': "2w",
-				'score': 5,
-				#'key': "fail2ban-test-suite",
-				#'bankey': "fail2ban-test-suite",
-				'timeout': (3 if unittest.F2B.fast else 60),
-				})
-			self.action = self.jail.actions["badips"]
-
-		def tearDown(self):
-			"""Call after every test case."""
-			# Must cancel timer!
-			if self.action._timer:
-				self.action._timer.cancel()
-			super(BadIPsActionTest, self).tearDown()
-
-		@skip_if_not_available
-		def testCategory(self):
-			categories = self.action.getCategories()
-			self.assertIn("ssh", categories)
-			self.assertTrue(len(categories) >= 10)
-
-			self.assertRaises(
-				ValueError, setattr, self.action, "category",
-				"invalid-category")
-
-			# Not valid for reporting category...
-			self.assertRaises(
-				ValueError, setattr, self.action, "category", "mail")
-			# but valid for blacklisting.
-			self.action.bancategory = "mail"
-
-		@skip_if_not_available
-		def testScore(self):
-			self.assertRaises(ValueError, setattr, self.action, "score", -5)
-			self.action.score = 3
-			self.action.score = "3"
-
-		@skip_if_not_available
-		def testBanaction(self):
-			self.assertRaises(
-				ValueError, setattr, self.action, "banaction",
-				"invalid-action")
-			self.action.banaction = "test"
-
-		@skip_if_not_available
-		def testUpdateperiod(self):
-			self.assertRaises(
-				ValueError, setattr, self.action, "updateperiod", -50)
-			self.assertRaises(
-				ValueError, setattr, self.action, "updateperiod", 0)
-			self.action.updateperiod = 900
-			self.action.updateperiod = "900"
-
-		@skip_if_not_available
-		def testStartStop(self):
-			self.action.start()
-			self.assertTrue(len(self.action._bannedips) > 10,
-				"%s is fewer as 10: %r" % (len(self.action._bannedips), self.action._bannedips))
-			self.action.stop()
-			self.assertTrue(len(self.action._bannedips) == 0)
-
-		@skip_if_not_available
-		def testBanIP(self):
-			aInfo = CallingMap({
-				'ip': IPAddr('192.0.2.1')
-			})
-			self.action.ban(aInfo)
-			self.assertLogged('badips.com: ban', wait=True)
-			self.pruneLog()
-			# produce an error using wrong category/IP:
-			self.action._category = 'f2b-this-category-dont-available-test-suite-only'
-			aInfo['ip'] = ''
-			self.assertRaises(BadIPsActionTest.pythonModule.HTTPError, self.action.ban, aInfo)
-			self.assertLogged('IP is invalid', 'invalid category', wait=True, all=False)
@@ -96,6 +96,8 @@ class ExecuteActions(LogCaptureTestCase):
 		self.assertLogged("stdout: %r" % 'ip flush', "stdout: %r" % 'ip stop')
 		self.assertEqual(self.__actions.status(),[("Currently banned", 0 ),
               ("Total banned", 0 ), ("Banned IP list", [] )])
+		self.assertEqual(self.__actions.status('short'),[("Currently banned", 0 ),
+               ("Total banned", 0 )])

 	def testAddActionPython(self):
 		self.__actions.add(
@@ -252,7 +252,7 @@ class CommandActionTest(LogCaptureTestCase):
 		delattr(self.__action, 'ac')
 		# produce self-referencing query except:
 		self.assertRaisesRegexp(ValueError, r"possible self referencing definitions in query",
-			lambda: self.__action.replaceTag("<x<x<x<x<x<x<x<x<x<x<x<x<x<x<x<x<x<x<x<x<x>>>>>>>>>>>>>>>>>>>>>", 
+			lambda: self.__action.replaceTag("<x"*30+">"*30,
 				self.__action._properties, conditional="family=inet6")
 		)

@@ -27,6 +27,7 @@ __license__ = "GPL"
 import unittest

 from ..server.banmanager import BanManager
+from ..server.ipdns import DNSUtils
 from ..server.ticket import BanTicket

 class AddFailure(unittest.TestCase):
@@ -137,10 +138,10 @@ class StatusExtendedCymruInfo(unittest.TestCase):
 		"""Call before every test case."""
 		super(StatusExtendedCymruInfo, self).setUp()
 		unittest.F2B.SkipIfNoNetwork()
-		self.__ban_ip = "93.184.216.34"
-		self.__asn = "15133"
-		self.__country = "EU"
-		self.__rir = "ripencc"
+		self.__ban_ip = iter(DNSUtils.dnsToIp("resolver1.opendns.com")).next()
+		self.__asn = "36692"
+		self.__country = "US"
+		self.__rir = "arin"
 		ticket = BanTicket(self.__ban_ip, 1167605999.0)
 		self.__banManager = BanManager()
 		self.assertTrue(self.__banManager.addBanTicket(ticket))
@@ -87,6 +87,21 @@ option = %s
 		self.assertTrue(self.c.read(f))	# we got some now
 		return self.c.getOptions('section', [("int", 'option')])['option']

+	def testConvert(self):
+		self.c.add_section("Definition")
+		self.c.set("Definition", "a", "1")
+		self.c.set("Definition", "b", "1")
+		self.c.set("Definition", "c", "test")
+		opts = self.c.getOptions("Definition", 
+			(('int', 'a', 0), ('bool', 'b', 0), ('int', 'c', 0)))
+		self.assertSortedEqual(opts, {'a': 1, 'b': True, 'c': 0})
+		opts = self.c.getOptions("Definition", 
+			(('int', 'a'), ('bool', 'b'), ('int', 'c')))
+		self.assertSortedEqual(opts, {'a': 1, 'b': True, 'c': None})
+		opts = self.c.getOptions("Definition", 
+			{'a': ('int', 0), 'b': ('bool', 0), 'c': ('int', 0)})
+		self.assertSortedEqual(opts, {'a': 1, 'b': True, 'c': 0})
+
 	def testInaccessibleFile(self):
 		f = os.path.join(self.d, "d.conf")  # inaccessible file
 		self._write('d.conf', 0)
@@ -249,6 +264,17 @@ class JailReaderTest(LogCaptureTestCase):
 	def __init__(self, *args, **kwargs):
 		super(JailReaderTest, self).__init__(*args, **kwargs)

+	def testSplitWithOptions(self):
+		# covering all separators - new-line and spaces:
+		for sep in ('\n', '\t', ' '):
+			self.assertEqual(splitWithOptions('a%sb' % (sep,)),           ['a',           'b'])
+			self.assertEqual(splitWithOptions('a[x=y]%sb' % (sep,)),      ['a[x=y]',      'b'])
+			self.assertEqual(splitWithOptions('a[x=y][z=z]%sb' % (sep,)), ['a[x=y][z=z]', 'b'])
+			self.assertEqual(splitWithOptions('a[x="y][z"]%sb' % (sep,)), ['a[x="y][z"]', 'b'])
+			self.assertEqual(splitWithOptions('a[x="y z"]%sb' % (sep,)),  ['a[x="y z"]',  'b'])
+			self.assertEqual(splitWithOptions('a[x="y\tz"]%sb' % (sep,)), ['a[x="y\tz"]', 'b'])
+			self.assertEqual(splitWithOptions('a[x="y\nz"]%sb' % (sep,)), ['a[x="y\nz"]', 'b'])
+
 	def testIncorrectJail(self):
 		jail = JailReader('XXXABSENTXXX', basedir=CONFIG_DIR, share_config=CONFIG_DIR_SHARE_CFG)
 		self.assertRaises(ValueError, jail.read)
@@ -328,7 +354,22 @@ class JailReaderTest(LogCaptureTestCase):
 			self.assertFalse(len(o) > 2 and o[2].endswith('regex'))
 			i += 1
 			if i > usednsidx: break
-		
+
+	def testLogTypeOfBackendInJail(self):
+		unittest.F2B.SkipIfCfgMissing(stock=True); # expected include of common.conf
+		# test twice to check cache works peoperly:
+		for i in (1, 2):
+			# backend-related, overwritten in definition, specified in init parameters:
+			for prefline in ('JRNL', 'FILE', 'TEST', 'INIT'):
+				jail = JailReader('checklogtype_'+prefline.lower(), basedir=IMPERFECT_CONFIG,
+					share_config=IMPERFECT_CONFIG_SHARE_CFG, force_enable=True)
+				self.assertTrue(jail.read())
+				self.assertTrue(jail.getOptions())
+				stream = jail.convert()
+				# 'JRNL' for systemd, 'FILE' for file backend, 'TEST' for custom logtype (overwrite it):
+				self.assertEqual([['set', jail.getName(), 'addfailregex', '^%s failure from <HOST>$' % prefline]],
+					[o for o in stream if len(o) > 2 and o[2] == 'addfailregex'])
+
 	def testSplitOption(self):
 		# Simple example
 		option = "mail-whois[name=SSH]"
@@ -340,13 +381,16 @@ class JailReaderTest(LogCaptureTestCase):
 		self.assertEqual(('mail.who_is', {'a':'cat', 'b':'dog'}), extractOptions("mail.who_is[a=cat,b=dog]"))
 		self.assertEqual(('mail--ho_is', {}), extractOptions("mail--ho_is"))

-		self.assertEqual(('mail--ho_is', {}), extractOptions("mail--ho_is['s']"))
-		#print(self.getLog())
-		#self.assertLogged("Invalid argument ['s'] in ''s''")
-
 		self.assertEqual(('mail', {'a': ','}), extractOptions("mail[a=',']"))
+		self.assertEqual(('mail', {'a': 'b'}), extractOptions("mail[a=b, ]"))

-		#self.assertRaises(ValueError, extractOptions ,'mail-how[')
+		self.assertRaises(ValueError, extractOptions ,'mail-how[')
+
+		self.assertRaises(ValueError, extractOptions, """mail[a="test with interim (wrong) "" quotes"]""")
+		self.assertRaises(ValueError, extractOptions, """mail[a='test with interim (wrong) '' quotes']""")
+		self.assertRaises(ValueError, extractOptions, """mail[a='x, y, z', b=x, y, z]""")
+
+		self.assertRaises(ValueError, extractOptions, """mail['s']""")

 		# Empty option
 		option = "abc[]"
@@ -414,8 +458,6 @@ class JailReaderTest(LogCaptureTestCase):
 			('sender', 'f2b-test@example.com'), ('blocklist_de_apikey', 'test-key'), 
 			('action', 
 				'%(action_blocklist_de)s\n'
-				'%(action_badips_report)s\n'
-				'%(action_badips)s\n'
 				'mynetwatchman[port=1234,protocol=udp,agent="%(fail2ban_agent)s"]'
 			),
 		))
@@ -429,16 +471,14 @@ class JailReaderTest(LogCaptureTestCase):
 			if len(cmd) <= 4:
 				continue
 			# differentiate between set and multi-set (wrop it here to single set):
-			if cmd[0] == 'set' and (cmd[4] == 'agent' or cmd[4].endswith('badips.py')):
+			if cmd[0] == 'set' and cmd[4] == 'agent':
 				act.append(cmd)
 			elif cmd[0] == 'multi-set':
 				act.extend([['set'] + cmd[1:4] + o for o in cmd[4] if o[0] == 'agent'])
 		useragent = 'Fail2Ban/%s' % version
-		self.assertEqual(len(act), 4)
+		self.assertEqual(len(act), 2)
 		self.assertEqual(act[0], ['set', 'blocklisttest', 'action', 'blocklist_de', 'agent', useragent])
-		self.assertEqual(act[1], ['set', 'blocklisttest', 'action', 'badips', 'agent', useragent])
-		self.assertEqual(eval(act[2][5]).get('agent', '<wrong>'), useragent)
-		self.assertEqual(act[3], ['set', 'blocklisttest', 'action', 'mynetwatchman', 'agent', useragent])
+		self.assertEqual(act[1], ['set', 'blocklisttest', 'action', 'mynetwatchman', 'agent', useragent])

 	@with_tmpdir
 	def testGlob(self, d):
@@ -468,14 +508,12 @@ class JailReaderTest(LogCaptureTestCase):
 		self.assertRaises(NoSectionError, c.getOptions, 'test', {})


-class FilterReaderTest(unittest.TestCase):
-
-	def __init__(self, *args, **kwargs):
-		super(FilterReaderTest, self).__init__(*args, **kwargs)
-		self.__share_cfg = {}
+class FilterReaderTest(LogCaptureTestCase):

 	def testConvert(self):
-		output = [['multi-set', 'testcase01', 'addfailregex', [
+		output = [
+			['set', 'testcase01', 'maxlines', 1],
+			['multi-set', 'testcase01', 'addfailregex', [
 			"^\\s*(?:\\S+ )?(?:kernel: \\[\\d+\\.\\d+\\] )?(?:@vserver_\\S+ )"
 			"?(?:(?:\\[\\d+\\])?:\\s+[\\[\\(]?sshd(?:\\(\\S+\\))?[\\]\\)]?:?|"
 			"[\\[\\(]?sshd(?:\\(\\S+\\))?[\\]\\)]?:?(?:\\[\\d+\\])?:)?\\s*(?:"
@@ -497,7 +535,6 @@ class FilterReaderTest(unittest.TestCase):
 			['set', 'testcase01', 'addjournalmatch',
 				"FIELD= with spaces ", "+", "AFIELD= with + char and spaces"],
 			['set', 'testcase01', 'datepattern', "%Y %m %d %H:%M:%S"],
-			['set', 'testcase01', 'maxlines', 1], # Last for overide test
 		]
 		filterReader = FilterReader("testcase01", "testcase01", {})
 		filterReader.setBaseDir(TEST_FILES_DIR)
@@ -514,9 +551,18 @@ class FilterReaderTest(unittest.TestCase):
 		filterReader.read()
 		#filterReader.getOptions(["failregex", "ignoreregex"])
 		filterReader.getOptions(None)
-		output[-1][-1] = "5"
+		output[0][-1] = 5; # maxlines = 5
 		self.assertSortedEqual(filterReader.convert(), output)

+	def testConvertOptions(self):
+		filterReader = FilterReader("testcase01", "testcase01", {'maxlines': '<test>', 'test': 'X'},
+		  share_config=TEST_FILES_DIR_SHARE_CFG, basedir=TEST_FILES_DIR)
+		filterReader.read()
+		filterReader.getOptions(None)
+		opts = filterReader.getCombined();
+		self.assertNotEqual(opts['maxlines'], 'X'); # wrong int value 'X' for 'maxlines'
+		self.assertLogged("Wrong int value 'X' for 'maxlines'. Using default one:")
+
 	def testFilterReaderSubstitionDefault(self):
 		output = [['set', 'jailname', 'addfailregex', 'to=sweet@example.com fromip=<IP>']]
 		filterReader = FilterReader('substition', "jailname", {},
@@ -526,6 +572,17 @@ class FilterReaderTest(unittest.TestCase):
 		c = filterReader.convert()
 		self.assertSortedEqual(c, output)

+	def testFilterReaderSubstKnown(self):
+		# testcase02.conf + testcase02.local, test covering that known/option is not overridden
+		# with unmodified (not available) value of option from .local config file, so wouldn't
+		# cause self-recursion if option already has a reference to known/option in .conf file.
+		filterReader = FilterReader('testcase02', "jailname", {},
+		  share_config=TEST_FILES_DIR_SHARE_CFG, basedir=TEST_FILES_DIR)
+		filterReader.read()
+		filterReader.getOptions(None)
+		opts = filterReader.getCombined()
+		self.assertTrue('sshd' in opts['failregex'])
+		
 	def testFilterReaderSubstitionSet(self):
 		output = [['set', 'jailname', 'addfailregex', 'to=sour@example.com fromip=<IP>']]
 		filterReader = FilterReader('substition', "jailname", {'honeypot': 'sour@example.com'},
@@ -694,9 +751,9 @@ class JailsReaderTest(LogCaptureTestCase):
 		         ['add', 'tz_correct', 'auto'],
 			 ['start', 'tz_correct'],
 			 ['config-error',
-				"Jail 'brokenactiondef' skipped, because of wrong configuration: Invalid action definition 'joho[foo'"],
+				"Jail 'brokenactiondef' skipped, because of wrong configuration: Invalid action definition 'joho[foo': unexpected option syntax"],
 			 ['config-error',
-				"Jail 'brokenfilterdef' skipped, because of wrong configuration: Invalid filter definition 'flt[test'"],
+				"Jail 'brokenfilterdef' skipped, because of wrong configuration: Invalid filter definition 'flt[test': unexpected option syntax"],
 			 ['config-error',
 				"Jail 'missingaction' skipped, because of wrong configuration: Unable to read action 'noactionfileforthisaction'"],
 			 ['config-error',
@@ -917,6 +974,7 @@ class JailsReaderTest(LogCaptureTestCase):
 		  ['set', 'syslogsocket', 'auto'],
 		  ['set', 'loglevel', "INFO"],
 		  ['set', 'logtarget', '/var/log/fail2ban.log'],
+		  ['set', 'allowipv6', 'auto'],
 		  ['set', 'dbfile', '/var/lib/fail2ban/fail2ban.sqlite3'],
 		  ['set', 'dbmaxmatches', 10],
 		  ['set', 'dbpurgeage', '1d'],
@@ -0,0 +1,31 @@
+# Fail2Ban configuration file
+#
+
+[INCLUDES]
+
+# Read common prefixes (logtype is set in default section)
+before = ../../../../config/filter.d/common.conf
+
+[Definition]
+
+_daemon = test
+
+failregex = ^<lt_<logtype>/__prefix_line> failure from <HOST>$
+ignoreregex = 
+
+# following sections define prefix line considering logtype:
+
+# backend-related (retrieved from backend, overwrite default):
+[lt_file]
+__prefix_line = FILE
+
+[lt_journal]
+__prefix_line = JRNL
+
+# specified in definition section of filter (see filter checklogtype_test.conf):
+[lt_test]
+__prefix_line = TEST
+
+# specified in init parameter of jail (see ../jail.conf, jail checklogtype_init):
+[lt_init]
+__prefix_line = INIT
@@ -0,0 +1,12 @@
+# Fail2Ban configuration file
+#
+
+[INCLUDES]
+
+# Read common prefixes (logtype is set in default section)
+before = checklogtype.conf
+
+[Definition]
+
+# overwrite logtype in definition (no backend anymore):
+logtype = test
@@ -37,7 +37,7 @@ __pam_auth = pam_[a-z]+
 cmnfailre = ^%(__prefix_line_sl)s[aA]uthentication (?:failure|error|failed) for .* from <HOST>( via \S+)?\s*%(__suff)s$
         ^%(__prefix_line_sl)sUser not known to the underlying authentication module for .* from <HOST>\s*%(__suff)s$
         ^%(__prefix_line_sl)sFailed \S+ for invalid user <F-USER>(?P<cond_user>\S+)|(?:(?! from ).)*?</F-USER> from <HOST>%(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$)
-         ^%(__prefix_line_sl)sFailed \b(?!publickey)\S+ for (?P<cond_inv>invalid user )?<F-USER>(?P<cond_user>\S+)|(?(cond_inv)(?:(?! from ).)*?|[^:]+)</F-USER> from <HOST>%(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$)
+         ^%(__prefix_line_sl)sFailed (?:<F-NOFAIL>publickey</F-NOFAIL>|\S+) for (?P<cond_inv>invalid user )?<F-USER>(?P<cond_user>\S+)|(?(cond_inv)(?:(?! from ).)*?|[^:]+)</F-USER> from <HOST>%(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$)
         ^%(__prefix_line_sl)sROOT LOGIN REFUSED FROM <HOST>
         ^%(__prefix_line_sl)s[iI](?:llegal|nvalid) user .*? from <HOST>%(__suff)s$
         ^%(__prefix_line_sl)sUser .+ from <HOST> not allowed because not listed in AllowUsers\s*%(__suff)s$
@@ -57,11 +57,10 @@ mdre-normal =

 mdre-ddos =  ^%(__prefix_line_sl)sDid not receive identification string from <HOST>
             ^%(__prefix_line_sl)sBad protocol version identification '.*' from <HOST>
-             ^%(__prefix_line_sl)sConnection closed by%(__authng_user)s <HOST>%(__on_port_opt)s\s+\[preauth\]\s*$
-             ^%(__prefix_line_sl)sConnection reset by <HOST>
+             ^%(__prefix_line_sl)sConnection (?:closed|reset) by%(__authng_user)s <HOST>%(__on_port_opt)s\s+\[preauth\]\s*$
             ^%(__prefix_line_ml1)sSSH: Server;Ltype: (?:Authname|Version|Kex);Remote: <HOST>-\d+;[A-Z]\w+:.*%(__prefix_line_ml2)sRead from socket failed: Connection reset by peer%(__suff)s$

-mdre-extra = ^%(__prefix_line_sl)sReceived disconnect from <HOST>%(__on_port_opt)s:\s*14: No supported authentication methods available
+mdre-extra = ^%(__prefix_line_sl)sReceived disconnect from <HOST>%(__on_port_opt)s:\s*14: No(?: supported)? authentication methods available
             ^%(__prefix_line_sl)sUnable to negotiate with <HOST>%(__on_port_opt)s: no matching <__alg_match> found.
             ^%(__prefix_line_ml1)sConnection from <HOST>%(__on_port_opt)s%(__prefix_line_ml2)sUnable to negotiate a <__alg_match>
             ^%(__prefix_line_ml1)sConnection from <HOST>%(__on_port_opt)s%(__prefix_line_ml2)sno matching <__alg_match> found:
@@ -74,3 +74,28 @@ journalmatch = _COMM=test
 maxlines = 2
 usedns = no
 enabled = false
+
+[checklogtype_jrnl]
+filter = checklogtype
+backend = systemd
+action = action
+enabled = false
+
+[checklogtype_file]
+filter = checklogtype
+backend = polling
+logpath = README.md
+action = action
+enabled = false
+
+[checklogtype_test]
+filter = checklogtype_test
+backend = systemd
+action = action
+enabled = false
+
+[checklogtype_init]
+filter = checklogtype_test[logtype=init]
+backend = systemd
+action = action
+enabled = false
@@ -225,6 +225,15 @@ class DatabaseTest(LogCaptureTestCase):
 			self.db.addLog(self.jail, self.fileContainer), None)
 		os.remove(filename)

+	def testUpdateJournal(self):
+		self.testAddJail() # Jail required
+		# not yet updated:
+		self.assertEqual(self.db.getJournalPos(self.jail, 'systemd-journal'), None)
+		# update 3 times (insert and 2 updates) and check it was set (and overwritten):
+		for t in (1500000000, 1500000001, 1500000002):
+			self.db.updateJournal(self.jail, 'systemd-journal', t, 'TEST'+str(t))
+			self.assertEqual(self.db.getJournalPos(self.jail, 'systemd-journal'), t)
+
 	def testAddBan(self):
 		self.testAddJail()
 		ticket = FailTicket("127.0.0.1", 0, ["abc\n"])
@@ -482,6 +491,7 @@ class DatabaseTest(LogCaptureTestCase):
 		# test action together with database functionality
 		self.testAddJail() # Jail required
 		self.jail.database = self.db
+		self.db.addJail(self.jail)
 		actions = Actions(self.jail)
 		actions.add(
 			"action_checkainfo",
@@ -330,6 +330,27 @@ class DateDetectorTest(LogCaptureTestCase):
 		dt = '2005 Jun 03'; self.assertEqual(t.matchDate(dt).group(1), dt)
 		dt = '2005 JUN 03'; self.assertEqual(t.matchDate(dt).group(1), dt)

+	def testNotAnchoredCollision(self):
+		# try for patterns with and without word boundaries:
+		for dp in (r'%H:%M:%S', r'{UNB}%H:%M:%S'):
+			dd = DateDetector()
+			dd.appendTemplate(dp)
+			# boundary of timestamp changes right and left (and time is left and right in line):
+			for fmt in ('%s test', '%8s test', 'test %s', 'test %8s'):
+				for dt in (
+					'00:01:02',
+					'00:01:2',
+					'00:1:2',
+					'0:1:2',
+					'00:1:2',
+					'00:01:2',
+					'00:01:02',
+					'0:1:2',
+					'00:01:02',
+				):
+					t = dd.getTime(fmt % dt)
+					self.assertEqual((t[0], t[1].group()), (1123970462.0, dt))
+
 	def testAmbiguousInOrderedTemplates(self):
 		dd = self.datedetector
 		for (debit, line, cnt) in (
@@ -530,6 +551,9 @@ class CustomDateFormatsTest(unittest.TestCase):
 			(1123970401.0,  "^%ExH:%ExM:%ExS**", '00:00:01'),
 			# cover date with current year, in test cases now == Aug 2005 -> back to last year (Sep 2004):
 			(1094068799.0,  "^%m/%d %ExH:%ExM:%ExS**", '09/01 21:59:59'),
+			# no time (only date) in pattern, assume local 00:00:00 for H:M:S :
+			(1093989600.0,  "^%Y-%m-%d**",   '2004-09-01'),
+			(1093996800.0,  "^%Y-%m-%d%z**", '2004-09-01Z'),
 		):
 			logSys.debug('== test: %r', (matched, dp, line))
 			dd = DateDetector()
@@ -35,7 +35,6 @@ class DummyJail(Jail):
 		self.lock = Lock()
 		self.queue = []
 		super(DummyJail, self).__init__(name='DummyJail', backend=backend)
-		self.__db = None
 		self.__actions = Actions(self)

 	def __len__(self):
@@ -50,6 +49,10 @@ class DummyJail(Jail):
 		with self.lock:
 			return bool(self.queue)

+	@property
+	def hasFailTickets(self):
+		return bool(self.queue)
+
 	def putFailTicket(self, ticket):
 		with self.lock:
 			self.queue.append(ticket)
@@ -63,7 +66,7 @@ class DummyJail(Jail):

 	@property
 	def name(self):
-		return "DummyJail #%s with %d tickets" % (id(self), len(self))
+		return "DummyJail" + ("" if self.database else " #%s with %d tickets" % (id(self), len(self)))

 	@property
 	def idle(self):
@@ -73,14 +76,6 @@ class DummyJail(Jail):
 	def idle(self, value):
 		pass

-	@property
-	def database(self):
-		return self.__db;
-	
-	@database.setter
-	def database(self, value):
-		self.__db = value;
-
 	@property
 	def actions(self):
 		return self.__actions;
@@ -37,7 +37,7 @@ from threading import Thread

 from ..client import fail2banclient, fail2banserver, fail2bancmdline
 from ..client.fail2bancmdline import Fail2banCmdLine
-from ..client.fail2banclient import exec_command_line as _exec_client, VisualWait
+from ..client.fail2banclient import exec_command_line as _exec_client, CSocket, VisualWait
 from ..client.fail2banserver import Fail2banServer, exec_command_line as _exec_server
 from .. import protocol
 from ..server import server
@@ -421,6 +421,14 @@ class Fail2banClientServerBase(LogCaptureTestCase):
 		self.assertRaises(exitType, self.exec_command_line[0],
 			(self.exec_command_line[1:] + startparams + args))

+	def execCmdDirect(self, startparams, *args):
+		sock = startparams[startparams.index('-s')+1]
+		s = CSocket(sock)
+		try:
+			return s.send(args)
+		finally:
+			s.close()
+
 	#
 	# Common tests
 	#
@@ -615,12 +623,6 @@ class Fail2banClientTest(Fail2banClientServerBase):
 		self.assertLogged("Base configuration directory " + pjoin(tmp, "miss") + " does not exist")
 		self.pruneLog()

-		## wrong socket
-		self.execCmd(FAILED, (),
-			"--async", "-c", pjoin(tmp, "config"), "-s", pjoin(tmp, "miss/f2b.sock"), "start")
-		self.assertLogged("There is no directory " + pjoin(tmp, "miss") + " to contain the socket file")
-		self.pruneLog()
-
 		## not running
 		self.execCmd(FAILED, (),
 			"-c", pjoin(tmp, "config"), "-s", pjoin(tmp, "f2b.sock"), "reload")
@@ -716,12 +718,6 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		self.assertLogged("Base configuration directory " + pjoin(tmp, "miss") + " does not exist")
 		self.pruneLog()

-		## wrong socket
-		self.execCmd(FAILED, (),
-			"-c", pjoin(tmp, "config"), "-x", "-s", pjoin(tmp, "miss/f2b.sock"))
-		self.assertLogged("There is no directory " + pjoin(tmp, "miss") + " to contain the socket file")
-		self.pruneLog()
-
 		## already exists:
 		open(pjoin(tmp, "f2b.sock"), 'a').close()
 		self.execCmd(FAILED, (),
@@ -860,7 +856,7 @@ class Fail2banServerTest(Fail2banClientServerBase):
 				"action = ",
 				"         test-action2[name='%(__name__)s', restore='restored: <restored>', info=', err-code: <F-ERRCODE>']" \
 					if 2 in actions else "",
-				"         test-action2[name='%(__name__)s', actname=test-action3, _exec_once=1, restore='restored: <restored>']"
+				"         test-action2[name='%(__name__)s', actname=test-action3, _exec_once=1, restore='restored: <restored>',"
 										" actionflush=<_use_flush_>]" \
 					if 3 in actions else "",
 				"logpath = " + test2log,
@@ -973,8 +969,8 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		# leave action2 just to test restored interpolation:
 		_write_jail_cfg(actions=[2,3])
 		
-		# write new failures:
 		self.pruneLog("[test-phase 2b]")
+		# write new failures:
 		_write_file(test2log, "w+", *(
 			(str(int(MyTime.time())) + "   error 403 from 192.0.2.2: test 2",) * 3 +
 		  (str(int(MyTime.time())) + "   error 403 from 192.0.2.3: test 2",) * 3 +
@@ -987,13 +983,19 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		self.assertLogged(
 			"2 ticket(s) in 'test-jail2",
 			"5 ticket(s) in 'test-jail1", all=True, wait=MID_WAITTIME)
+		# ban manually to cover restore in restart (phase 2c):
+		self.execCmd(SUCCESS, startparams,
+			"set", "test-jail2", "banip", "192.0.2.9")
+		self.assertLogged(
+			"3 ticket(s) in 'test-jail2", wait=MID_WAITTIME)
 		self.assertLogged(
 			"[test-jail1] Ban 192.0.2.2",
 			"[test-jail1] Ban 192.0.2.3",
 			"[test-jail1] Ban 192.0.2.4",
 			"[test-jail1] Ban 192.0.2.8",
 			"[test-jail2] Ban 192.0.2.4",
-			"[test-jail2] Ban 192.0.2.8", all=True)
+			"[test-jail2] Ban 192.0.2.8", 
+			"[test-jail2] Ban 192.0.2.9", all=True)
 		# test ips at all not visible for jail2:
 		self.assertNotLogged(
 			"[test-jail2] Found 192.0.2.2", 
@@ -1001,6 +1003,30 @@ class Fail2banServerTest(Fail2banClientServerBase):
 			"[test-jail2] Found 192.0.2.3", 
 			"[test-jail2] Ban 192.0.2.3", 
 			all=True)
+		# test banned command:
+		self.assertSortedEqual(self.execCmdDirect(startparams,
+			'banned'), (0, [
+				{'test-jail1': ['192.0.2.4', '192.0.2.1', '192.0.2.8', '192.0.2.3', '192.0.2.2']},
+				{'test-jail2': ['192.0.2.4', '192.0.2.9', '192.0.2.8']}
+			]
+		))
+		self.assertSortedEqual(self.execCmdDirect(startparams,
+			'banned', '192.0.2.1', '192.0.2.4', '192.0.2.222'), (0, [
+			  ['test-jail1'], ['test-jail1', 'test-jail2'], []
+			]
+		))
+		self.assertSortedEqual(self.execCmdDirect(startparams,
+			'get', 'test-jail1', 'banned')[1], [
+				'192.0.2.4', '192.0.2.1', '192.0.2.8', '192.0.2.3', '192.0.2.2'])
+		self.assertSortedEqual(self.execCmdDirect(startparams,
+			'get', 'test-jail2', 'banned')[1], [
+				'192.0.2.4', '192.0.2.9', '192.0.2.8'])
+		self.assertEqual(self.execCmdDirect(startparams,
+			'get', 'test-jail1', 'banned', '192.0.2.3')[1],  1)
+		self.assertEqual(self.execCmdDirect(startparams,
+			'get', 'test-jail1', 'banned', '192.0.2.9')[1],  0)
+		self.assertEqual(self.execCmdDirect(startparams,
+			'get', 'test-jail1', 'banned', '192.0.2.3', '192.0.2.9')[1],  [1, 0])

 		# rotate logs:
 		_write_file(test1log, "w+")
@@ -1013,15 +1039,17 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		self.assertLogged(
 			"Reload finished.",
 			"Restore Ban",
-			"2 ticket(s) in 'test-jail2", all=True, wait=MID_WAITTIME)
+			"3 ticket(s) in 'test-jail2", all=True, wait=MID_WAITTIME)
 		# stop/start and unban/restore ban:
 		self.assertLogged(
-			"Jail 'test-jail2' stopped",
-			"Jail 'test-jail2' started",
 			"[test-jail2] Unban 192.0.2.4",
 			"[test-jail2] Unban 192.0.2.8",
+			"[test-jail2] Unban 192.0.2.9",
+			"Jail 'test-jail2' stopped",
+			"Jail 'test-jail2' started",
 			"[test-jail2] Restore Ban 192.0.2.4",
-			"[test-jail2] Restore Ban 192.0.2.8", all=True
+			"[test-jail2] Restore Ban 192.0.2.8",
+			"[test-jail2] Restore Ban 192.0.2.9", all=True
 		)
 		# test restored is 1 (only test-action2):
 		self.assertLogged(
@@ -1055,7 +1083,8 @@ class Fail2banServerTest(Fail2banClientServerBase):
 			"Jail 'test-jail2' stopped",
 			"Jail 'test-jail2' started",
 			"[test-jail2] Unban 192.0.2.4",
-			"[test-jail2] Unban 192.0.2.8", all=True
+			"[test-jail2] Unban 192.0.2.8",
+			"[test-jail2] Unban 192.0.2.9", all=True
 		)
 		# test unban (action2):
 		self.assertLogged(
@@ -1129,13 +1158,41 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		self.assertNotLogged("[test-jail1] Found 192.0.2.5")

 		# unban single ips:
-		self.pruneLog("[test-phase 6]")
+		self.pruneLog("[test-phase 6a]")
 		self.execCmd(SUCCESS, startparams,
 			"--async", "unban", "192.0.2.5", "192.0.2.6")
 		self.assertLogged(
 			"192.0.2.5 is not banned",
 			"[test-jail1] Unban 192.0.2.6", all=True, wait=MID_WAITTIME
 		)
+		# unban ips by subnet (cidr/mask):
+		self.pruneLog("[test-phase 6b]")
+		self.execCmd(SUCCESS, startparams,
+			"--async", "unban", "192.0.2.2/31")
+		self.assertLogged(
+			"[test-jail1] Unban 192.0.2.2",
+			"[test-jail1] Unban 192.0.2.3", all=True, wait=MID_WAITTIME
+		)		
+		self.execCmd(SUCCESS, startparams,
+			"--async", "unban", "192.0.2.8/31", "192.0.2.100/31")
+		self.assertLogged(
+			"[test-jail1] Unban 192.0.2.8",
+			"192.0.2.100/31 is not banned", all=True, wait=MID_WAITTIME)
+
+		# ban/unban subnet(s):
+		self.pruneLog("[test-phase 6c]")
+		self.execCmd(SUCCESS, startparams,
+			"--async", "set", "test-jail1", "banip", "192.0.2.96/28", "192.0.2.112/28")
+		self.assertLogged(
+			"[test-jail1] Ban 192.0.2.96/28",
+			"[test-jail1] Ban 192.0.2.112/28", all=True, wait=MID_WAITTIME
+		)
+		self.execCmd(SUCCESS, startparams,
+			"--async", "set", "test-jail1", "unbanip", "192.0.2.64/26"); # contains both subnets .96/28 and .112/28
+		self.assertLogged(
+			"[test-jail1] Unban 192.0.2.96/28",
+			"[test-jail1] Unban 192.0.2.112/28", all=True, wait=MID_WAITTIME
+		)

 		# reload all (one jail) with unban all:
 		self.pruneLog("[test-phase 7]")
@@ -1146,8 +1203,6 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		self.assertLogged(
 			"Jail 'test-jail1' reloaded",
 			"[test-jail1] Unban 192.0.2.1",
-			"[test-jail1] Unban 192.0.2.2",
-			"[test-jail1] Unban 192.0.2.3",
 			"[test-jail1] Unban 192.0.2.4", all=True
 		)
 		# no restart occurred, no more ban (unbanned all using option "--unban"):
@@ -1155,8 +1210,6 @@ class Fail2banServerTest(Fail2banClientServerBase):
 			"Jail 'test-jail1' stopped",
 			"Jail 'test-jail1' started",
 			"[test-jail1] Ban 192.0.2.1",
-			"[test-jail1] Ban 192.0.2.2",
-			"[test-jail1] Ban 192.0.2.3",
 			"[test-jail1] Ban 192.0.2.4", all=True
 		)

@@ -1228,7 +1281,7 @@ class Fail2banServerTest(Fail2banClientServerBase):
 			'backend = polling',
 			'usedns = no',
 			'logpath = %(tmp)s/blck-failures.log',
-			'action = nginx-block-map[blck_lst_reload="", blck_lst_file="%(tmp)s/blck-lst.map"]',
+			'action = nginx-block-map[srv_cmd="echo nginx", srv_pid="%(tmp)s/f2b.pid", blck_lst_file="%(tmp)s/blck-lst.map"]',
 			'         blocklist_de[actionban=\'curl() { echo "*** curl" "$*";}; <Definition/actionban>\', email="Fail2Ban <fail2ban@localhost>", '
 													  'apikey="TEST-API-KEY", agent="fail2ban-test-agent", service=<name>]',
 			'filter =',
@@ -1268,6 +1321,8 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		self.assertIn('\\125-000-004 1;\n', mp)
 		self.assertIn('\\125-000-005 1;\n', mp)

+		# check nginx reload is logged (pid of fail2ban is used to simulate success check nginx is running):
+		self.assertLogged("stdout: 'nginx -qt'", "stdout: 'nginx -s reload'", all=True)
 		# check blocklist_de substitution (e. g. new-line after <matches>):
 		self.assertLogged(
 			"stdout: '*** curl --fail --data-urlencode server=Fail2Ban <fail2ban@localhost>"
@@ -81,15 +81,32 @@ def _test_exec_command_line(*args):
 	return _exit_code

 STR_00 = "Dec 31 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 192.0.2.0"
+STR_00_NODT = "[sshd] error: PAM: Authentication failure for kevin from 192.0.2.0"

 RE_00 = r"(?:(?:Authentication failure|Failed [-/\w+]+) for(?: [iI](?:llegal|nvalid) user)?|[Ii](?:llegal|nvalid) user|ROOT LOGIN REFUSED) .*(?: from|FROM) <HOST>"
-RE_00_ID = r"Authentication failure for <F-ID>.*?</F-ID> from <HOST>$"
-RE_00_USER = r"Authentication failure for <F-USER>.*?</F-USER> from <HOST>$"
+RE_00_ID = r"Authentication failure for <F-ID>.*?</F-ID> from <ADDR>$"
+RE_00_USER = r"Authentication failure for <F-USER>.*?</F-USER> from <ADDR>$"

 FILENAME_01 = os.path.join(TEST_FILES_DIR, "testcase01.log")
 FILENAME_02 = os.path.join(TEST_FILES_DIR, "testcase02.log")
 FILENAME_WRONGCHAR = os.path.join(TEST_FILES_DIR, "testcase-wrong-char.log")

+# STR_ML_SSHD -- multiline log-excerpt with two sessions:
+#   192.0.2.1 (sshd[32307]) makes 2 failed attempts using public keys (without "Disconnecting: Too many authentication"),
+#     and delayed success on accepted (STR_ML_SSHD_OK) or no success by close on preauth phase (STR_ML_SSHD_FAIL)
+#   192.0.2.2 (sshd[32310]) makes 2 failed attempts using public keys (with "Disconnecting: Too many authentication"),
+#     and closed on preauth phase
+STR_ML_SSHD = """Nov 28 09:16:03 srv sshd[32307]: Failed publickey for git from 192.0.2.1 port 57904 ssh2: ECDSA 0e:ff:xx:xx:xx:xx:xx:xx:xx:xx:xx:...
+Nov 28 09:16:03 srv sshd[32307]: Failed publickey for git from 192.0.2.1 port 57904 ssh2: RSA 04:bc:xx:xx:xx:xx:xx:xx:xx:xx:xx:...
+Nov 28 09:16:03 srv sshd[32307]: Postponed publickey for git from 192.0.2.1 port 57904 ssh2 [preauth]
+Nov 28 09:16:05 srv sshd[32310]: Failed publickey for git from 192.0.2.2 port 57910 ssh2: ECDSA 1e:fe:xx:xx:xx:xx:xx:xx:xx:xx:xx:...
+Nov 28 09:16:05 srv sshd[32310]: Failed publickey for git from 192.0.2.2 port 57910 ssh2: RSA 14:ba:xx:xx:xx:xx:xx:xx:xx:xx:xx:...
+Nov 28 09:16:05 srv sshd[32310]: Disconnecting: Too many authentication failures for git [preauth]
+Nov 28 09:16:05 srv sshd[32310]: Connection closed by 192.0.2.2 [preauth]"""
+STR_ML_SSHD_OK   = "Nov 28 09:16:06 srv sshd[32307]: Accepted publickey for git from 192.0.2.1 port 57904 ssh2: DSA 36:48:xx:xx:xx:xx:xx:xx:xx:xx:xx:..."
+STR_ML_SSHD_FAIL = "Nov 28 09:16:06 srv sshd[32307]: Connection closed by 192.0.2.1 [preauth]"
+
+
 FILENAME_SSHD = os.path.join(TEST_FILES_DIR, "logs", "sshd")
 FILTER_SSHD = os.path.join(CONFIG_DIR, 'filter.d', 'sshd.conf')
 FILENAME_ZZZ_SSHD = os.path.join(TEST_FILES_DIR, 'zzz-sshd-obsolete-multiline.log')
@@ -124,6 +141,12 @@ class Fail2banRegexTest(LogCaptureTestCase):
 		))
 		self.assertLogged("Unable to compile regular expression")

+	def testWrongFilterOptions(self):
+		self.assertFalse(_test_exec(
+			"test", "flt[a='x,y,z',b=z,y,x]"
+		))
+		self.assertLogged("Wrong filter name or options", "wrong syntax at 14: y,x", all=True)
+
 	def testDirectFound(self):
 		self.assertTrue(_test_exec(
 			"--datepattern", r"^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?",
@@ -156,7 +179,7 @@ class Fail2banRegexTest(LogCaptureTestCase):
 			"--print-all-matched",
 			FILENAME_01, RE_00
 		))
-		self.assertLogged('Lines: 19 lines, 0 ignored, 13 matched, 6 missed')
+		self.assertLogged('Lines: 19 lines, 0 ignored, 16 matched, 3 missed')

 		self.assertLogged('Error decoding line');
 		self.assertLogged('Continuing to process line ignoring invalid characters')
@@ -170,7 +193,7 @@ class Fail2banRegexTest(LogCaptureTestCase):
 			"--print-all-matched", "--raw",
 			FILENAME_01, RE_00
 		))
-		self.assertLogged('Lines: 19 lines, 0 ignored, 16 matched, 3 missed')
+		self.assertLogged('Lines: 19 lines, 0 ignored, 19 matched, 0 missed')

 	def testDirectRE_1raw_noDns(self):
 		self.assertTrue(_test_exec(
@@ -178,7 +201,7 @@ class Fail2banRegexTest(LogCaptureTestCase):
 			"--print-all-matched", "--raw", "--usedns=no",
 			FILENAME_01, RE_00
 		))
-		self.assertLogged('Lines: 19 lines, 0 ignored, 13 matched, 6 missed')
+		self.assertLogged('Lines: 19 lines, 0 ignored, 16 matched, 3 missed')
 		# usage of <F-ID>\S+</F-ID> causes raw handling automatically:
 		self.pruneLog()
 		self.assertTrue(_test_exec(
@@ -291,10 +314,10 @@ class Fail2banRegexTest(LogCaptureTestCase):
 		# 
 		self.assertTrue(_test_exec(
 			"--usedns", "no", "-d", "^Epoch", "--print-all-matched",
-			"1490349000 FAIL: failure\nhost: 192.0.2.35",
+			"-L", "2", "1490349000 FAIL: failure\nhost: 192.0.2.35",
 			r"^\s*FAIL:\s*.*\nhost:\s+<HOST>$"
 		))
-		self.assertLogged('Lines: 1 lines, 0 ignored, 1 matched, 0 missed')
+		self.assertLogged('Lines: 2 lines, 0 ignored, 2 matched, 0 missed')

 	def testRegexEpochPatterns(self):
 		self.assertTrue(_test_exec(
@@ -340,6 +363,83 @@ class Fail2banRegexTest(LogCaptureTestCase):
 		self.assertTrue(_test_exec('-o', 'user', STR_00, RE_00_USER))
 		self.assertLogged('kevin')
 		self.pruneLog()
+		# complex substitution using tags (ip, user, family):
+		self.assertTrue(_test_exec('-o', '<ip>, <F-USER>, <family>', STR_00, RE_00_USER))
+		self.assertLogged('192.0.2.0, kevin, inet4')
+		self.pruneLog()
+
+	def testNoDateTime(self):
+		# datepattern doesn't match:
+		self.assertTrue(_test_exec('-d', '{^LN-BEG}EPOCH', '-o', 'Found-ID:<F-ID>', STR_00_NODT, RE_00_ID))
+		self.assertLogged(
+			"Found a match but no valid date/time found",
+			"Match without a timestamp:",
+			"Found-ID:kevin", all=True)
+		self.pruneLog()
+		# explicitly no datepattern:
+		self.assertTrue(_test_exec('-d', '{NONE}', '-o', 'Found-ID:<F-ID>', STR_00_NODT, RE_00_ID))
+		self.assertLogged(
+			"Found-ID:kevin", all=True)
+		self.assertNotLogged(
+			"Found a match but no valid date/time found",
+			"Match without a timestamp:", all=True)
+
+	def testIncompleteDateTime(self):
+		# datepattern in followed lines doesn't match previously known pattern + line is too short
+		# (logging break-off, no flush, etc):
+		self.assertTrue(_test_exec(
+			'-o', 'Found-ADDR:<ip>',
+			'192.0.2.1 - - [02/May/2021:18:40:55 +0100] "GET / HTTP/1.1" 302 328 "-" "Mozilla/5.0" "-"\n'
+			'192.0.2.2 - - [02/May/2021:18:40:55 +0100\n'
+			'192.0.2.3 - - [02/May/2021:18:40:55',
+			'^<ADDR>'))
+		self.assertLogged(
+			"Found-ADDR:192.0.2.1", "Found-ADDR:192.0.2.2", "Found-ADDR:192.0.2.3", all=True)	
+
+	def testFrmtOutputWrapML(self):
+		unittest.F2B.SkipIfCfgMissing(stock=True)
+		# complex substitution using tags and message (ip, user, msg):
+		self.assertTrue(_test_exec('-o', '<ip>, <F-USER>, <msg>',
+			'-c', CONFIG_DIR, '--usedns', 'no',
+			STR_ML_SSHD + "\n" + STR_ML_SSHD_OK, 'sshd[logtype=short, publickey=invalid]'))
+		# be sure we don't have IP in one line and have it in another:
+		lines = STR_ML_SSHD.split("\n")
+		self.assertTrue('192.0.2.2' not in lines[-2] and '192.0.2.2' in lines[-1])
+		# but both are in output "merged" with IP and user:
+		self.assertLogged(
+			'192.0.2.2, git, '+lines[-2],
+			'192.0.2.2, git, '+lines[-1],
+			all=True)
+		# nothing should be found for 192.0.2.1 (mode is not aggressive):
+		self.assertNotLogged('192.0.2.1, git, ')
+
+		# test with publickey (nofail) - would not produce output for 192.0.2.1 because accepted:
+		self.pruneLog("[test-phase 1] mode=aggressive & publickey=nofail + OK (accepted)")
+		self.assertTrue(_test_exec('-o', '<ip>, <F-USER>, <msg>',
+			'-c', CONFIG_DIR, '--usedns', 'no',
+			STR_ML_SSHD + "\n" + STR_ML_SSHD_OK, 'sshd[logtype=short, mode=aggressive]'))
+		self.assertLogged(
+			'192.0.2.2, git, '+lines[-4],
+			'192.0.2.2, git, '+lines[-3],
+			'192.0.2.2, git, '+lines[-2],
+			'192.0.2.2, git, '+lines[-1],
+			all=True)
+		# nothing should be found for 192.0.2.1 (access gained so failures ignored):
+		self.assertNotLogged('192.0.2.1, git, ')
+
+		# now same test but "accepted" replaced with "closed" on preauth phase:
+		self.pruneLog("[test-phase 2] mode=aggressive & publickey=nofail + FAIL (closed on preauth)")
+		self.assertTrue(_test_exec('-o', '<ip>, <F-USER>, <msg>',
+			'-c', CONFIG_DIR, '--usedns', 'no',
+			STR_ML_SSHD + "\n" + STR_ML_SSHD_FAIL, 'sshd[logtype=short, mode=aggressive]'))
+		# 192.0.2.1 should be found for every failure (2x failed key + 1x closed):
+		lines = STR_ML_SSHD.split("\n")[0:2] + STR_ML_SSHD_FAIL.split("\n")[-1:]
+		self.assertLogged(
+			'192.0.2.1, git, '+lines[-3],
+			'192.0.2.1, git, '+lines[-2],
+			'192.0.2.1, git, '+lines[-1],
+			all=True)
+

 	def testWrongFilterFile(self):
 		# use test log as filter file to cover eror cases...
@@ -420,7 +520,7 @@ class Fail2banRegexTest(LogCaptureTestCase):

 	def testLogtypeSystemdJournal(self): # pragma: no cover
 		if not fail2banregex.FilterSystemd:
-			raise unittest.SkipTest('Skip test because no systemd backand available')
+			raise unittest.SkipTest('Skip test because no systemd backend available')
 		self.assertTrue(_test_exec(
 			"systemd-journal", FILTER_ZZZ_GEN
 			  +'[journalmatch="SYSLOG_IDENTIFIER=\x01\x02dummy\x02\x01",'
@@ -0,0 +1,12 @@
+[INCLUDES]
+
+# Read common prefixes. If any customizations available -- read them from
+# common.local
+before = testcase-common.conf
+
+[Definition]
+
+_daemon = sshd
+__prefix_line = %(known/__prefix_line)s(?:\w{14,20}: )?
+
+failregex = %(__prefix_line)s test
@@ -0,0 +1,4 @@
+[Definition]
+
+# no options here, coverage for testFilterReaderSubstKnown:
+# avoid to overwrite known/option with unmodified (not available) value of option from .local config file
@@ -6,3 +6,6 @@

 # failJSON: { "time": "2018-09-28T09:18:06", "match": true , "host": "192.0.2.1", "desc": "two client entries in message (gh-2247)" }
 [Sat Sep 28 09:18:06 2018] [error] [client 192.0.2.1:55555] [client 192.0.2.1] ModSecurity: [file "/etc/httpd/modsecurity.d/10_asl_rules.conf"] [line "635"] [id "340069"] [rev "4"] [msg "Atomicorp.com UNSUPPORTED DELAYED Rules: Web vulnerability scanner"] [severity "CRITICAL"] Access denied with code 403 (phase 2). Pattern match "(?:nessus(?:_is_probing_you_|test)|^/w00tw00t\\\\.at\\\\.)" at REQUEST_URI. [hostname "192.81.249.191"] [uri "/w00tw00t.at.blackhats.romanian.anti-sec:)"] [unique_id "4Q6RdsBR@b4AAA65LRUAAAAA"] 
+
+# failJSON: { "time": "2020-05-09T00:35:52", "match": true , "host": "192.0.2.2", "desc": "new format - apache 2.4 and php-fpm (gh-2717)" }
+[Sat May 09 00:35:52.389262 2020] [:error] [pid 22406:tid 139985298601728] [client 192.0.2.2:47762] [client 192.0.2.2] ModSecurity: Access denied with code 401 (phase 2). Operator EQ matched 1 at IP:blocked. [file "/etc/httpd/modsecurity.d/activated_rules/modsecurity_wp_login.conf"] [line "14"] [id "500000"] [msg "Ip address blocked for 15 minutes, more than 5 login attempts in 3 minutes."] [hostname "example.com"] [uri "/wp-login.php"] [unique_id "XrYlGL5IY3I@EoLOgAAAA8"], referer: https://example.com/wp-login.php
--- a/Show More
+++ b/Show More