suppress unneeded info (moved to debug level)

see #3186

.github/workflows/main.yml

@@ -0,0 +1,98 @@
+name: CI
+
+# Controls when the action will run. Triggers the workflow on push or pull request
+# events but only for the master branch
+on:
+  push:
+    paths-ignore:
+    - 'doc/**'
+    - 'files/**'
+    - 'man/**'
+  pull_request:
+    paths-ignore:
+    - 'doc/**'
+    - 'files/**'
+    - 'man/**'
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+jobs:
+  # This workflow contains a single job called "build"
+  build:
+    # The type of runner that the job will run on
+    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, '3.10', '3.11.0-alpha.1', pypy2, pypy3]
+      fail-fast: false
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - uses: actions/checkout@v2
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Grant systemd-journal access
+        run: sudo usermod -a -G systemd-journal "$USER" || echo 'no systemd-journal access'
+
+      - name: Python version
+        run: |
+          F2B_PY=$(python -c "import sys; print(sys.version)")
+          echo "Python: ${{ matrix.python-version }} -- ${F2B_PY/$'\n'/ }"
+          F2B_PYV=$(echo "${F2B_PY}" | grep -oP '^\d+(?:\.\d+)')
+          F2B_PY=${F2B_PY:0:1}
+          echo "Set F2B_PY=$F2B_PY, F2B_PYV=$F2B_PYV"
+          echo "F2B_PY=$F2B_PY" >> $GITHUB_ENV
+          echo "F2B_PYV=$F2B_PYV" >> $GITHUB_ENV
+          # for GHA we need to monitor all journals, since it cannot be found using SYSTEM_ONLY(4):
+          echo "F2B_SYSTEMD_DEFAULT_FLAGS=0" >> $GITHUB_ENV
+
+      - name: Install dependencies
+        run: |
+          if [[ "$F2B_PY" = 3 ]]; then python -m pip install --upgrade pip || echo "can't upgrade pip"; fi
+          if [[ "$F2B_PY" = 3 ]] && ! command -v 2to3x -v 2to3 > /dev/null; then
+            #pip install 2to3
+            sudo apt-get -y install 2to3
+          fi
+          #sudo apt-get -y install python${F2B_PY/2/}-pyinotify || echo 'inotify not available'
+          python -m pip install pyinotify || echo 'inotify not available'
+          #sudo apt-get -y install python${F2B_PY/2/}-systemd || echo 'systemd not available'
+          sudo apt-get -y install libsystemd-dev || echo 'systemd dependencies seems to be unavailable'
+          python -m pip install systemd-python || echo 'systemd not available'
+
+      - name: Before scripts
+        run: |
+          cd "$GITHUB_WORKSPACE"
+          # Manually execute 2to3 for now
+          if [[ "$F2B_PY" = 3 ]]; then echo "2to3 ..." && ./fail2ban-2to3; fi
+          _debug() { echo -n "$1 "; err=$("${@:2}" 2>&1) && echo 'OK' || echo -e "FAIL\n$err"; }
+          # (debug) output current preferred encoding:
+          _debug 'Encodings:' python -c 'import locale, sys; from fail2ban.helpers import PREFER_ENC; print(PREFER_ENC, locale.getpreferredencoding(), (sys.stdout and sys.stdout.encoding))'
+          # (debug) backend availabilities:
+          echo 'Backends:'
+          _debug '- systemd:' python -c 'from fail2ban.server.filtersystemd import FilterSystemd'
+          #_debug '- systemd (root): ' sudo python -c 'from fail2ban.server.filtersystemd import FilterSystemd'
+          _debug '- pyinotify:' python -c 'from fail2ban.server.filterpyinotify import FilterPyinotify'
+
+      - name: Test suite
+        run: |
+          if [[ "$F2B_PY" = 2 ]]; then
+            python setup.py test
+          elif dpkg --compare-versions "$F2B_PYV" lt 3.10; then 
+            python bin/fail2ban-testcases --verbosity=2
+          else
+            echo "Skip systemd backend since systemd-python module must be fixed for python >= v.3.10 in GHA ..."
+            python bin/fail2ban-testcases --verbosity=2 -i "[sS]ystemd|[jJ]ournal"
+          fi
+
+      #- name: Test suite (debug some systemd tests only)
+         #run: python bin/fail2ban-testcases --verbosity=2 "[sS]ystemd|[jJ]ournal"
+         #run: python bin/fail2ban-testcases --verbosity=2 -l 5 "test_WrongChar"
+
+      - name: Build
+        run: python setup.py build
+
+      #- name: Test initd scripts
+      #  run: shellcheck -s bash -e SC1090,SC1091 files/debian-initd

.travis.yml

@@ -10,22 +10,14 @@ dist: xenial
 matrix:
   fast_finish: true
   include:
-  - python: 2.6
-    dist: trusty   # required for Python 2.6
   - python: 2.7
-    dist: trusty   # required for packages like gamin
-    name: 2.7 (trusty)
-  - python: 2.7
-    name: 2.7 (xenial)
-  - python: pypy
-    dist: trusty
-  - python: 3.3
-    dist: trusty
+  #- python: pypy
   - python:  3.4
   - python:  3.5
   - python:  3.6
   - python:  3.7
-  - python:  3.8-dev
+  - python:  3.8
+  - python:  3.9-dev
   - python:  pypy3.5
 before_install:
   - echo "running under $TRAVIS_PYTHON_VERSION"
@@ -69,8 +61,8 @@ script:
   - if [[ "$F2B_PY" = 3 ]]; then coverage run bin/fail2ban-testcases --verbosity=2; fi
   # Use $VENV_BIN (not python) or else sudo will always run the system's python (2.7)
   - sudo $VENV_BIN/pip install .
-  # Doc files should get installed on Travis under Linux (python >= 3.8 seem to use another path segment)
-  - if [[ $TRAVIS_PYTHON_VERSION < 3.8 ]]; then test -e /usr/share/doc/fail2ban/FILTERS; fi
+  # Doc files should get installed on Travis under Linux (some builds/python's seem to use another path segment)
+  - test -e /usr/share/doc/fail2ban/FILTERS && echo 'found' || echo 'not found'
   # Test initd script
   - shellcheck -s bash -e SC1090,SC1091 files/debian-initd
 after_success:

ChangeLog

@@ -6,60 +6,91 @@
 Fail2Ban: Changelog
 ===================
 
-ver. 0.11.1 (2020/01/11) - this-is-the-way
+ver. 0.10.6 (2020/11/23) - just-what-the-doctor-ordered
 -----------
 
-### Compatibility:
-* to v.0.10:
-  - 0.11 is totally compatible to 0.10 (configuration- and API-related stuff), but the database
-    got some new tables and fields (auto-converted during the first start), so once updated to 0.11, you
-    have to remove the database /var/lib/fail2ban/fail2ban.sqlite3 (or its different to 0.10 schema)
-    if you would need to downgrade to 0.10 for some reason.
-* to v.0.9:
-  - Filter (or `failregex`) internal capture-groups:
+### Incompatibility list (v.0.10 compared to v.0.9):
+* Filter (or `failregex`) internal capture-groups:
 
-    * If you've your own `failregex` or custom filters using conditional match `(?P=host)`, you should
-      rewrite the regex like in example below resp. using `(?:(?P=ip4)|(?P=ip6)` instead of `(?P=host)`
-      (or `(?:(?P=ip4)|(?P=ip6)|(?P=dns))` corresponding your `usedns` and `raw` settings).
+  - If you've your own `failregex` or custom filters using conditional match `(?P=host)`, you should
+    rewrite the regex like in example below resp. using `(?:(?P=ip4)|(?P=ip6)` instead of `(?P=host)`
+    (or `(?:(?P=ip4)|(?P=ip6)|(?P=dns))` corresponding your `usedns` and `raw` settings).
 
-      Of course you can always define your own capture-group (like below `_cond_ip_`) to do this.
-      ```
-      testln="1500000000 failure from 192.0.2.1: bad host 192.0.2.1"
-      fail2ban-regex "$testln" "^\s*failure from (?P<_cond_ip_><HOST>): bad host (?P=_cond_ip_)$"
-      ```
-    * New internal groups (currently reserved for internal usage):
-      `ip4`, `ip6`, `dns`, `fid`, `fport`, additionally `user` and another captures in lower case if
-      mapping from tag `<F-*>` used in failregex (e. g. `user` by `<F-USER>`).
+    Of course you can always define your own capture-group (like below `_cond_ip_`) to do this.
+    ```
+    testln="1500000000 failure from 192.0.2.1: bad host 192.0.2.1"
+    fail2ban-regex "$testln" "^\s*failure from (?P<_cond_ip_><HOST>): bad host (?P=_cond_ip_)$"
+    ```
+  - New internal groups (currently reserved for internal usage):
+    `ip4`, `ip6`, `dns`, `fid`, `fport`, additionally `user` and another captures in lower case if
+    mapping from tag `<F-*>` used in failregex (e. g. `user` by `<F-USER>`).
 
-  - v.0.10 and 0.11 use more precise date template handling, that can be theoretically incompatible to some
-    user configurations resp. `datepattern`.
+* v.0.10 uses more precise date template handling, that can be theoretically incompatible to some
+  user configurations resp. `datepattern`.
 
-  - Since v0.10 fail2ban supports the matching of IPv6 addresses, but not all ban actions are
-    IPv6-capable now.
+* Since v0.10 fail2ban supports the matching of IPv6 addresses, but not all ban actions are
+  IPv6-capable now.
 
 ### Fixes
-* purge database will be executed now (within observer).
-* restoring currently banned ip after service restart fixed 
-  (now < timeofban + bantime), ignore old log failures (already banned)
-* upgrade database: update new created table `bips` with entries from table `bans` (allows restore
-  current bans after upgrade from version <= 0.10)
+* [stability] prevent race condition - no ban if filter (backend) is continuously busy if
+  too many messages will be found in log, e. g. initial scan of large log-file or journal (gh-2660)
+* pyinotify-backend sporadically avoided initial scanning of log-file by start
+* python 3.9 compatibility (and Travis CI support)
+* restoring a large number (500+ depending on files ulimit) of current bans when using PyPy fixed
+* manual ban is written to database, so can be restored by restart (gh-2647)
+* `jail.conf`: don't specify `action` directly in jails (use `action_` or `banaction` instead)
+* no mails-action added per default anymore (e. g. to allow that `action = %(action_mw)s` should be specified
+  per jail or in default section in jail.local), closes gh-2357
+* ensure we've unique action name per jail (also if parameter `actname` is not set but name deviates from standard name, gh-2686)
+* don't use `%(banaction)s` interpolation because it can be complex value (containing `[...]` and/or quotes), 
+  so would bother the action interpolation
+* fixed type conversion in config readers (take place after all interpolations get ready), that allows to 
+  specify typed parameters variable (as substitutions) as well as to supply it in other sections or as init parameters.
+* `action.d/*-ipset*.conf`: several ipset actions fixed (no timeout per default anymore), so no discrepancy
+  between ipset and fail2ban (removal from ipset will be managed by fail2ban only, gh-2703)
+* `action.d/cloudflare.conf`: fixed `actionunban` (considering new-line chars and optionally real json-parsing
+   with `jq`, gh-2140, gh-2656)
+* `action.d/nftables.conf` (type=multiport only): fixed port range selector, replacing `:` with `-` (gh-2763)
+* `action.d/firewallcmd-*.conf` (multiport only): fixed port range selector, replacing `:` with `-` (gh-2821)
+* `action.d/bsd-ipfw.conf`: fixed selection of rule-no by large list or initial `lowest_rule_num` (gh-2836)
+* `filter.d/common.conf`: avoid substitute of default values in related `lt_*` section, `__prefix_line`
+  should be interpolated in definition section (inside the filter-config, gh-2650)
+* `filter.d/dovecot.conf`: 
+  - add managesieve and submission support (gh-2795);
+  - accept messages with more verbose logging (gh-2573);
+* `filter.d/courier-smtp.conf`: prefregex extended to consider port in log-message (gh-2697)
+* `filter.d/traefik-auth.conf`: filter extended with parameter mode (`normal`, `ddos`, `aggressive`) to handle 
+  the match of username differently (gh-2693):
+  - `normal`: matches 401 with supplied username only
+  - `ddos`: matches 401 without supplied username only
+  - `aggressive`: matches 401 and any variant (with and without username)
+* `filter.d/sshd.conf`: normalizing of user pattern in all RE's, allowing empty user (gh-2749)
 
-### New Features
-* Increment ban time (+ observer) functionality introduced.
-* Database functionality extended with bad ips.
-* New tags (usable in actions):
-  - `<bancount>` - ban count of this offender if known as bad (started by 1 for unknown)
-  - `<bantime>` - current ban-time of the ticket (prolongation can be retarded up to 10 sec.)
-* Introduced new action command `actionprolong` to prolong ban-time (e. g. set new timeout if expected);
-  Several actions (like ipset, etc.) rewritten using net logic with `actionprolong`.
-  Note: because ban-time is dynamic, it was removed from jail.conf as timeout argument (check jail.local).
-
-### Enhancements
-* algorithm of restore current bans after restart changed: update the restored ban-time (and therefore 
-  end of ban) of the ticket with ban-time of jail (as maximum), for all tickets with ban-time greater
-  (or persistent); not affected if ban-time of the jail is unchanged between stop/start.
-* added new setup-option `--without-tests` to skip building and installing of tests files (gh-2287).
-* added new command `fail2ban-client get <JAIL> banip ?sep-char|--with-time?` to get the banned ip addresses (gh-1916).
+### New Features and Enhancements
+* fail2ban-regex:
+  - speedup formatted output (bypass unneeded stats creation)
+  - extended with prefregex statistic
+  - more informative output for `datepattern` (e. g. set from filter) - pattern : description
+* parsing of action in jail-configs considers space between action-names as separator also
+  (previously only new-line was allowed), for example `action = a b` would specify 2 actions `a` and `b`
+* new filter and jail for GitLab recognizing failed application logins (gh-2689)
+* new filter and jail for Grafana recognizing failed application logins (gh-2855)
+* new filter and jail for SoftEtherVPN recognizing failed application logins (gh-2723)
+* `filter.d/guacamole.conf` extended with `logging` parameter to follow webapp-logging if it's configured (gh-2631)
+* `filter.d/bitwarden.conf` enhanced to support syslog (gh-2778)
+* introduced new prefix `{UNB}` for `datepattern` to disable word boundaries in regex;
+* datetemplate: improved anchor detection for capturing groups `(^...)`;
+* datepattern: improved handling with wrong recognized timestamps (timezones, no datepattern, etc)
+  as well as some warnings signaling user about invalid pattern or zone (gh-2814):
+  - filter gets mode in-operation, which gets activated if filter starts processing of new messages;
+    in this mode a timestamp read from log-line that appeared recently (not an old line), deviating too much
+    from now (up too 24h), will be considered as now (assuming a timezone issue), so could avoid unexpected 
+    bypass of failure (previously exceeding `findtime`);
+  - better interaction with non-matching optional datepattern or invalid timestamps;
+  - implements special datepattern `{NONE}` - allow to find failures totally without date-time in log messages,
+    whereas filter will use now as timestamp (gh-2802)
+* performance optimization of `datepattern` (better search algorithm in datedetector, especially for single template);
+* fail2ban-client: extended to unban IP range(s) by subnet (CIDR/mask) or hostname (DNS), gh-2791;
 
 
 ver. 0.10.5 (2020/01/10) - deserve-more-respect-a-jedis-weapon-must
@@ -440,14 +471,9 @@ TODO: implementing of options resp. other tasks from PR #1346
   - `<fid>` - failure identifier (if raw resp. failures without IP address)
   - `<ip-rev>` - PTR reversed representation of IP address
   - `<ip-host>` - host name of the IP address
-  - `<bancount>` - ban count of this offender if known as bad (started by 1 for unknown)
-  - `<bantime>` - current ban-time of the ticket (prolongation can be retarded up to 10 sec.)
   - `<F-...>` - interpolates to the corresponding filter group capture `...`
   - `<fq-hostname>` - fully-qualified name of host (the same as `$(hostname -f)`)
   - `<sh-hostname>` - short hostname (the same as `$(uname -n)`)
-* Introduced new action command `actionprolong` to prolong ban-time (e. g. set new timeout if expected);
-  Several actions (like ipset, etc.) rewritten using net logic with `actionprolong`.
-  Note: because ban-time is dynamic, it was removed from jail.conf as timeout argument (check jail.local).
 * Allow to use filter options by `fail2ban-regex`, example:
   fail2ban-regex text.log "sshd[mode=aggressive]"
 * Samples test case factory extended with filter options - dict in JSON to control 
@@ -521,9 +547,6 @@ ver. 0.10.0-alpha-1 (2016/07/14) - ipv6-support-etc
 * testSocket: sporadical bug repaired - wait for server thread starts a socket (listener)
 * testExecuteTimeoutWithNastyChildren: sporadical bug repaired - wait for pid file inside bash,
   kill tree in any case (gh-1155)
-* purge database will be executed now (within observer).
-* restoring currently banned ip after service restart fixed 
-  (now < timeofban + bantime), ignore old log failures (already banned)
 * Fixed high-load of pyinotify-backend,
   see https://github.com/fail2ban/fail2ban/issues/885#issuecomment-248964591
 * Database: stability fix - repack cursor iterator as long as locked
@@ -561,9 +584,6 @@ ver. 0.10.0-alpha-1 (2016/07/14) - ipv6-support-etc
     - new conditional section functionality used in config resp. includes:
       - [Init?family=inet4] - IPv4 qualified hosts only
       - [Init?family=inet6] - IPv6 qualified hosts only
-* Increment ban time (+ observer) functionality introduced.
-  Thanks Serg G. Brester (sebres)
-* Database functionality extended with bad ips.
 * New reload functionality (now totally without restart, unbanning/rebanning, etc.),
   see gh-1557
 * Several commands extended and new commands introduced:

MANIFEST

@@ -5,8 +5,6 @@ bin/fail2ban-testcases
 ChangeLog
 config/action.d/abuseipdb.conf
 config/action.d/apf.conf
-config/action.d/badips.conf
-config/action.d/badips.py
 config/action.d/blocklist_de.conf
 config/action.d/bsd-ipfw.conf
 config/action.d/cloudflare.conf
@@ -100,6 +98,8 @@ config/filter.d/exim.conf
 config/filter.d/exim-spam.conf
 config/filter.d/freeswitch.conf
 config/filter.d/froxlor-auth.conf
+config/filter.d/gitlab.conf
+config/filter.d/grafana.conf
 config/filter.d/groupoffice.conf
 config/filter.d/gssftpd.conf
 config/filter.d/guacamole.conf
@@ -139,6 +139,7 @@ config/filter.d/sendmail-auth.conf
 config/filter.d/sendmail-reject.conf
 config/filter.d/sieve.conf
 config/filter.d/slapd.conf
+config/filter.d/softethervpn.conf
 config/filter.d/sogo-auth.conf
 config/filter.d/solid-pop3d.conf
 config/filter.d/squid.conf
@@ -207,7 +208,6 @@ fail2ban/server/jail.py
 fail2ban/server/jails.py
 fail2ban/server/jailthread.py
 fail2ban/server/mytime.py
-fail2ban/server/observer.py
 fail2ban/server/server.py
 fail2ban/server/strptime.py
 fail2ban/server/ticket.py
@@ -217,7 +217,6 @@ fail2ban/setup.py
 fail2ban-testcases-all
 fail2ban-testcases-all-python3
 fail2ban/tests/action_d/__init__.py
-fail2ban/tests/action_d/test_badips.py
 fail2ban/tests/action_d/test_smtp.py
 fail2ban/tests/actionstestcase.py
 fail2ban/tests/actiontestcase.py
@@ -227,6 +226,8 @@ fail2ban/tests/clientreadertestcase.py
 fail2ban/tests/config/action.d/action.conf
 fail2ban/tests/config/action.d/brokenaction.conf
 fail2ban/tests/config/fail2ban.conf
+fail2ban/tests/config/filter.d/checklogtype.conf
+fail2ban/tests/config/filter.d/checklogtype_test.conf
 fail2ban/tests/config/filter.d/simple.conf
 fail2ban/tests/config/filter.d/test.conf
 fail2ban/tests/config/filter.d/test.local
@@ -262,9 +263,10 @@ fail2ban/tests/files/config/apache-auth/digest_wrongrelm/.htpasswd
 fail2ban/tests/files/config/apache-auth/noentry/.htaccess
 fail2ban/tests/files/config/apache-auth/README
 fail2ban/tests/files/database_v1.db
-fail2ban/tests/files/database_v2.db
 fail2ban/tests/files/filter.d/substition.conf
 fail2ban/tests/files/filter.d/testcase01.conf
+fail2ban/tests/files/filter.d/testcase02.conf
+fail2ban/tests/files/filter.d/testcase02.local
 fail2ban/tests/files/filter.d/testcase-common.conf
 fail2ban/tests/files/ignorecommand.py
 fail2ban/tests/files/logs/3proxy
@@ -299,6 +301,8 @@ fail2ban/tests/files/logs/exim
 fail2ban/tests/files/logs/exim-spam
 fail2ban/tests/files/logs/freeswitch
 fail2ban/tests/files/logs/froxlor-auth
+fail2ban/tests/files/logs/gitlab
+fail2ban/tests/files/logs/grafana
 fail2ban/tests/files/logs/groupoffice
 fail2ban/tests/files/logs/gssftpd
 fail2ban/tests/files/logs/guacamole
@@ -336,6 +340,7 @@ fail2ban/tests/files/logs/sendmail-auth
 fail2ban/tests/files/logs/sendmail-reject
 fail2ban/tests/files/logs/sieve
 fail2ban/tests/files/logs/slapd
+fail2ban/tests/files/logs/softethervpn
 fail2ban/tests/files/logs/sogo-auth
 fail2ban/tests/files/logs/solid-pop3d
 fail2ban/tests/files/logs/squid
@@ -368,7 +373,6 @@ fail2ban/tests/files/zzz-sshd-obsolete-multiline.log
 fail2ban/tests/filtertestcase.py
 fail2ban/tests/__init__.py
 fail2ban/tests/misctestcase.py
-fail2ban/tests/observertestcase.py
 fail2ban/tests/samplestestcase.py
 fail2ban/tests/servertestcase.py
 fail2ban/tests/sockettestcase.py

README.md

@@ -2,7 +2,7 @@
                         / _|__ _(_) |_  ) |__  __ _ _ _  
                        |  _/ _` | | |/ /| '_ \/ _` | ' \ 
                        |_| \__,_|_|_/___|_.__/\__,_|_||_|
-                       v0.11.0.dev1            20??/??/??
+                       v0.10.3.dev1            20??/??/??
 
 ## Fail2Ban: ban hosts that cause multiple authentication errors
 
@@ -18,12 +18,11 @@ attempts, it cannot eliminate the risk presented by weak authentication.
 Set up services to use only two factor, or public/private authentication
 mechanisms if you really want to protect services.
      
-<img src="http://www.worldipv6launch.org/wp-content/themes/ipv6/downloads/World_IPv6_launch_logo.svg" height="52pt"/> | Since v0.10 fail2ban supports the matching of IPv6 addresses.
+<img src="http://www.worldipv6launch.org/wp-content/themes/ipv6/downloads/World_IPv6_launch_logo.svg" height="52pt"/> | Since v0.10 fail2ban supports the matching of the IPv6 addresses.
 ------|------
 
 This README is a quick introduction to Fail2Ban. More documentation, FAQ, and HOWTOs
-to be found on fail2ban(1) manpage, [Wiki](https://github.com/fail2ban/fail2ban/wiki),
-[Developers documentation](https://fail2ban.readthedocs.io/)
+to be found on fail2ban(1) manpage, [Wiki](https://github.com/fail2ban/fail2ban/wiki)
 and the website: https://www.fail2ban.org
 
 Installation:
@@ -46,16 +45,10 @@ Optional:
 
 To install:
 
-    tar xvfj fail2ban-0.11.0.tar.bz2
-    cd fail2ban-0.11.0
-    sudo python setup.py install
-   
-Alternatively, you can clone the source from GitHub to a directory of Your choice, and do the install from there. Pick the correct branch, for example, 0.11
+    tar xvfj fail2ban-0.10.3.tar.bz2
+    cd fail2ban-0.10.3
+    python setup.py install
 
-    git clone https://github.com/fail2ban/fail2ban.git
-    cd fail2ban
-    sudo python setup.py install 
-    
 This will install Fail2Ban into the python library directory. The executable
 scripts are placed into `/usr/bin`, and configuration in `/etc/fail2ban`.
 
@@ -65,9 +58,6 @@ Fail2Ban should be correctly installed now. Just type:
 
 to see if everything is alright. You should always use fail2ban-client and
 never call fail2ban-server directly.
-You can verify that you have the correct version installed with 
-
-    fail2ban-client version
 
 Please note that the system init/service script is not automatically installed.
 To enable fail2ban as an automatic service, simply copy the script for your
@@ -89,11 +79,11 @@ fail2ban(1) and jail.conf(5)  manpages for further references.
 Code status:
 ------------
 
-* travis-ci.org: [![tests status](https://secure.travis-ci.org/fail2ban/fail2ban.svg?branch=0.11)](https://travis-ci.org/fail2ban/fail2ban?branch=0.11) (0.11 branch) / [![tests status](https://secure.travis-ci.org/fail2ban/fail2ban.svg?branch=0.10)](https://travis-ci.org/fail2ban/fail2ban?branch=0.10) (0.10 branch) 
+* [![tests status](https://secure.travis-ci.org/fail2ban/fail2ban.png?branch=0.10)](https://travis-ci.org/fail2ban/fail2ban?branch=0.10) travis-ci.org (0.10 branch) / [![tests status](https://secure.travis-ci.org/fail2ban/fail2ban.png?branch=master)](https://travis-ci.org/fail2ban/fail2ban) travis-ci.org (master branch)
 
-* coveralls.io: [![Coverage Status](https://coveralls.io/repos/fail2ban/fail2ban/badge.svg?branch=0.11)](https://coveralls.io/github/fail2ban/fail2ban?branch=0.11) (0.11 branch) / [![Coverage Status](https://coveralls.io/repos/fail2ban/fail2ban/badge.svg?branch=0.10)](https://coveralls.io/github/fail2ban/fail2ban?branch=0.10) / (0.10 branch)
+* [![Coverage Status](https://coveralls.io/repos/fail2ban/fail2ban/badge.png?branch=0.10)](https://coveralls.io/github/fail2ban/fail2ban?branch=0.10)
 
-* codecov.io: [![codecov.io](https://codecov.io/gh/fail2ban/fail2ban/coverage.svg?branch=0.11)](https://codecov.io/gh/fail2ban/fail2ban/branch/0.11) (0.11 branch) / [![codecov.io](https://codecov.io/gh/fail2ban/fail2ban/coverage.svg?branch=0.10)](https://codecov.io/gh/fail2ban/fail2ban/branch/0.10) (0.10 branch)
+* [![codecov.io](https://codecov.io/gh/fail2ban/fail2ban/coverage.svg?branch=0.10)](https://codecov.io/gh/fail2ban/fail2ban/branch/0.10)
 
 Contact:
 --------

THANKS

@@ -111,7 +111,7 @@ Russell Odom
 SATO Kentaro
 Sean DuBois
 Sebastian Arcus
-Serg G. Brester (sebres)
+Serg G. Brester
 Sergey Safarov
 Shaun C.
 Sireyessire

config/action.d/abuseipdb.conf

@@ -21,14 +21,13 @@
 #
 # Example, for ssh bruteforce (in section [sshd] of `jail.local`): 
 #   action = %(known/action)s
-#            %(action_abuseipdb)s[abuseipdb_apikey="my-api-key", abuseipdb_category="18,22"]
+#            abuseipdb[abuseipdb_apikey="my-api-key", abuseipdb_category="18,22"]
 #
-# See below for catagories.
+# See below for categories.
 #
-# Original Ref: https://wiki.shaunc.com/wikka.php?wakka=ReportingToAbuseIPDBWithFail2Ban
 # Added to fail2ban by Andrew James Collett (ajcollett)
 
-## abuseIPDB Catagories, `the abuseipdb_category` MUST be set in the jail.conf action call.
+## abuseIPDB Categories, `the abuseipdb_category` MUST be set in the jail.conf action call.
 # Example, for ssh bruteforce: action = %(action_abuseipdb)s[abuseipdb_category="18,22"]
 # ID	Title	Description
 # 3	  Fraud Orders

config/action.d/badips.conf

@@ -1,19 +0,0 @@
-# Fail2ban reporting to badips.com
-#
-# Note: This reports an IP only and does not actually ban traffic. Use
-# another action in the same jail if you want bans to occur.
-#
-# Set the category to the appropriate value before use.
-#
-# To get see register and optional key to get personalised graphs see:
-# http://www.badips.com/blog/personalized-statistics-track-the-attackers-of-all-your-servers-with-one-key
-
-[Definition]
-
-actionban = curl --fail  --user-agent "<agent>" http://www.badips.com/add/<category>/<ip>
-
-[Init]
-
-# Option: category
-# Notes.: Values are from the list here: http://www.badips.com/get/categories
-category = 

config/action.d/badips.py

@@ -1,391 +0,0 @@
-# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*-
-# vi: set ft=python sts=4 ts=4 sw=4 noet :
-
-# This file is part of Fail2Ban.
-#
-# Fail2Ban is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# Fail2Ban is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Fail2Ban; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
-
-import sys
-if sys.version_info < (2, 7): # pragma: no cover
-	raise ImportError("badips.py action requires Python >= 2.7")
-import json
-import threading
-import logging
-if sys.version_info >= (3, ): # pragma: 2.x no cover
-	from urllib.request import Request, urlopen
-	from urllib.parse import urlencode
-	from urllib.error import HTTPError
-else: # pragma: 3.x no cover
-	from urllib2 import Request, urlopen, HTTPError
-	from urllib import urlencode
-
-from fail2ban.server.actions import Actions, ActionBase, BanTicket
-from fail2ban.helpers import splitwords, str2LogLevel
-
-
-
-class BadIPsAction(ActionBase): # pragma: no cover - may be unavailable
-	"""Fail2Ban action which reports bans to badips.com, and also
-	blacklist bad IPs listed on badips.com by using another action's
-	ban method.
-
-	Parameters
-	----------
-	jail : Jail
-		The jail which the action belongs to.
-	name : str
-		Name assigned to the action.
-	category : str
-		Valid badips.com category for reporting failures.
-	score : int, optional
-		Minimum score for bad IPs. Default 3.
-	age : str, optional
-		Age of last report for bad IPs, per badips.com syntax.
-		Default "24h" (24 hours)
-	banaction : str, optional
-		Name of banaction to use for blacklisting bad IPs. If `None`,
-		no blacklist of IPs will take place.
-		Default `None`.
-	bancategory : str, optional
-		Name of category to use for blacklisting, which can differ
-		from category used for reporting. e.g. may want to report
-		"postfix", but want to use whole "mail" category for blacklist.
-		Default `category`.
-	bankey : str, optional
-		Key issued by badips.com to retrieve personal list
-		of blacklist IPs.
-	updateperiod : int, optional
-		Time in seconds between updating bad IPs blacklist.
-		Default 900 (15 minutes)
-	loglevel : int/str, optional
-		Log level of the message when an IP is (un)banned.
-		Default `DEBUG`.
-		Can be also supplied as two-value list (comma- or space separated) to
-		provide level of the summary message when a group of IPs is (un)banned.
-		Example `DEBUG,INFO`.
-	agent : str, optional
-		User agent transmitted to server.
-		Default `Fail2Ban/ver.`
-
-	Raises
-	------
-	ValueError
-		If invalid `category`, `score`, `banaction` or `updateperiod`.
-	"""
-
-	TIMEOUT = 10
-	_badips = "https://www.badips.com"
-	def _Request(self, url, **argv):
-		return Request(url, headers={'User-Agent': self.agent}, **argv)
-
-	def __init__(self, jail, name, category, score=3, age="24h",
-		banaction=None, bancategory=None, bankey=None, updateperiod=900, 
-		loglevel='DEBUG', agent="Fail2Ban", timeout=TIMEOUT):
-		super(BadIPsAction, self).__init__(jail, name)
-
-		self.timeout = timeout
-		self.agent = agent
-		self.category = category
-		self.score = score
-		self.age = age
-		self.banaction = banaction
-		self.bancategory = bancategory or category
-		self.bankey = bankey
-		loglevel = splitwords(loglevel)
-		self.sumloglevel = str2LogLevel(loglevel[-1])
-		self.loglevel = str2LogLevel(loglevel[0])
-		self.updateperiod = updateperiod
-
-		self._bannedips = set()
-		# Used later for threading.Timer for updating badips
-		self._timer = None
-
-	@staticmethod
-	def isAvailable(timeout=1):
-		try:
-			response = urlopen(Request("/".join([BadIPsAction._badips]),
-					headers={'User-Agent': "Fail2Ban"}), timeout=timeout)
-			return True, ''
-		except Exception as e: # pragma: no cover
-			return False, e
-
-	def logError(self, response, what=''): # pragma: no cover - sporadical (502: Bad Gateway, etc)
-		messages = {}
-		try:
-			messages = json.loads(response.read().decode('utf-8'))
-		except:
-			pass
-		self._logSys.error(
-			"%s. badips.com response: '%s'", what,
-				messages.get('err', 'Unknown'))
-
-	def getCategories(self, incParents=False):
-		"""Get badips.com categories.
-
-		Returns
-		-------
-		set
-			Set of categories.
-
-		Raises
-		------
-		HTTPError
-			Any issues with badips.com request.
-		ValueError
-			If badips.com response didn't contain necessary information
-		"""
-		try:
-			response = urlopen(
-				self._Request("/".join([self._badips, "get", "categories"])), timeout=self.timeout)
-		except HTTPError as response: # pragma: no cover
-			self.logError(response, "Failed to fetch categories")
-			raise
-		else:
-			response_json = json.loads(response.read().decode('utf-8'))
-			if not 'categories' in response_json:
-				err = "badips.com response lacked categories specification. Response was: %s" \
-				  % (response_json,)
-				self._logSys.error(err)
-				raise ValueError(err)
-			categories = response_json['categories']
-			categories_names = set(
-				value['Name'] for value in categories)
-			if incParents:
-				categories_names.update(set(
-					value['Parent'] for value in categories
-					if "Parent" in value))
-			return categories_names
-
-	def getList(self, category, score, age, key=None):
-		"""Get badips.com list of bad IPs.
-
-		Parameters
-		----------
-		category : str
-			Valid badips.com category.
-		score : int
-			Minimum score for bad IPs.
-		age : str
-			Age of last report for bad IPs, per badips.com syntax.
-		key : str, optional
-			Key issued by badips.com to fetch IPs reported with the
-			associated key.
-
-		Returns
-		-------
-		set
-			Set of bad IPs.
-
-		Raises
-		------
-		HTTPError
-			Any issues with badips.com request.
-		"""
-		try:
-			url = "?".join([
-				"/".join([self._badips, "get", "list", category, str(score)]),
-				urlencode({'age': age})])
-			if key:
-				url = "&".join([url, urlencode({'key': key})])
-			self._logSys.debug('badips.com: get list, url: %r', url)
-			response = urlopen(self._Request(url), timeout=self.timeout)
-		except HTTPError as response: # pragma: no cover
-			self.logError(response, "Failed to fetch bad IP list")
-			raise
-		else:
-			return set(response.read().decode('utf-8').split())
-
-	@property
-	def category(self):
-		"""badips.com category for reporting IPs.
-		"""
-		return self._category
-
-	@category.setter
-	def category(self, category):
-		if category not in self.getCategories():
-			self._logSys.error("Category name '%s' not valid. "
-				"see badips.com for list of valid categories",
-				category)
-			raise ValueError("Invalid category: %s" % category)
-		self._category = category
-
-	@property
-	def bancategory(self):
-		"""badips.com bancategory for fetching IPs.
-		"""
-		return self._bancategory
-
-	@bancategory.setter
-	def bancategory(self, bancategory):
-		if bancategory != "any" and bancategory not in self.getCategories(incParents=True):
-			self._logSys.error("Category name '%s' not valid. "
-				"see badips.com for list of valid categories",
-				bancategory)
-			raise ValueError("Invalid bancategory: %s" % bancategory)
-		self._bancategory = bancategory
-
-	@property
-	def score(self):
-		"""badips.com minimum score for fetching IPs.
-		"""
-		return self._score
-
-	@score.setter
-	def score(self, score):
-		score = int(score)
-		if 0 <= score <= 5:
-			self._score = score
-		else:
-			raise ValueError("Score must be 0-5")
-
-	@property
-	def banaction(self):
-		"""Jail action to use for banning/unbanning.
-		"""
-		return self._banaction
-
-	@banaction.setter
-	def banaction(self, banaction):
-		if banaction is not None and banaction not in self._jail.actions:
-			self._logSys.error("Action name '%s' not in jail '%s'",
-				banaction, self._jail.name)
-			raise ValueError("Invalid banaction")
-		self._banaction = banaction
-
-	@property
-	def updateperiod(self):
-		"""Period in seconds between banned bad IPs will be updated.
-		"""
-		return self._updateperiod
-
-	@updateperiod.setter
-	def updateperiod(self, updateperiod):
-		updateperiod = int(updateperiod)
-		if updateperiod > 0:
-			self._updateperiod = updateperiod
-		else:
-			raise ValueError("Update period must be integer greater than 0")
-
-	def _banIPs(self, ips):
-		for ip in ips:
-			try:
-				ai = Actions.ActionInfo(BanTicket(ip), self._jail)
-				self._jail.actions[self.banaction].ban(ai)
-			except Exception as e:
-				self._logSys.error(
-					"Error banning IP %s for jail '%s' with action '%s': %s",
-					ip, self._jail.name, self.banaction, e,
-					exc_info=self._logSys.getEffectiveLevel()<=logging.DEBUG)
-			else:
-				self._bannedips.add(ip)
-				self._logSys.log(self.loglevel,
-					"Banned IP %s for jail '%s' with action '%s'",
-					ip, self._jail.name, self.banaction)
-
-	def _unbanIPs(self, ips):
-		for ip in ips:
-			try:
-				ai = Actions.ActionInfo(BanTicket(ip), self._jail)
-				self._jail.actions[self.banaction].unban(ai)
-			except Exception as e:
-				self._logSys.error(
-					"Error unbanning IP %s for jail '%s' with action '%s': %s",
-					ip, self._jail.name, self.banaction, e,
-					exc_info=self._logSys.getEffectiveLevel()<=logging.DEBUG)
-			else:
-				self._logSys.log(self.loglevel,
-					"Unbanned IP %s for jail '%s' with action '%s'",
-					ip, self._jail.name, self.banaction)
-			finally:
-				self._bannedips.remove(ip)
-
-	def start(self):
-		"""If `banaction` set, blacklists bad IPs.
-		"""
-		if self.banaction is not None:
-			self.update()
-
-	def update(self):
-		"""If `banaction` set, updates blacklisted IPs.
-
-		Queries badips.com for list of bad IPs, removing IPs from the
-		blacklist if no longer present, and adds new bad IPs to the
-		blacklist.
-		"""
-		if self.banaction is not None:
-			if self._timer:
-				self._timer.cancel()
-				self._timer = None
-
-			try:
-				ips = self.getList(
-					self.bancategory, self.score, self.age, self.bankey)
-				# Remove old IPs no longer listed
-				s = self._bannedips - ips
-				m = len(s)
-				self._unbanIPs(s)
-				# Add new IPs which are now listed
-				s = ips - self._bannedips
-				p = len(s)
-				self._banIPs(s)
-				if m != 0 or p != 0:
-					self._logSys.log(self.sumloglevel,
-						"Updated IPs for jail '%s' (-%d/+%d)",
-						self._jail.name, m, p)
-				self._logSys.debug(
-					"Next update for jail '%' in %i seconds",
-					self._jail.name, self.updateperiod)
-			finally:
-				self._timer = threading.Timer(self.updateperiod, self.update)
-				self._timer.start()
-
-	def stop(self):
-		"""If `banaction` set, clears blacklisted IPs.
-		"""
-		if self.banaction is not None:
-			if self._timer:
-				self._timer.cancel()
-				self._timer = None
-			self._unbanIPs(self._bannedips.copy())
-
-	def ban(self, aInfo):
-		"""Reports banned IP to badips.com.
-
-		Parameters
-		----------
-		aInfo : dict
-			Dictionary which includes information in relation to
-			the ban.
-
-		Raises
-		------
-		HTTPError
-			Any issues with badips.com request.
-		"""
-		try:
-			url = "/".join([self._badips, "add", self.category, str(aInfo['ip'])])
-			self._logSys.debug('badips.com: ban, url: %r', url)
-			response = urlopen(self._Request(url), timeout=self.timeout)
-		except HTTPError as response: # pragma: no cover
-			self.logError(response, "Failed to ban")
-			raise
-		else:
-			messages = json.loads(response.read().decode('utf-8'))
-			self._logSys.debug(
-				"Response from badips.com report: '%s'",
-				messages['suc'])
-
-Action = BadIPsAction

config/action.d/bsd-ipfw.conf

@@ -14,7 +14,10 @@
 # Notes.:  command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false).
 # Values:  CMD
 #
-actionstart = ipfw show | fgrep -c -m 1 -s 'table(<table>)' > /dev/null 2>&1 || ( ipfw show | awk 'BEGIN { b = <lowest_rule_num> } { if ($1 < b) {} else if ($1 == b) { b = $1 + 1 } else { e = b } } END { if (e) exit e <br> else exit b }'; num=$?; ipfw -q add $num <blocktype> <block> from table\(<table>\) to me <port>; echo $num > "<startstatefile>" )
+actionstart = ipfw show | fgrep -c -m 1 -s 'table(<table>)' > /dev/null 2>&1 || (
+    num=$(ipfw show | awk 'BEGIN { b = <lowest_rule_num> } { if ($1 == b) { b = $1 + 1 } } END { print b }');
+    ipfw -q add "$num" <blocktype> <block> from table\(<table>\) to me <port>; echo "$num" > "<startstatefile>"
+  )
 
 
 # Option:  actionstop

config/action.d/cloudflare.conf

@@ -5,7 +5,7 @@
 #
 # Please set jail.local's permission to 640 because it contains your CF API key.
 #
-# This action depends on curl.
+# This action depends on curl (and optionally jq).
 # Referenced from http://www.normyee.net/blog/2012/02/02/adding-cloudflare-support-to-fail2ban by NORM YEE
 #
 # To get your CloudFlare API Key: https://www.cloudflare.com/a/account/my-account
@@ -43,9 +43,9 @@ actioncheck =
 # API v1
 #actionban = curl -s -o /dev/null https://www.cloudflare.com/api_json.html -d 'a=ban' -d 'tkn=<cftoken>' -d 'email=<cfuser>' -d 'key=<ip>'
 # API v4
-actionban = curl -s -o /dev/null -X POST -H 'X-Auth-Email: <cfuser>' -H 'X-Auth-Key: <cftoken>' \
-            -H 'Content-Type: application/json' -d '{ "mode": "block", "configuration": { "target": "ip", "value": "<ip>" } }' \
-            https://api.cloudflare.com/client/v4/user/firewall/access_rules/rules
+actionban = curl -s -o /dev/null -X POST <_cf_api_prms> \
+            -d '{"mode":"block","configuration":{"target":"<cftarget>","value":"<ip>"},"notes":"Fail2Ban <name>"}' \
+            <_cf_api_url>
 
 # Option:  actionunban
 # Notes.:  command executed when unbanning an IP. Take care that the
@@ -58,9 +58,14 @@ actionban = curl -s -o /dev/null -X POST -H 'X-Auth-Email: <cfuser>' -H 'X-Auth-
 # API v1
 #actionunban = curl -s -o /dev/null https://www.cloudflare.com/api_json.html -d 'a=nul' -d 'tkn=<cftoken>' -d 'email=<cfuser>' -d 'key=<ip>'
 # API v4
-actionunban = curl -s -o /dev/null -X DELETE -H 'X-Auth-Email: <cfuser>' -H 'X-Auth-Key: <cftoken>' \
-              https://api.cloudflare.com/client/v4/user/firewall/access_rules/rules/$(curl -s -X GET -H 'X-Auth-Email: <cfuser>' -H 'X-Auth-Key: <cftoken>' \
-              'https://api.cloudflare.com/client/v4/user/firewall/access_rules/rules?mode=block&configuration_target=ip&configuration_value=<ip>&page=1&per_page=1' | cut -d'"' -f6)
+actionunban = id=$(curl -s -X GET <_cf_api_prms> \
+                   "<_cf_api_url>?mode=block&configuration_target=<cftarget>&configuration_value=<ip>&page=1&per_page=1&notes=Fail2Ban%%20<name>" \
+                   | { jq -r '.result[0].id' 2>/dev/null || tr -d '\n' | sed -nE 's/^.*"result"\s*:\s*\[\s*\{\s*"id"\s*:\s*"([^"]+)".*$/\1/p'; })
+              if [ -z "$id" ]; then echo "<name>: id for <ip> cannot be found"; exit 0; fi;
+              curl -s -o /dev/null -X DELETE <_cf_api_prms> "<_cf_api_url>/$id"
+
+_cf_api_url = https://api.cloudflare.com/client/v4/user/firewall/access_rules/rules
+_cf_api_prms = -H 'X-Auth-Email: <cfuser>' -H 'X-Auth-Key: <cftoken>' -H 'Content-Type: application/json'
 
 [Init]
 
@@ -76,3 +81,8 @@ actionunban = curl -s -o /dev/null -X DELETE -H 'X-Auth-Email: <cfuser>' -H 'X-A
 cftoken =
 
 cfuser =
+
+cftarget = ip
+
+[Init?family=inet6]
+cftarget = ip6

config/action.d/complain.conf

@@ -102,7 +102,7 @@ logpath = /dev/null
 # Notes.:  Your system mail command. Is passed 2 args: subject and recipient
 # Values:  CMD
 #
-mailcmd = mail -s
+mailcmd = mail -E 'set escape' -s
 
 # Option:  mailargs
 # Notes.:  Additional arguments to mail command. e.g. for standard Unix mail:

config/action.d/dshield.conf

@@ -179,7 +179,7 @@ tcpflags =
 # Notes.:  Your system mail command. Is passed 2 args: subject and recipient
 # Values:  CMD
 #
-mailcmd = mail -s
+mailcmd = mail -E 'set escape' -s
 
 # Option:  mailargs
 # Notes.:  Additional arguments to mail command. e.g. for standard Unix mail:

config/action.d/firewallcmd-ipset.conf

@@ -18,7 +18,7 @@ before = firewallcmd-common.conf
 
 [Definition]
 
-actionstart = ipset create <ipmset> hash:ip timeout <default-timeout><familyopt>
+actionstart = ipset create <ipmset> hash:ip timeout <default-ipsettime> <familyopt>
               firewall-cmd --direct --add-rule <family> filter <chain> 0 <actiontype> -m set --match-set <ipmset> src -j <blocktype>
 
 actionflush = ipset flush <ipmset>
@@ -27,9 +27,7 @@ actionstop = firewall-cmd --direct --remove-rule <family> filter <chain> 0 <acti
              <actionflush>
              ipset destroy <ipmset>
 
-actionban = ipset add <ipmset> <ip> timeout <bantime> -exist
-
-actionprolong = %(actionban)s
+actionban = ipset add <ipmset> <ip> timeout <ipsettime> -exist
 
 actionunban = ipset del <ipmset> <ip> -exist
 
@@ -42,11 +40,19 @@ actionunban = ipset del <ipmset> <ip> -exist
 #
 chain = INPUT_direct
 
-# Option: default-timeout
+# Option: default-ipsettime
 # Notes:  specifies default timeout in seconds (handled default ipset timeout only)
-# Values:  [ NUM ]  Default: 600
+# Values:  [ NUM ]  Default: 0 (no timeout, managed by fail2ban by unban)
+default-ipsettime = 0
 
-default-timeout = 600
+# Option: ipsettime
+# Notes:  specifies ticket timeout (handled ipset timeout only)
+# Values:  [ NUM ]  Default: 0 (managed by fail2ban by unban)
+ipsettime = 0
+
+# expresion to caclulate timeout from bantime, example:
+# banaction = %(known/banaction)s[ipsettime='<timeout-bantime>']
+timeout-bantime = $([ "<bantime>" -le 2147483 ] && echo "<bantime>" || echo 0)
 
 # Option: actiontype
 # Notes.: defines additions to the blocking rule
@@ -71,7 +77,7 @@ familyopt =
 [Init?family=inet6]
 
 ipmset = f2b-<name>6
-familyopt = <sp>family inet6
+familyopt = family inet6
 
 
 # DEV NOTES:

config/action.d/firewallcmd-rich-logging.conf

@@ -1,6 +1,6 @@
 # Fail2Ban configuration file
 #
-# Author: Donald Yandt 
+# Authors: Donald Yandt, Sergey G. Brester
 # 
 # Because of the rich rule commands requires firewalld-0.3.1+
 # This action uses firewalld rich-rules which gives you a cleaner iptables since it stores rules according to zones and not
@@ -10,36 +10,15 @@
 #
 # If you use the --permanent rule you get a xml file in /etc/firewalld/zones/<zone>.xml that can be shared and parsed easliy
 #
-# Example commands to view rules:
-# firewall-cmd [--zone=<zone>] --list-rich-rules
-# firewall-cmd [--zone=<zone>] --list-all
-# firewall-cmd [--zone=zone] --query-rich-rule='rule'
+# This is an derivative of firewallcmd-rich-rules.conf, see there for details and other parameters.
 
 [INCLUDES]
 
-before = firewallcmd-common.conf
+before = firewallcmd-rich-rules.conf
 
 [Definition]
 
-actionstart = 
-
-actionstop = 
-
-actioncheck = 
-
-# you can also use zones and/or service names. 
-#
-# zone example: 
-# firewall-cmd --zone=<zone> --add-rich-rule="rule family='<family>' source address='<ip>' port port='<port>' protocol='<protocol>' log prefix='f2b-<name>' level='<level>' limit value='<rate>/m' <rich-blocktype>"
-#
-# service name example:
-# firewall-cmd --zone=<zone> --add-rich-rule="rule family='<family>' source address='<ip>' service name='<service>' log prefix='f2b-<name>' level='<level>' limit value='<rate>/m' <rich-blocktype>"
-#
-# Because rich rules can only handle single or a range of ports we must split ports and execute the command for each port. Ports can be single and ranges separated by a comma or space for an example: http, https, 22-60, 18 smtp 
-
-actionban = ports="<port>"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --add-rich-rule="rule family='<family>' source address='<ip>' port port='$p' protocol='<protocol>' log prefix='f2b-<name>' level='<level>' limit value='<rate>/m' <rich-blocktype>"; done
-	   
-actionunban = ports="<port>"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --remove-rich-rule="rule family='<family>' source address='<ip>' port port='$p' protocol='<protocol>' log prefix='f2b-<name>' level='<level>' limit value='<rate>/m' <rich-blocktype>"; done
+rich-suffix = log prefix='f2b-<name>' level='<level>' limit value='<rate>/m' <rich-blocktype>
 
 [Init]
 
@@ -48,4 +27,3 @@ level = info
 
 # log rate per minute
 rate = 1
-

config/action.d/firewallcmd-rich-rules.conf

@@ -35,8 +35,10 @@ actioncheck =
 #
 # Because rich rules can only handle single or a range of ports we must split ports and execute the command for each port. Ports can be single and ranges separated by a comma or space for an example: http, https, 22-60, 18 smtp 
 
-actionban = ports="<port>"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --add-rich-rule="rule family='<family>' source address='<ip>' port port='$p' protocol='<protocol>' <rich-blocktype>"; done
+fwcmd_rich_rule = rule family='<family>' source address='<ip>' port port='$p' protocol='<protocol>' %(rich-suffix)s
+
+actionban = ports="<port>"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --add-rich-rule="%(fwcmd_rich_rule)s"; done
 	   
-actionunban = ports="<port>"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --remove-rich-rule="rule family='<family>' source address='<ip>' port port='$p' protocol='<protocol>' <rich-blocktype>"; done
-
+actionunban = ports="<port>"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --remove-rich-rule="%(fwcmd_rich_rule)s"; done
 
+rich-suffix = <rich-blocktype>

config/action.d/iptables-ipset-proto6-allports.conf

@@ -26,7 +26,7 @@ before = iptables-common.conf
 # Notes.:  command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false).
 # Values:  CMD
 #
-actionstart = ipset create <ipmset> hash:ip timeout <default-timeout><familyopt>
+actionstart = ipset create <ipmset> hash:ip timeout <default-ipsettime> <familyopt>
               <iptables> -I <chain> -m set --match-set <ipmset> src -j <blocktype>
 
 # Option:  actionflush
@@ -49,9 +49,7 @@ actionstop = <iptables> -D <chain> -m set --match-set <ipmset> src -j <blocktype
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = ipset add <ipmset> <ip> timeout <bantime> -exist
-
-actionprolong = %(actionban)s
+actionban = ipset add <ipmset> <ip> timeout <ipsettime> -exist
 
 # Option:  actionunban
 # Notes.:  command executed when unbanning an IP. Take care that the
@@ -63,11 +61,19 @@ actionunban = ipset del <ipmset> <ip> -exist
 
 [Init]
 
-# Option: default-timeout
+# Option: default-ipsettime
 # Notes:  specifies default timeout in seconds (handled default ipset timeout only)
-# Values:  [ NUM ]  Default: 600
+# Values:  [ NUM ]  Default: 0 (no timeout, managed by fail2ban by unban)
+default-ipsettime = 0
 
-default-timeout = 600
+# Option: ipsettime
+# Notes:  specifies ticket timeout (handled ipset timeout only)
+# Values:  [ NUM ]  Default: 0 (managed by fail2ban by unban)
+ipsettime = 0
+
+# expresion to caclulate timeout from bantime, example:
+# banaction = %(known/banaction)s[ipsettime='<timeout-bantime>']
+timeout-bantime = $([ "<bantime>" -le 2147483 ] && echo "<bantime>" || echo 0)
 
 ipmset = f2b-<name>
 familyopt =
@@ -76,4 +82,4 @@ familyopt =
 [Init?family=inet6]
 
 ipmset = f2b-<name>6
-familyopt = <sp>family inet6
+familyopt = family inet6

config/action.d/iptables-ipset-proto6.conf

@@ -26,7 +26,7 @@ before = iptables-common.conf
 # Notes.:  command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false).
 # Values:  CMD
 #
-actionstart = ipset create <ipmset> hash:ip timeout <default-timeout><familyopt>
+actionstart = ipset create <ipmset> hash:ip timeout <default-ipsettime> <familyopt>
               <iptables> -I <chain> -p <protocol> -m multiport --dports <port> -m set --match-set <ipmset> src -j <blocktype>
 
 # Option:  actionflush
@@ -49,9 +49,7 @@ actionstop = <iptables> -D <chain> -p <protocol> -m multiport --dports <port> -m
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = ipset add <ipmset> <ip> timeout <bantime> -exist
-
-actionprolong = %(actionban)s
+actionban = ipset add <ipmset> <ip> timeout <ipsettime> -exist
 
 # Option:  actionunban
 # Notes.:  command executed when unbanning an IP. Take care that the
@@ -63,11 +61,19 @@ actionunban = ipset del <ipmset> <ip> -exist
 
 [Init]
 
-# Option: default-timeout
+# Option: default-ipsettime
 # Notes:  specifies default timeout in seconds (handled default ipset timeout only)
-# Values:  [ NUM ]  Default: 600
+# Values:  [ NUM ]  Default: 0 (no timeout, managed by fail2ban by unban)
+default-ipsettime = 0
 
-default-timeout = 600
+# Option: ipsettime
+# Notes:  specifies ticket timeout (handled ipset timeout only)
+# Values:  [ NUM ]  Default: 0 (managed by fail2ban by unban)
+ipsettime = 0
+
+# expresion to caclulate timeout from bantime, example:
+# banaction = %(known/banaction)s[ipsettime='<timeout-bantime>']
+timeout-bantime = $([ "<bantime>" -le 2147483 ] && echo "<bantime>" || echo 0)
 
 ipmset = f2b-<name>
 familyopt =
@@ -76,4 +82,4 @@ familyopt =
 [Init?family=inet6]
 
 ipmset = f2b-<name>6
-familyopt = <sp>family inet6
+familyopt = family inet6

config/action.d/mail-buffered.conf

@@ -17,7 +17,7 @@ actionstart = printf %%b "Hi,\n
               The jail <name> has been started successfully.\n
               Output will be buffered until <lines> lines are available.\n
               Regards,\n
-              Fail2Ban"|mail -s "[Fail2Ban] <name>: started on <fq-hostname>" <dest>
+              Fail2Ban"|mail -E 'set escape' -s "[Fail2Ban] <name>: started on <fq-hostname>" <dest>
 
 # Option:  actionstop
 # Notes.:  command executed at the stop of jail (or at the end of Fail2Ban)
@@ -28,13 +28,13 @@ actionstop = if [ -f <tmpfile> ]; then
                  These hosts have been banned by Fail2Ban.\n
                  `cat <tmpfile>`
                  Regards,\n
-                 Fail2Ban"|mail -s "[Fail2Ban] <name>: Summary from <fq-hostname>" <dest>
+                 Fail2Ban"|mail -E 'set escape' -s "[Fail2Ban] <name>: Summary from <fq-hostname>" <dest>
                  rm <tmpfile>
              fi
              printf %%b "Hi,\n
              The jail <name> has been stopped.\n
              Regards,\n
-             Fail2Ban"|mail -s "[Fail2Ban] <name>: stopped on <fq-hostname>" <dest>
+             Fail2Ban"|mail -E 'set escape' -s "[Fail2Ban] <name>: stopped on <fq-hostname>" <dest>
 
 # Option:  actioncheck
 # Notes.:  command executed once before each actionban command
@@ -55,7 +55,7 @@ actionban = printf %%b "`date`: <ip> (<failures> failures)\n" >> <tmpfile>
                 These hosts have been banned by Fail2Ban.\n
                 `cat <tmpfile>`
                 \nRegards,\n
-                Fail2Ban"|mail -s "[Fail2Ban] <name>: Summary" <dest>
+                Fail2Ban"|mail -E 'set escape' -s "[Fail2Ban] <name>: Summary" <dest>
                 rm <tmpfile>
             fi
 

config/action.d/mail-whois-lines.conf

@@ -72,7 +72,7 @@ actionunban =
 # Notes.:  Your system mail command. Is passed 2 args: subject and recipient
 # Values:  CMD
 #
-mailcmd = mail -s
+mailcmd = mail -E 'set escape' -s
 
 # Default name of the chain
 #

config/action.d/mail-whois.conf

@@ -20,7 +20,7 @@ norestored = 1
 actionstart = printf %%b "Hi,\n
               The jail <name> has been started successfully.\n
               Regards,\n
-              Fail2Ban"|mail -s "[Fail2Ban] <name>: started on <fq-hostname>" <dest>
+              Fail2Ban"|mail -E 'set escape' -s "[Fail2Ban] <name>: started on <fq-hostname>" <dest>
 
 # Option:  actionstop
 # Notes.:  command executed at the stop of jail (or at the end of Fail2Ban)
@@ -29,7 +29,7 @@ actionstart = printf %%b "Hi,\n
 actionstop = printf %%b "Hi,\n
              The jail <name> has been stopped.\n
              Regards,\n
-             Fail2Ban"|mail -s "[Fail2Ban] <name>: stopped on <fq-hostname>" <dest>
+             Fail2Ban"|mail -E 'set escape' -s "[Fail2Ban] <name>: stopped on <fq-hostname>" <dest>
 
 # Option:  actioncheck
 # Notes.:  command executed once before each actionban command
@@ -49,7 +49,7 @@ actionban = printf %%b "Hi,\n
             Here is more information about <ip> :\n
             `%(_whois_command)s`\n
             Regards,\n
-            Fail2Ban"|mail -s "[Fail2Ban] <name>: banned <ip> from <fq-hostname>" <dest>
+            Fail2Ban"|mail -E 'set escape' -s "[Fail2Ban] <name>: banned <ip> from <fq-hostname>" <dest>
 
 # Option:  actionunban
 # Notes.:  command executed when unbanning an IP. Take care that the

config/action.d/mail.conf

@@ -16,7 +16,7 @@ norestored = 1
 actionstart = printf %%b "Hi,\n
               The jail <name> has been started successfully.\n
               Regards,\n
-              Fail2Ban"|mail -s "[Fail2Ban] <name>: started  on <fq-hostname>" <dest>
+              Fail2Ban"|mail -E 'set escape' -s "[Fail2Ban] <name>: started  on <fq-hostname>" <dest>
 
 # Option:  actionstop
 # Notes.:  command executed at the stop of jail (or at the end of Fail2Ban)
@@ -25,7 +25,7 @@ actionstart = printf %%b "Hi,\n
 actionstop = printf %%b "Hi,\n
              The jail <name> has been stopped.\n
              Regards,\n
-             Fail2Ban"|mail -s "[Fail2Ban] <name>: stopped on <fq-hostname>" <dest>
+             Fail2Ban"|mail -E 'set escape' -s "[Fail2Ban] <name>: stopped on <fq-hostname>" <dest>
 
 # Option:  actioncheck
 # Notes.:  command executed once before each actionban command
@@ -43,7 +43,7 @@ actionban = printf %%b "Hi,\n
             The IP <ip> has just been banned by Fail2Ban after
             <failures> attempts against <name>.\n
             Regards,\n
-            Fail2Ban"|mail -s "[Fail2Ban] <name>: banned <ip> from <fq-hostname>" <dest>
+            Fail2Ban"|mail -E 'set escape' -s "[Fail2Ban] <name>: banned <ip> from <fq-hostname>" <dest>
 
 # Option:  actionunban
 # Notes.:  command executed when unbanning an IP. Take care that the

config/action.d/nftables.conf

@@ -34,7 +34,7 @@ type = multiport
 
 rule_match-custom =
 rule_match-allports = meta l4proto \{ <protocol> \}
-rule_match-multiport = $proto dport \{ <port> \}
+rule_match-multiport = $proto dport \{ $(echo '<port>' | sed s/:/-/g) \}
 match = <rule_match-<type>>
 
 # Option:  rule_stat

config/action.d/nginx-block-map.conf

@@ -84,8 +84,15 @@ srv_cfg_path = /etc/nginx/
 #srv_cmd = nginx -c %(srv_cfg_path)s/nginx.conf
 srv_cmd = nginx
 
-# first test configuration is correct, hereafter send reload signal:
-blck_lst_reload = %(srv_cmd)s -qt; if [ $? -eq 0 ]; then
+# pid file (used to check nginx is running):
+srv_pid = /run/nginx.pid
+
+# command used to check whether nginx is running and configuration is valid:
+srv_is_running = [ -f "%(srv_pid)s" ]
+srv_check_cmd = %(srv_is_running)s && %(srv_cmd)s -qt
+
+# first test nginx is running and configuration is correct, hereafter send reload signal:
+blck_lst_reload = %(srv_check_cmd)s; if [ $? -eq 0 ]; then
                     %(srv_cmd)s -s reload; if [ $? -ne 0 ]; then echo 'reload failed.'; fi;
                   fi;
 
@@ -103,6 +110,8 @@ actionstop = %(actionflush)s
 
 actioncheck = 
 
-actionban = echo "\\\\<fid> 1;" >> '%(blck_lst_file)s'; %(blck_lst_reload)s
+_echo_blck_row = printf '\%%s 1;\n' "<fid>"
 
-actionunban = id=$(echo "<fid>" | sed -e 's/[]\/$*.^|[]/\\&/g'); sed -i "/^\\\\$id 1;$/d" %(blck_lst_file)s; %(blck_lst_reload)s
+actionban = %(_echo_blck_row)s >> '%(blck_lst_file)s'; %(blck_lst_reload)s
+
+actionunban = id=$(%(_echo_blck_row)s | sed -e 's/[]\/$*.^|[]/\\&/g'); sed -i "/^$id$/d" %(blck_lst_file)s; %(blck_lst_reload)s

config/action.d/osx-afctl.conf

@@ -12,5 +12,5 @@ actioncheck =
 actionban = /usr/libexec/afctl -a <ip> -t <bantime>
 actionunban = /usr/libexec/afctl -r <ip>
 
-actionprolong = %(actionunban)s && %(actionban)s
-
+[Init]
+bantime = 2880

config/action.d/shorewall-ipset-proto6.conf

@@ -51,7 +51,7 @@
 # Values:  CMD
 #
 actionstart = if ! ipset -quiet -name list f2b-<name> >/dev/null;
-              then ipset -quiet -exist create f2b-<name> hash:ip timeout <default-timeout>;
+              then ipset -quiet -exist create f2b-<name> hash:ip timeout <default-ipsettime>;
               fi
 
 # Option:  actionstop
@@ -66,9 +66,7 @@ actionstop = ipset flush f2b-<name>
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = ipset add f2b-<name> <ip> timeout <bantime> -exist
-
-actionprolong = %(actionban)s
+actionban = ipset add f2b-<name> <ip> timeout <ipsettime> -exist
 
 # Option:  actionunban
 # Notes.:  command executed when unbanning an IP. Take care that the
@@ -78,8 +76,16 @@ actionprolong = %(actionban)s
 #
 actionunban = ipset del f2b-<name> <ip> -exist
 
-# Option: default-timeout
+# Option: default-ipsettime
 # Notes:  specifies default timeout in seconds (handled default ipset timeout only)
-# Values:  [ NUM ]  Default: 600
+# Values:  [ NUM ]  Default: 0 (no timeout, managed by fail2ban by unban)
+default-ipsettime = 0
 
-default-timeout = 600
+# Option: ipsettime
+# Notes:  specifies ticket timeout (handled ipset timeout only)
+# Values:  [ NUM ]  Default: 0 (managed by fail2ban by unban)
+ipsettime = 0
+
+# expresion to caclulate timeout from bantime, example:
+# banaction = %(known/banaction)s[ipsettime='<timeout-bantime>']
+timeout-bantime = $([ "<bantime>" -le 2147483 ] && echo "<bantime>" || echo 0)

config/fail2ban.conf

@@ -19,7 +19,7 @@
 #         NOTICE
 #         INFO
 #         DEBUG
-# Values: [ LEVEL ]  Default: ERROR
+# Values: [ LEVEL ]  Default: INFO
 #
 loglevel = INFO
 
@@ -55,6 +55,12 @@ socket = /var/run/fail2ban/fail2ban.sock
 #
 pidfile = /var/run/fail2ban/fail2ban.pid
 
+# Option: allowipv6
+# Notes.: Allows IPv6 interface:
+#         Default: auto
+# Values: [ auto yes (on, true, 1) no (off, false, 0) ] Default: auto
+#allowipv6 = auto
+
 # Options: dbfile
 # Notes.: Set the file for the fail2ban persistent data to be stored.
 #         A value of ":memory:" means database is only stored in memory 

config/filter.d/apache-noscript.conf

@@ -17,9 +17,9 @@ before = apache-common.conf
 
 [Definition]
 
-script = /\S*(?:php(?:[45]|[.-]cgi)?|\.asp|\.exe|\.pl)
+script = /\S*(?:php(?:[45]|[.-]cgi)?|\.asp|\.exe|\.pl|\bcgi-bin/)
 
-prefregex = ^%(_apache_error_client)s (?:AH0(?:01(?:28|30)|1(?:264|071)): )?(?:(?:[Ff]ile|script|[Gg]ot) )<F-CONTENT>.+</F-CONTENT>$
+prefregex = ^%(_apache_error_client)s (?:AH0(?:01(?:28|30)|1(?:264|071)|2811): )?(?:(?:[Ff]ile|script|[Gg]ot) )<F-CONTENT>.+</F-CONTENT>$
 
 failregex = ^(?:does not exist|not found or unable to stat): <script>\b
             ^'<script>\S*' not found or unable to stat

config/filter.d/apache-overflows.conf

@@ -8,7 +8,7 @@ before = apache-common.conf
 
 [Definition]
 
-failregex = ^%(_apache_error_client)s (?:(?:AH0013[456]: )?Invalid (method|URI) in request\b|(?:AH00565: )?request failed: URI too long \(longer than \d+\)|request failed: erroneous characters after protocol string:|(?:AH00566: )?request failed: invalid characters in URI\b)
+failregex = ^%(_apache_error_client)s (?:(?:AH001[23][456]: )?Invalid (method|URI) in request\b|(?:AH00565: )?request failed: URI too long \(longer than \d+\)|request failed: erroneous characters after protocol string:|(?:AH00566: )?request failed: invalid characters in URI\b)
 
 ignoreregex =
 

config/filter.d/bitwarden.conf

@@ -2,5 +2,12 @@
 # Detecting failed login attempts
 # Logged in bwdata/logs/identity/Identity/log.txt
 
+[INCLUDES]
+before = common.conf
+
 [Definition]
-failregex = ^\s*\[WRN\]\s+Failed login attempt(?:, 2FA invalid)?\. <HOST>$
+_daemon = Bitwarden-Identity
+failregex = ^%(__prefix_line)s\s*\[(?:W(?:RN|arning)|Bit\.Core\.[^\]]+)\]\s+Failed login attempt(?:, 2FA invalid)?\. <ADDR>$
+
+# DEV Notes:
+# __prefix_line can result to an empty string, so it can support syslog and non-syslog at once.

config/filter.d/common.conf

@@ -25,7 +25,7 @@ __pid_re = (?:\[\d+\])
 
 # Daemon name (with optional source_file:line or whatever)
 # EXAMPLES: pam_rhosts_auth, [sshd], pop(pam_unix)
-__daemon_re = [\[\(]?%(_daemon)s(?:\(\S+\))?[\]\)]?:?
+__daemon_re = [\[\(]?<_daemon>(?:\(\S+\))?[\]\)]?:?
 
 # extra daemon info
 # EXAMPLE: [ID 800047 auth.info]
@@ -33,7 +33,7 @@ __daemon_extra_re = \[ID \d+ \S+\]
 
 # Combinations of daemon name and PID
 # EXAMPLES: sshd[31607], pop(pam_unix)[4920]
-__daemon_combs_re = (?:%(__pid_re)s?:\s+%(__daemon_re)s|%(__daemon_re)s%(__pid_re)s?:?)
+__daemon_combs_re = (?:<__pid_re>?:\s+<__daemon_re>|<__daemon_re><__pid_re>?:?)
 
 # Some messages have a kernel prefix with a timestamp
 # EXAMPLES: kernel: [769570.846956]
@@ -69,12 +69,12 @@ datepattern = <lt_<logtype>/datepattern>
 
 [lt_file]
 # Common line prefixes for logtype "file":
-__prefix_line = %(__date_ambit)s?\s*(?:%(__bsd_syslog_verbose)s\s+)?(?:%(__hostname)s\s+)?(?:%(__kernel_prefix)s\s+)?(?:%(__vserver)s\s+)?(?:%(__daemon_combs_re)s\s+)?(?:%(__daemon_extra_re)s\s+)?
+__prefix_line = <__date_ambit>?\s*(?:<__bsd_syslog_verbose>\s+)?(?:<__hostname>\s+)?(?:<__kernel_prefix>\s+)?(?:<__vserver>\s+)?(?:<__daemon_combs_re>\s+)?(?:<__daemon_extra_re>\s+)?
 datepattern = {^LN-BEG}
 
 [lt_short]
 # Common (short) line prefix for logtype "journal" (corresponds output of formatJournalEntry):
-__prefix_line = \s*(?:%(__hostname)s\s+)?(?:%(_daemon)s%(__pid_re)s?:?\s+)?(?:%(__kernel_prefix)s\s+)?
+__prefix_line = \s*(?:<__hostname>\s+)?(?:<_daemon><__pid_re>?:?\s+)?(?:<__kernel_prefix>\s+)?
 datepattern = %(lt_file/datepattern)s
 [lt_journal]
 __prefix_line = %(lt_short/__prefix_line)s

config/filter.d/courier-smtp.conf

@@ -12,7 +12,7 @@ before = common.conf
 
 _daemon = courieresmtpd
 
-prefregex = ^%(__prefix_line)serror,relay=<HOST>,<F-CONTENT>.+</F-CONTENT>$
+prefregex = ^%(__prefix_line)serror,relay=<HOST>,(?:port=\d+,)?<F-CONTENT>.+</F-CONTENT>$
 
 failregex = ^[^:]*: 550 User (<.*> )?unknown\.?$
             ^msg="535 Authentication failed\.",cmd:( AUTH \S+)?( [0-9a-zA-Z\+/=]+)?(?: \S+)$

config/filter.d/dovecot.conf

@@ -8,17 +8,18 @@ before = common.conf
 [Definition]
 
 _auth_worker = (?:dovecot: )?auth(?:-worker)?
+_auth_worker_info = (?:conn \w+:auth(?:-worker)? \(uid=\w+\): auth(?:-worker)?<\d+>: )?
 _daemon = (?:dovecot(?:-auth)?|auth)
 
-prefregex = ^%(__prefix_line)s(?:%(_auth_worker)s(?:\([^\)]+\))?: )?(?:%(__pam_auth)s(?:\(dovecot:auth\))?: |(?:pop3|imap)-login: )?(?:Info: )?<F-CONTENT>.+</F-CONTENT>$
+prefregex = ^%(__prefix_line)s(?:%(_auth_worker)s(?:\([^\)]+\))?: )?(?:%(__pam_auth)s(?:\(dovecot:auth\))?: |(?:pop3|imap|managesieve|submission)-login: )?(?:Info: )?%(_auth_worker_info)s<F-CONTENT>.+</F-CONTENT>$
 
 failregex = ^authentication failure; logname=<F-ALT_USER1>\S*</F-ALT_USER1> uid=\S* euid=\S* tty=dovecot ruser=<F-USER>\S*</F-USER> rhost=<HOST>(?:\s+user=<F-ALT_USER>\S*</F-ALT_USER>)?\s*$
-            ^(?:Aborted login|Disconnected)(?::(?: [^ \(]+)+)? \((?:auth failed, \d+ attempts(?: in \d+ secs)?|tried to use (?:disabled|disallowed) \S+ auth|proxy dest auth failed)\):(?: user=<<F-USER>[^>]*</F-USER>>,)?(?: method=\S+,)? rip=<HOST>(?:[^>]*(?:, session=<\S+>)?)\s*$
-            ^pam\(\S+,<HOST>(?:,\S*)?\): pam_authenticate\(\) failed: (?:User not known to the underlying authentication module: \d+ Time\(s\)|Authentication failure \(password mismatch\?\)|Permission denied)\s*$
-            ^[a-z\-]{3,15}\(\S*,<HOST>(?:,\S*)?\): (?:unknown user|invalid credentials|Password mismatch)\s*$
+            ^(?:Aborted login|Disconnected|Remote closed connection|Client has quit the connection)(?::(?: [^ \(]+)+)? \((?:auth failed, \d+ attempts(?: in \d+ secs)?|tried to use (?:disabled|disallowed) \S+ auth|proxy dest auth failed)\):(?: user=<<F-USER>[^>]*</F-USER>>,)?(?: method=\S+,)? rip=<HOST>(?:[^>]*(?:, session=<\S+>)?)\s*$
+            ^pam\(\S+,<HOST>(?:,\S*)?\): pam_authenticate\(\) failed: (?:User not known to the underlying authentication module: \d+ Time\(s\)|Authentication failure \([Pp]assword mismatch\?\)|Permission denied)\s*$
+            ^[a-z\-]{3,15}\(\S*,<HOST>(?:,\S*)?\): (?:[Uu]nknown user|[Ii]nvalid credentials|[Pp]assword mismatch)
             <mdre-<mode>>
 
-mdre-aggressive = ^(?:Aborted login|Disconnected)(?::(?: [^ \(]+)+)? \((?:no auth attempts|disconnected before auth was ready,|client didn't finish \S+ auth,)(?: (?:in|waited) \d+ secs)?\):(?: user=<[^>]*>,)?(?: method=\S+,)? rip=<HOST>(?:[^>]*(?:, session=<\S+>)?)\s*$
+mdre-aggressive = ^(?:Aborted login|Disconnected|Remote closed connection|Client has quit the connection)(?::(?: [^ \(]+)+)? \((?:no auth attempts|disconnected before auth was ready,|client didn't finish \S+ auth,)(?: (?:in|waited) \d+ secs)?\):(?: user=<[^>]*>,)?(?: method=\S+,)? rip=<HOST>(?:[^>]*(?:, session=<\S+>)?)\s*$
 
 mdre-normal = 
 

config/filter.d/exim-common.conf

@@ -12,7 +12,7 @@ after = exim-common.local
 host_info_pre = (?:H=([\w.-]+ )?(?:\(\S+\) )?)?
 host_info_suf = (?::\d+)?(?: I=\[\S+\](:\d+)?)?(?: U=\S+)?(?: P=e?smtp)?(?: F=(?:<>|[^@]+@\S+))?\s
 host_info = %(host_info_pre)s\[<HOST>\]%(host_info_suf)s
-pid = (?: \[\d+\])?
+pid = (?: \[\d+\]| \w+ exim\[\d+\]:)?
 
 # DEV Notes:
 # From exim source code: ./src/receive.c:add_host_info_for_log

config/filter.d/gitlab.conf

@@ -0,0 +1,6 @@
+# Fail2Ban filter for Gitlab
+# Detecting unauthorized access to the Gitlab Web portal
+# typically logged in /var/log/gitlab/gitlab-rails/application.log
+
+[Definition]
+failregex = ^: Failed Login: username=<F-USER>.+</F-USER> ip=<HOST>$

config/filter.d/grafana.conf

@@ -0,0 +1,9 @@
+# Fail2Ban filter for Grafana
+# Detecting unauthorized access
+# Typically logged in /var/log/grafana/grafana.log
+
+[Init]
+datepattern = ^t=%%Y-%%m-%%dT%%H:%%M:%%S%%z
+
+[Definition]
+failregex = ^(?: lvl=err?or)? msg="Invalid username or password"(?: uname=(?:"<F-ALT_USER>[^"]+</F-ALT_USER>"|<F-USER>\S+</F-USER>)| error="<F-ERROR>[^"]+</F-ERROR>"| \S+=(?:\S*|"[^"]+"))* remote_addr=<ADDR>$

config/filter.d/guacamole.conf

@@ -5,21 +5,47 @@
 
 [Definition]
 
-# Option: failregex
-# Notes.: regex to match the password failures messages in the logfile.
-# Values: TEXT
-#
+logging = catalina
+failregex = <L_<logging>/failregex>
+maxlines = <L_<logging>/maxlines>
+datepattern = <L_<logging>/datepattern>
+
+[L_catalina]
+
 failregex = ^.*\nWARNING: Authentication attempt from <HOST> for user "[^"]*" failed\.$
 
-# Option:  ignoreregex
-# Notes.:  regex to ignore. If this regex matches, the line is ignored.
-# Values:  TEXT
-#
-ignoreregex = 
-
-# "maxlines" is number of log lines to buffer for multi-line regex searches
 maxlines = 2
 
 datepattern = ^%%b %%d, %%ExY %%I:%%M:%%S %%p
               ^WARNING:()**
-              {^LN-BEG}
+              {^LN-BEG}
+
+[L_webapp]
+
+failregex = ^ \[\S+\] WARN  \S+ - Authentication attempt from <HOST> for user "<F-USER>[^"]+</F-USER>" failed.
+
+maxlines = 1
+
+datepattern = ^%%H:%%M:%%S.%%f
+
+# DEV Notes:
+#
+# failregex is based on the default pattern given in Guacamole documentation :
+# https://guacamole.apache.org/doc/gug/configuring-guacamole.html#webapp-logging
+#
+# The following logback.xml Guacamole configuration file can then be used accordingly :
+# <configuration>
+#   <appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
+#     <file>/var/log/guacamole.log</file>
+#     <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+#       <fileNamePattern>/var/log/guacamole.%d.log.gz</fileNamePattern>
+#       <maxHistory>32</maxHistory>
+#     </rollingPolicy>
+#     <encoder>
+#       <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
+#     </encoder>
+#   </appender>
+#   <root level="info">
+#     <appender-ref ref="FILE" />
+#   </root>
+# </configuration>

config/filter.d/ignorecommands/apache-fakegooglebot

@@ -6,24 +6,35 @@
 #
 import sys
 from fail2ban.server.ipdns import DNSUtils, IPAddr
+from threading import Thread
 
 def process_args(argv):
-    if len(argv) != 2:
-       raise ValueError("Please provide a single IP as an argument. Got: %s\n"
-                        % (argv[1:]))
+    if len(argv) - 1 not in (1, 2):
+       raise ValueError("Usage %s ip ?timeout?. Got: %s\n"
+                        % (argv[0], argv[1:]))
     ip = argv[1]
 
     if not IPAddr(ip).isValid:
        raise ValueError("Argument must be a single valid IP. Got: %s\n"
                         % ip)
-    return ip
+    return argv[1:]
 
 google_ips = None
 
-def is_googlebot(ip):
+def is_googlebot(ip, timeout=55):
     import re
 
-    host = DNSUtils.ipToName(ip)
+    timeout = float(timeout or 0)
+    if timeout:
+      def ipToNameTO(host, ip, timeout):
+        host[0] = DNSUtils.ipToName(ip)
+      host = [None]
+      th = Thread(target=ipToNameTO, args=(host, ip, timeout)); th.daemon=True; th.start()
+      th.join(timeout)
+      host = host[0]
+    else:
+      host = DNSUtils.ipToName(ip)
+
     if not host or not re.match(r'.*\.google(bot)?\.com$', host):
        return False
     host_ips = DNSUtils.dnsToIp(host)
@@ -31,7 +42,7 @@ def is_googlebot(ip):
 
 if __name__ == '__main__': # pragma: no cover
     try:
-      ret = is_googlebot(process_args(sys.argv))
+      ret = is_googlebot(*process_args(sys.argv))
     except ValueError as e:
       sys.stderr.write(str(e))
       sys.exit(2)

config/filter.d/lighttpd-auth.conf

@@ -3,7 +3,7 @@
 
 [Definition]
 
-failregex = ^: \((?:http|mod)_auth\.c\.\d+\) (?:password doesn\'t match .* username: .*|digest: auth failed for .*: wrong password|get_password failed), IP: <HOST>\s*$
+failregex = ^\s*(?:: )?\(?(?:http|mod)_auth\.c\.\d+\) (?:password doesn\'t match for (?:\S+|.*?) username:\s+<F-USER>(?:\S+|.*?)</F-USER>\s*|digest: auth failed(?: for\s+<F-ALT_USER>(?:\S+|.*?)</F-ALT_USER>\s*)?: (?:wrong password|uri mismatch \([^\)]*\))|get_password failed),? IP: <HOST>\s*$
 
 ignoreregex = 
 

config/filter.d/monit.conf

@@ -8,13 +8,17 @@
 # common.local
 before = common.conf
 
+# [DEFAULT]
+# logtype = short
+
 [Definition]
 
 _daemon = monit
 
+_prefix = Warning|HttpRequest
+
 # Regexp for previous (accessing monit httpd) and new (access denied) versions
-failregex = ^\[\s*\]\s*error\s*:\s*Warning:\s+Client '<HOST>' supplied (?:unknown user '[^']+'|wrong password for user '[^']*') accessing monit httpd$
-            ^%(__prefix_line)s\w+: access denied -- client <HOST>: (?:unknown user '[^']+'|wrong password for user '[^']*'|empty password)$
+failregex = ^%(__prefix_line)s(?:error\s*:\s+)?(?:%(_prefix)s):\s+(?:access denied\s+--\s+)?[Cc]lient '?<HOST>'?(?:\s+supplied|\s*:)\s+(?:unknown user '<F-ALT_USER>[^']+</F-ALT_USER>'|wrong password for user '<F-USER>[^']*</F-USER>'|empty password)
 
 # Ignore login with empty user (first connect, no user specified)
 # ignoreregex = %(__prefix_line)s\w+: access denied -- client <HOST>: (?:unknown user '')

config/filter.d/mysqld-auth.conf

@@ -17,7 +17,7 @@ before = common.conf
 
 _daemon = mysqld
 
-failregex = ^%(__prefix_line)s(?:(?:\d{6}|\d{4}-\d{2}-\d{2})[ T]\s?\d{1,2}:\d{2}:\d{2} )?(?:\d+ )?\[\w+\] (?:\[[^\]]+\] )*Access denied for user '[^']+'@'<HOST>' (to database '[^']*'|\(using password: (YES|NO)\))*\s*$
+failregex = ^%(__prefix_line)s(?:(?:\d{6}|\d{4}-\d{2}-\d{2})[ T]\s?\d{1,2}:\d{2}:\d{2} )?(?:\d+ )?\[\w+\] (?:\[[^\]]+\] )*Access denied for user '<F-USER>[^']+</F-USER>'@'<HOST>' (to database '[^']*'|\(using password: (YES|NO)\))*\s*$
 
 ignoreregex = 
 

config/filter.d/named-refused.conf

@@ -32,7 +32,7 @@ __daemon_combs_re=(?:%(__pid_re)s?:\s+%(__daemon_re)s|%(__daemon_re)s%(__pid_re)
 
 #       hostname       daemon_id         spaces
 # this can be optional (for instance if we match named native log files)
-__line_prefix=(?:\s\S+ %(__daemon_combs_re)s\s+)?
+__line_prefix=(?:\s*\S+ %(__daemon_combs_re)s\s+)?
 
 prefregex = ^%(__line_prefix)s(?: error:)?\s*client(?: @\S*)? <HOST>#\S+(?: \([\S.]+\))?: <F-CONTENT>.+</F-CONTENT>\s(?:denied|\(NOTAUTH\))\s*$
 

config/filter.d/phpmyadmin-syslog.conf

@@ -1,4 +1,4 @@
-# Fail2Ban fitler for the phpMyAdmin-syslog
+# Fail2Ban filter for the phpMyAdmin-syslog
 #
 
 [INCLUDES]

config/filter.d/postfix.conf

@@ -12,16 +12,12 @@ before = common.conf
 
 _daemon = postfix(-\w+)?/\w+(?:/smtp[ds])?
 _port = (?::\d+)?
+_pref = [A-Z]{4}
 
 prefregex = ^%(__prefix_line)s<mdpr-<mode>> <F-CONTENT>.+</F-CONTENT>$
 
-mdpr-normal = (?:\w+: reject:|(?:improper command pipelining|too many errors) after \S+)
-mdre-normal=^RCPT from [^[]*\[<HOST>\]%(_port)s: 55[04] 5\.7\.1\s
-            ^RCPT from [^[]*\[<HOST>\]%(_port)s: 45[04] 4\.7\.\d+ (?:Service unavailable\b|Client host rejected: cannot find your (reverse )?hostname\b)
-            ^RCPT from [^[]*\[<HOST>\]%(_port)s: 450 4\.7\.\d+ (<[^>]*>)?: Helo command rejected: Host not found\b
-            ^EHLO from [^[]*\[<HOST>\]%(_port)s: 504 5\.5\.\d+ (<[^>]*>)?: Helo command rejected: need fully-qualified hostname\b
-            ^(RCPT|VRFY) from [^[]*\[<HOST>\]%(_port)s: 550 5\.1\.1\s
-            ^RCPT from [^[]*\[<HOST>\]%(_port)s: 450 4\.1\.\d+ (<[^>]*>)?: Sender address rejected: Domain not found\b
+mdpr-normal = (?:\w+: (?:milter-)?reject:|(?:improper command pipelining|too many errors) after \S+)
+mdre-normal=^%(_pref)s from [^[]*\[<HOST>\]%(_port)s: [45][50][04] [45]\.\d\.\d+ (?:(?:<[^>]*>)?: )?(?:(?:Helo command|(?:Sender|Recipient) address) rejected: )?(?:Service unavailable|User unknown|(?:Client host|Command|Data command) rejected|Relay access denied|(?:Host|Domain) not found|need fully-qualified hostname|match)\b
             ^from [^[]*\[<HOST>\]%(_port)s:?
 
 mdpr-auth = warning:
@@ -31,13 +27,13 @@ mdre-auth2= ^[^[]*\[<HOST>\]%(_port)s: SASL ((?i)LOGIN|PLAIN|(?:CRAM|DIGEST)-MD5
 
 # Mode "rbl" currently included in mode "normal", but if needed for jail "postfix-rbl" only:
 mdpr-rbl = %(mdpr-normal)s
-mdre-rbl  = ^RCPT from [^[]*\[<HOST>\]%(_port)s: [45]54 [45]\.7\.1 Service unavailable; Client host \[\S+\] blocked\b
+mdre-rbl  = ^%(_pref)s from [^[]*\[<HOST>\]%(_port)s: [45]54 [45]\.7\.1 Service unavailable; Client host \[\S+\] blocked\b
 
 # Mode "rbl" currently included in mode "normal" (within 1st rule)
 mdpr-more = %(mdpr-normal)s
 mdre-more = %(mdre-normal)s
 
-mdpr-ddos = lost connection after(?! DATA) [A-Z]+
+mdpr-ddos = (?:lost connection after(?! DATA) [A-Z]+|disconnect(?= from \S+(?: \S+=\d+)* auth=0/(?:[1-9]|\d\d+)))
 mdre-ddos = ^from [^[]*\[<HOST>\]%(_port)s:?
 
 mdpr-extra = (?:%(mdpr-auth)s|%(mdpr-normal)s)

config/filter.d/proftpd.conf

@@ -1,4 +1,4 @@
-# Fail2Ban fitler for the Proftpd FTP daemon
+# Fail2Ban filter for the Proftpd FTP daemon
 #
 # Set "UseReverseDNS off" in proftpd.conf to avoid the need for DNS.
 # See: http://www.proftpd.org/docs/howto/DNS.html
@@ -14,16 +14,15 @@ before = common.conf
 
 _daemon = proftpd
 
-__suffix_failed_login = (User not authorized for login|No such user found|Incorrect password|Password expired|Account disabled|Invalid shell: '\S+'|User in \S+|Limit (access|configuration) denies login|Not a UserAlias|maximum login length exceeded).?
+__suffix_failed_login = ([uU]ser not authorized for login|[nN]o such user found|[iI]ncorrect password|[pP]assword expired|[aA]ccount disabled|[iI]nvalid shell: '\S+'|[uU]ser in \S+|[lL]imit (access|configuration) denies login|[nN]ot a UserAlias|[mM]aximum login length exceeded)
 
 
-prefregex = ^%(__prefix_line)s%(__hostname)s \(\S+\[<HOST>\]\)[: -]+ <F-CONTENT>(?:USER|SECURITY|Maximum).+</F-CONTENT>$
+prefregex = ^%(__prefix_line)s%(__hostname)s \(\S+\[<HOST>\]\)[: -]+ <F-CONTENT>(?:USER|SECURITY|Maximum) .+</F-CONTENT>$
 
 
-failregex = ^USER .*: no such user found from \S+ \[\S+\] to \S+:\S+ *$
-            ^USER .* \(Login failed\): %(__suffix_failed_login)s\s*$
-            ^SECURITY VIOLATION: .* login attempted\. *$
-            ^Maximum login attempts \(\d+\) exceeded *$
+failregex = ^USER <F-USER>\S+|.*?</F-USER>(?: \(Login failed\))?: %(__suffix_failed_login)s
+            ^SECURITY VIOLATION: <F-USER>\S+|.*?</F-USER> login attempted
+            ^Maximum login attempts \(\d+\) exceeded
 
 ignoreregex = 
 

config/filter.d/roundcube-auth.conf

@@ -18,7 +18,7 @@ prefregex = ^\s*(\[\])?(%(__hostname)s\s*(?:roundcube(?:\[(\d*)\])?:)?\s*(<[\w]+
 failregex = ^(?:FAILED login|Login failed) for <F-USER>.*</F-USER> from <HOST>(?:(?:\([^\)]*\))?\. (?:(?! from ).)*(?: user=(?P=user))? in \S+\.php on line \d+ \(\S+ \S+\))?$
             ^(?:<[\w]+> )?Failed login for <F-USER>.*</F-USER> from <HOST> in session \w+( \(error: \d\))?$
 
-ignoreregex = Could not connect to .* Connection refused
+ignoreregex = 
 
 journalmatch = SYSLOG_IDENTIFIER=roundcube
 

config/filter.d/sendmail-auth.conf

@@ -8,11 +8,14 @@ before = common.conf
 [Definition]
 
 _daemon = (?:sendmail|sm-(?:mta|acceptingconnections))
+# "\w{14,20}" will give support for IDs from 14 up to 20 characters long
 __prefix_line = %(known/__prefix_line)s(?:\w{14,20}: )?
+addr = (?:IPv6:<IP6>|<IP4>)
 
-# "w{14,20}" will give support for IDs from 14 up to 20 characters long
-failregex = ^%(__prefix_line)s(\S+ )?\[(?:IPv6:<IP6>|<IP4>)\]( \(may be forged\))?: possible SMTP attack: command=AUTH, count=\d+$
+prefregex = ^<F-MLFID>%(__prefix_line)s</F-MLFID><F-CONTENT>.+</F-CONTENT>$
 
+failregex = ^(\S+ )?\[%(addr)s\]( \(may be forged\))?: possible SMTP attack: command=AUTH, count=\d+$
+            ^AUTH failure \([^\)]+\):(?: [^:]+:)? (?:authentication failure|user not found): [^,]*, user=<F-USER>(?:\S+|.*?)</F-USER>, relay=(?:\S+ )?\[%(addr)s\](?: \(may be forged\))?$
 ignoreregex =
 
 journalmatch = _SYSTEMD_UNIT=sendmail.service

config/filter.d/sendmail-reject.conf

@@ -21,19 +21,20 @@ before = common.conf
 
 _daemon = (?:(sm-(mta|acceptingconnections)|sendmail))
 __prefix_line = %(known/__prefix_line)s(?:\w{14,20}: )?
+addr = (?:(?:IPv6:)?<IP6>|<IP4>)
 
 prefregex = ^<F-MLFID>%(__prefix_line)s</F-MLFID><F-CONTENT>.+</F-CONTENT>$
 
-cmnfailre = ^ruleset=check_rcpt, arg1=(?P<email><\S+@\S+>), relay=(\S+ )?\[(?:IPv6:<IP6>|<IP4>)\](?: \(may be forged\))?, reject=(550 5\.7\.1 (?P=email)\.\.\. Relaying denied\. (IP name possibly forged \[(\d+\.){3}\d+\]|Proper authentication required\.|IP name lookup failed \[(\d+\.){3}\d+\])|553 5\.1\.8 (?P=email)\.\.\. Domain of sender address \S+ does not exist|550 5\.[71]\.1 (?P=email)\.\.\. (Rejected: .*|User unknown))$
-            ^ruleset=check_relay, arg1=(?P<dom>\S+), arg2=(?:IPv6:<IP6>|<IP4>), relay=((?P=dom) )?\[(\d+\.){3}\d+\](?: \(may be forged\))?, reject=421 4\.3\.2 (Connection rate limit exceeded\.|Too many open connections\.)$
-            ^rejecting commands from (\S* )?\[(?:IPv6:<IP6>|<IP4>)\] due to pre-greeting traffic after \d+ seconds$
-            ^(?:\S+ )?\[(?:IPv6:<IP6>|<IP4>)\]: (?:(?i)expn|vrfy) \S+ \[rejected\]$
+cmnfailre = ^ruleset=check_rcpt, arg1=(?P<email><\S+@\S+>), relay=(\S+ )?\[%(addr)s\](?: \(may be forged\))?, reject=(?:550 5\.7\.1(?: (?P=email)\.\.\.)?(?: Relaying denied\.)? (?:IP name possibly forged \[(\d+\.){3}\d+\]|Proper authentication required\.|IP name lookup failed \[(\d+\.){3}\d+\]|Fix reverse DNS for \S+)|553 5\.1\.8(?: (?P=email)\.\.\.)? Domain of sender address \S+ does not exist|550 5\.[71]\.1 (?P=email)\.\.\. (Rejected: .*|User unknown))$
+            ^ruleset=check_relay(?:, arg\d+=\S*)*, relay=(\S+ )?\[%(addr)s\](?: \(may be forged\))?, reject=421 4\.3\.2 (Connection rate limit exceeded\.|Too many open connections\.)$
+            ^rejecting commands from (\S* )?\[%(addr)s\] due to pre-greeting traffic after \d+ seconds$
+            ^(?:\S+ )?\[%(addr)s\]: (?:(?i)expn|vrfy) \S+ \[rejected\]$
             ^<[^@]+@[^>]+>\.\.\. No such user here$
-            ^<F-NOFAIL>from=<[^@]+@[^>]+></F-NOFAIL>, size=\d+, class=\d+, nrcpts=\d+, bodytype=\w+, proto=E?SMTP, daemon=MTA, relay=\S+ \[(?:IPv6:<IP6>|<IP4>)\]$
+            ^<F-NOFAIL>from=<[^@]+@[^>]+></F-NOFAIL>, size=\d+, class=\d+, nrcpts=\d+, bodytype=\w+, proto=E?SMTP, daemon=MTA, relay=\S+ \[%(addr)s\]$
 
 mdre-normal =
 
-mdre-extra = ^(?:\S+ )?\[(?:IPv6:<IP6>|<IP4>)\](?: \(may be forged\))? did not issue (?:[A-Z]{4}[/ ]?)+during connection to (?:TLS)?M(?:TA|S[PA])(?:-\w+)?$
+mdre-extra = ^(?:\S+ )?\[%(addr)s\](?: \(may be forged\))? did not issue \S+ during connection
 
 mdre-aggressive = %(mdre-extra)s
 

config/filter.d/softethervpn.conf

@@ -0,0 +1,9 @@
+# Fail2Ban filter for SoftEtherVPN
+# Detecting unauthorized access to SoftEtherVPN
+# typically logged in /usr/local/vpnserver/security_log/*/sec.log, or in syslog, depending on configuration
+
+[INCLUDES]
+before = common.conf
+
+[Definition]
+failregex = ^%(__prefix_line)s(?:(?:\([\d\-]+ [\d:.]+\) )?<SECURITY_LOG>: )?Connection "[^"]+": User authentication failed. The user name that has been provided was "<F-USER>(?:[^"]+|.+)</F-USER>", from <ADDR>\.$

config/filter.d/sshd.conf

@@ -25,7 +25,7 @@ __pref = (?:(?:error|fatal): (?:PAM: )?)?
 __suff = (?: (?:port \d+|on \S+|\[preauth\])){0,3}\s*
 __on_port_opt = (?: (?:port \d+|on \S+)){0,2}
 # close by authenticating user:
-__authng_user = (?: (?:invalid|authenticating) user <F-USER>\S+|.+?</F-USER>)?
+__authng_user = (?: (?:invalid|authenticating) user <F-USER>\S+|.*?</F-USER>)?
 
 # for all possible (also future) forms of "no matching (cipher|mac|MAC|compression method|key exchange method|host key type) found",
 # see ssherr.c for all possible SSH_ERR_..._ALG_MATCH errors.
@@ -40,39 +40,45 @@ prefregex = ^<F-MLFID>%(__prefix_line)s</F-MLFID>%(__pref)s<F-CONTENT>.+</F-CONT
 
 cmnfailre = ^[aA]uthentication (?:failure|error|failed) for <F-USER>.*</F-USER> from <HOST>( via \S+)?%(__suff)s$
             ^User not known to the underlying authentication module for <F-USER>.*</F-USER> from <HOST>%(__suff)s$
-            ^Failed publickey for invalid user <F-USER>(?P<cond_user>\S+)|(?:(?! from ).)*?</F-USER> from <HOST>%(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$)
-            ^Failed \b(?!publickey)\S+ for (?P<cond_inv>invalid user )?<F-USER>(?P<cond_user>\S+)|(?(cond_inv)(?:(?! from ).)*?|[^:]+)</F-USER> from <HOST>%(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$)
+            <cmnfailre-failed-pub-<publickey>>
+            ^Failed <cmnfailed> for (?P<cond_inv>invalid user )?<F-USER>(?P<cond_user>\S+)|(?(cond_inv)(?:(?! from ).)*?|[^:]+)</F-USER> from <HOST>%(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$)
             ^<F-USER>ROOT</F-USER> LOGIN REFUSED FROM <HOST>
             ^[iI](?:llegal|nvalid) user <F-USER>.*?</F-USER> from <HOST>%(__suff)s$
-            ^User <F-USER>.+</F-USER> from <HOST> not allowed because not listed in AllowUsers%(__suff)s$
-            ^User <F-USER>.+</F-USER> from <HOST> not allowed because listed in DenyUsers%(__suff)s$
-            ^User <F-USER>.+</F-USER> from <HOST> not allowed because not in any group%(__suff)s$
+            ^User <F-USER>\S+|.*?</F-USER> from <HOST> not allowed because not listed in AllowUsers%(__suff)s$
+            ^User <F-USER>\S+|.*?</F-USER> from <HOST> not allowed because listed in DenyUsers%(__suff)s$
+            ^User <F-USER>\S+|.*?</F-USER> from <HOST> not allowed because not in any group%(__suff)s$
             ^refused connect from \S+ \(<HOST>\)
             ^Received <F-MLFFORGET>disconnect</F-MLFFORGET> from <HOST>%(__on_port_opt)s:\s*3: .*: Auth fail%(__suff)s$
-            ^User <F-USER>.+</F-USER> from <HOST> not allowed because a group is listed in DenyGroups%(__suff)s$
-            ^User <F-USER>.+</F-USER> from <HOST> not allowed because none of user's groups are listed in AllowGroups%(__suff)s$
+            ^User <F-USER>\S+|.*?</F-USER> from <HOST> not allowed because a group is listed in DenyGroups%(__suff)s$
+            ^User <F-USER>\S+|.*?</F-USER> from <HOST> not allowed because none of user's groups are listed in AllowGroups%(__suff)s$
             ^<F-NOFAIL>%(__pam_auth)s\(sshd:auth\):\s+authentication failure;</F-NOFAIL>(?:\s+(?:(?:logname|e?uid|tty)=\S*)){0,4}\s+ruser=<F-ALT_USER>\S*</F-ALT_USER>\s+rhost=<HOST>(?:\s+user=<F-USER>\S*</F-USER>)?%(__suff)s$
-            ^(error: )?maximum authentication attempts exceeded for <F-USER>.*</F-USER> from <HOST>%(__on_port_opt)s(?: ssh\d*)?%(__suff)s$
-            ^User <F-USER>.+</F-USER> not allowed because account is locked%(__suff)s
+            ^maximum authentication attempts exceeded for <F-USER>.*</F-USER> from <HOST>%(__on_port_opt)s(?: ssh\d*)?%(__suff)s$
+            ^User <F-USER>\S+|.*?</F-USER> not allowed because account is locked%(__suff)s
             ^<F-MLFFORGET>Disconnecting</F-MLFFORGET>(?: from)?(?: (?:invalid|authenticating)) user <F-USER>\S+</F-USER> <HOST>%(__on_port_opt)s:\s*Change of username or service not allowed:\s*.*\[preauth\]\s*$
-            ^<F-MLFFORGET>Disconnecting</F-MLFFORGET>: Too many authentication failures(?: for <F-USER>.+?</F-USER>)?%(__suff)s$
+            ^Disconnecting: Too many authentication failures(?: for <F-USER>\S+|.*?</F-USER>)?%(__suff)s$
             ^<F-NOFAIL>Received <F-MLFFORGET>disconnect</F-MLFFORGET></F-NOFAIL> from <HOST>%(__on_port_opt)s:\s*11:
             <mdre-<mode>-other>
             ^<F-MLFFORGET><F-MLFGAINED>Accepted \w+</F-MLFGAINED></F-MLFFORGET> for <F-USER>\S+</F-USER> from <HOST>(?:\s|$)
 
+cmnfailed-any = \S+
+cmnfailed-ignore = \b(?!publickey)\S+
+cmnfailed-invalid = <cmnfailed-ignore>
+cmnfailed-nofail = (?:<F-NOFAIL>publickey</F-NOFAIL>|\S+)
+cmnfailed = <cmnfailed-<publickey>>
+
 mdre-normal =
 # used to differentiate "connection closed" with and without `[preauth]` (fail/nofail cases in ddos mode)
 mdre-normal-other = ^<F-NOFAIL><F-MLFFORGET>(Connection closed|Disconnected)</F-MLFFORGET></F-NOFAIL> (?:by|from)%(__authng_user)s <HOST>(?:%(__suff)s|\s*)$
 
 mdre-ddos = ^Did not receive identification string from <HOST>
+            ^kex_exchange_identification: (?:[Cc]lient sent invalid protocol identifier|[Cc]onnection closed by remote host)
             ^Bad protocol version identification '.*' from <HOST>
-            ^Connection <F-MLFFORGET>reset</F-MLFFORGET> by <HOST>
             ^<F-NOFAIL>SSH: Server;Ltype:</F-NOFAIL> (?:Authname|Version|Kex);Remote: <HOST>-\d+;[A-Z]\w+:
             ^Read from socket failed: Connection <F-MLFFORGET>reset</F-MLFFORGET> by peer
 # same as mdre-normal-other, but as failure (without <F-NOFAIL>) and [preauth] only:
-mdre-ddos-other = ^<F-MLFFORGET>(Connection closed|Disconnected)</F-MLFFORGET> (?:by|from)%(__authng_user)s <HOST>%(__on_port_opt)s\s+\[preauth\]\s*$
+mdre-ddos-other = ^<F-MLFFORGET>(Connection (?:closed|reset)|Disconnected)</F-MLFFORGET> (?:by|from)%(__authng_user)s <HOST>%(__on_port_opt)s\s+\[preauth\]\s*$
 
-mdre-extra = ^Received <F-MLFFORGET>disconnect</F-MLFFORGET> from <HOST>%(__on_port_opt)s:\s*14: No supported authentication methods available
+mdre-extra = ^Received <F-MLFFORGET>disconnect</F-MLFFORGET> from <HOST>%(__on_port_opt)s:\s*14: No(?: supported)? authentication methods available
             ^Unable to negotiate with <HOST>%(__on_port_opt)s: no matching <__alg_match> found.
             ^Unable to negotiate a <__alg_match>
             ^no matching <__alg_match> found:
@@ -84,6 +90,17 @@ mdre-aggressive = %(mdre-ddos)s
 # mdre-extra-other is fully included within mdre-ddos-other:
 mdre-aggressive-other = %(mdre-ddos-other)s
 
+# Parameter "publickey": nofail (default), invalid, any, ignore
+publickey = nofail
+# consider failed publickey for invalid users only:
+cmnfailre-failed-pub-invalid = ^Failed publickey for invalid user <F-USER>(?P<cond_user>\S+)|(?:(?! from ).)*?</F-USER> from <HOST>%(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$)
+# consider failed publickey for valid users too (don't need RE, see cmnfailed):
+cmnfailre-failed-pub-any =
+# same as invalid, but consider failed publickey for valid users too, just as no failure (helper to get IP and user-name only, see cmnfailed):
+cmnfailre-failed-pub-nofail = <cmnfailre-failed-pub-invalid>
+# don't consider failed publickey as failures (don't need RE, see cmnfailed):
+cmnfailre-failed-pub-ignore =
+
 cfooterre = ^<F-NOFAIL>Connection from</F-NOFAIL> <HOST>
 
 failregex = %(cmnfailre)s

config/filter.d/traefik-auth.conf

@@ -51,6 +51,26 @@
 
 [Definition]
 
-failregex = ^<HOST> \- (?!- )\S+ \[\] \"(GET|POST|HEAD) [^\"]+\" 401\b
+# Parameter "method" can be used to specifiy request method
+req-method = \S+
+# Usage example (for jail.local):
+#   filter = traefik-auth[req-method="GET|POST|HEAD"]
+
+failregex = ^<HOST> \- <usrre-<mode>> \[\] \"(?:<req-method>) [^\"]+\" 401\b
 
 ignoreregex =
+
+# Parameter "mode": normal (default), ddos or aggressive
+# Usage example (for jail.local):
+#   [traefik-auth]
+#   mode = aggressive
+#   # or another jail (rewrite filter parameters of jail):
+#   [traefik-auth-ddos]
+#   filter = traefik-auth[mode=ddos]
+#
+mode = normal
+
+# part of failregex matches user name (must be available in normal mode, must be empty in ddos mode, and both for aggressive mode):
+usrre-normal = (?!- )<F-USER>\S+</F-USER>
+usrre-ddos = -
+usrre-aggressive = <F-USER>\S+</F-USER>

config/jail.conf

@@ -44,44 +44,6 @@ before = paths-debian.conf
 # MISCELLANEOUS OPTIONS
 #
 
-# "bantime.increment" allows to use database for searching of previously banned ip's to increase a 
-# default ban time using special formula, default it is banTime * 1, 2, 4, 8, 16, 32...
-#bantime.increment = true
-
-# "bantime.rndtime" is the max number of seconds using for mixing with random time 
-# to prevent "clever" botnets calculate exact time IP can be unbanned again:
-#bantime.rndtime = 
-
-# "bantime.maxtime" is the max number of seconds using the ban time can reach (don't grows further)
-#bantime.maxtime = 
-
-# "bantime.factor" is a coefficient to calculate exponent growing of the formula or common multiplier,
-# default value of factor is 1 and with default value of formula, the ban time 
-# grows by 1, 2, 4, 8, 16 ...
-#bantime.factor = 1
-
-# "bantime.formula" used by default to calculate next value of ban time, default value bellow,
-# the same ban time growing will be reached by multipliers 1, 2, 4, 8, 16, 32...
-#bantime.formula = ban.Time * (1<<(ban.Count if ban.Count<20 else 20)) * banFactor
-#
-# more aggressive example of formula has the same values only for factor "2.0 / 2.885385" :
-#bantime.formula = ban.Time * math.exp(float(ban.Count+1)*banFactor)/math.exp(1*banFactor)
-
-# "bantime.multipliers" used to calculate next value of ban time instead of formula, coresponding 
-# previously ban count and given "bantime.factor" (for multipliers default is 1);
-# following example grows ban time by 1, 2, 4, 8, 16 ... and if last ban count greater as multipliers count, 
-# always used last multiplier (64 in example), for factor '1' and original ban time 600 - 10.6 hours
-#bantime.multipliers = 1 2 4 8 16 32 64
-# following example can be used for small initial ban time (bantime=60) - it grows more aggressive at begin,
-# for bantime=60 the multipliers are minutes and equal: 1 min, 5 min, 30 min, 1 hour, 5 hour, 12 hour, 1 day, 2 day
-#bantime.multipliers = 1 5 30 60 300 720 1440 2880
-
-# "bantime.overalljails" (if true) specifies the search of IP in the database will be executed 
-# cross over all jails, if false (dafault), only current jail of the ban IP will be searched
-#bantime.overalljails = false
-
-# --------------------
-
 # "ignoreself" specifies whether the local resp. own IP addresses should be ignored
 # (default is true). Fail2ban will not ban a host which matches such addresses.
 #ignoreself = true
@@ -209,28 +171,28 @@ banaction = iptables-multiport
 banaction_allports = iptables-allports
 
 # The simplest action to take: ban only
-action_ = %(banaction)s[name=%(__name__)s, port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"]
+action_ = %(banaction)s[bantime="%(bantime)s", port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"]
 
 # ban & send an e-mail with whois report to the destemail.
-action_mw = %(banaction)s[name=%(__name__)s, port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"]
-            %(mta)s-whois[name=%(__name__)s, sender="%(sender)s", dest="%(destemail)s", protocol="%(protocol)s", chain="%(chain)s"]
+action_mw = %(action_)s
+            %(mta)s-whois[sender="%(sender)s", dest="%(destemail)s", protocol="%(protocol)s", chain="%(chain)s"]
 
 # ban & send an e-mail with whois report and relevant log lines
 # to the destemail.
-action_mwl = %(banaction)s[name=%(__name__)s, port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"]
-             %(mta)s-whois-lines[name=%(__name__)s, sender="%(sender)s", dest="%(destemail)s", logpath="%(logpath)s", chain="%(chain)s"]
+action_mwl = %(action_)s
+             %(mta)s-whois-lines[sender="%(sender)s", dest="%(destemail)s", logpath="%(logpath)s", chain="%(chain)s"]
 
 # See the IMPORTANT note in action.d/xarf-login-attack for when to use this action
 #
 # ban & send a xarf e-mail to abuse contact of IP address and include relevant log lines
 # to the destemail.
-action_xarf = %(banaction)s[name=%(__name__)s, port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"]
+action_xarf = %(action_)s
              xarf-login-attack[service=%(__name__)s, sender="%(sender)s", logpath="%(logpath)s", port="%(port)s"]
 
 # ban IP on CloudFlare & send an e-mail with whois report and relevant log lines
 # to the destemail.
 action_cf_mwl = cloudflare[cfuser="%(cfemail)s", cftoken="%(cfapikey)s"]
-                %(mta)s-whois-lines[name=%(__name__)s, sender="%(sender)s", dest="%(destemail)s", logpath="%(logpath)s", chain="%(chain)s"]
+                %(mta)s-whois-lines[sender="%(sender)s", dest="%(destemail)s", logpath="%(logpath)s", chain="%(chain)s"]
 
 # Report block via blocklist.de fail2ban reporting service API
 # 
@@ -240,21 +202,7 @@ action_cf_mwl = cloudflare[cfuser="%(cfemail)s", cftoken="%(cfapikey)s"]
 # in your `jail.local` globally (section [DEFAULT]) or per specific jail section (resp. in 
 # corresponding jail.d/my-jail.local file).
 #
-action_blocklist_de  = blocklist_de[email="%(sender)s", service=%(filter)s, apikey="%(blocklist_de_apikey)s", agent="%(fail2ban_agent)s"]
-
-# Report ban via badips.com, and use as blacklist
-#
-# See BadIPsAction docstring in config/action.d/badips.py for
-# documentation for this action.
-#
-# NOTE: This action relies on banaction being present on start and therefore
-# should be last action defined for a jail.
-#
-action_badips = badips.py[category="%(__name__)s", banaction="%(banaction)s", agent="%(fail2ban_agent)s"]
-#
-# Report ban via badips.com (uses action.d/badips.conf for reporting only)
-#
-action_badips_report = badips[category="%(__name__)s", agent="%(fail2ban_agent)s"]
+action_blocklist_de  = blocklist_de[email="%(sender)s", service="%(__name__)s", apikey="%(blocklist_de_apikey)s", agent="%(fail2ban_agent)s"]
 
 # Report ban via abuseipdb.com.
 #
@@ -371,7 +319,7 @@ maxretry = 1
 [openhab-auth]
 
 filter = openhab
-action = iptables-allports[name=NoAuthFailures]
+banaction = %(banaction_allports)s
 logpath = /opt/openhab/logs/request.log
 
 
@@ -478,6 +426,7 @@ backend  = %(syslog_backend)s
 
 port     = http,https
 logpath  = /var/log/tomcat*/catalina.out
+#logpath  = /var/log/guacamole.log
 
 [monit]
 #Ban clients brute-forcing the monit gui login
@@ -744,8 +693,8 @@ logpath  = /var/log/named/security.log
 [nsd]
 
 port     = 53
-action   = %(banaction)s[name=%(__name__)s-tcp, port="%(port)s", protocol="tcp", chain="%(chain)s", actname=%(banaction)s-tcp]
-           %(banaction)s[name=%(__name__)s-udp, port="%(port)s", protocol="udp", chain="%(chain)s", actname=%(banaction)s-udp]
+action_  = %(default/action_)s[name=%(__name__)s-tcp, protocol="tcp"]
+           %(default/action_)s[name=%(__name__)s-udp, protocol="udp"]
 logpath = /var/log/nsd.log
 
 
@@ -756,9 +705,8 @@ logpath = /var/log/nsd.log
 [asterisk]
 
 port     = 5060,5061
-action   = %(banaction)s[name=%(__name__)s-tcp, port="%(port)s", protocol="tcp", chain="%(chain)s", actname=%(banaction)s-tcp]
-           %(banaction)s[name=%(__name__)s-udp, port="%(port)s", protocol="udp", chain="%(chain)s", actname=%(banaction)s-udp]
-           %(mta)s-whois[name=%(__name__)s, dest="%(destemail)s"]
+action_  = %(default/action_)s[name=%(__name__)s-tcp, protocol="tcp"]
+           %(default/action_)s[name=%(__name__)s-udp, protocol="udp"]
 logpath  = /var/log/asterisk/messages
 maxretry = 10
 
@@ -766,9 +714,8 @@ maxretry = 10
 [freeswitch]
 
 port     = 5060,5061
-action   = %(banaction)s[name=%(__name__)s-tcp, port="%(port)s", protocol="tcp", chain="%(chain)s", actname=%(banaction)s-tcp]
-           %(banaction)s[name=%(__name__)s-udp, port="%(port)s", protocol="udp", chain="%(chain)s", actname=%(banaction)s-udp]
-           %(mta)s-whois[name=%(__name__)s, dest="%(destemail)s"]
+action_  = %(default/action_)s[name=%(__name__)s-tcp, protocol="tcp"]
+           %(default/action_)s[name=%(__name__)s-udp, protocol="udp"]
 logpath  = /var/log/freeswitch.log
 maxretry = 10
 
@@ -853,11 +800,23 @@ logpath = /var/log/ejabberd/ejabberd.log
 [counter-strike]
 
 logpath = /opt/cstrike/logs/L[0-9]*.log
-# Firewall: http://www.cstrike-planet.com/faq/6
 tcpport = 27030,27031,27032,27033,27034,27035,27036,27037,27038,27039
 udpport = 1200,27000,27001,27002,27003,27004,27005,27006,27007,27008,27009,27010,27011,27012,27013,27014,27015
-action  = %(banaction)s[name=%(__name__)s-tcp, port="%(tcpport)s", protocol="tcp", chain="%(chain)s", actname=%(banaction)s-tcp]
-           %(banaction)s[name=%(__name__)s-udp, port="%(udpport)s", protocol="udp", chain="%(chain)s", actname=%(banaction)s-udp]
+action_  = %(default/action_)s[name=%(__name__)s-tcp, port="%(tcpport)s", protocol="tcp"]
+           %(default/action_)s[name=%(__name__)s-udp, port="%(udpport)s", protocol="udp"]
+
+[softethervpn]
+port     = 500,4500
+protocol = udp
+logpath  = /usr/local/vpnserver/security_log/*/sec.log
+
+[gitlab]
+port    = http,https
+logpath = /var/log/gitlab/gitlab-rails/application.log
+
+[grafana]
+port    = http,https
+logpath = /var/log/grafana/grafana.log
 
 [bitwarden]
 port    = http,https
@@ -909,8 +868,8 @@ findtime     = 1
 [murmur]
 # AKA mumble-server
 port     = 64738
-action   = %(banaction)s[name=%(__name__)s-tcp, port="%(port)s", protocol=tcp, chain="%(chain)s", actname=%(banaction)s-tcp]
-           %(banaction)s[name=%(__name__)s-udp, port="%(port)s", protocol=udp, chain="%(chain)s", actname=%(banaction)s-udp]
+action_  = %(default/action_)s[name=%(__name__)s-tcp, protocol="tcp"]
+           %(default/action_)s[name=%(__name__)s-udp, protocol="udp"]
 logpath  = /var/log/mumble-server/mumble-server.log
 
 

debian/NEWS

@@ -1,91 +0,0 @@
-fail2ban (0.10.2-1) unstable; urgency=medium
-
-  This version is a major development leap forward to provide
-  IPv6 support, which also required extensions to the configuration
-  system.  That is why it is not unlikely that configuration left from the
-  previous version(s) would either not work or would not work as intended.
-
-  You are advised to accept new configuration and adjust it for your
-  customizations (if any).  See changelog.Debian.gz for more information.
-
- -- Yaroslav Halchenko <debian@onerussian.com>  Sun, 21 Jan 2018 22:25:26 -0500
-
-fail2ban (0.9.0+git48-gabcab00-1) experimental; urgency=low
-
-  [ Yaroslav Halchenko ]
-  * This version went through big refactoring which allowed to gain new
-    features such as multiline matching (see upstream's changelog for more
-    information).
-  * Although .local files are still supported, customizations are advised
-    to be provided under corresponding .d/ directories.  E.g. see
-    /etc/fail2ban/jail.d/defaults-debian.conf which is where now sshd
-    jail is enabled by default to match previous behavior of Fail2Ban in
-    Debian.
-
-  [ Daniel Schaal ]
-  * All jails definitions were rewritten to become more concise and uniform.
-    From this version on log paths are defined in distro specific files,
-    for Debian this is in /etc/fail2ban/paths-debian.conf.
-
- -- Yaroslav Halchenko <debian@onerussian.com>  Tue, 25 Mar 2014 08:38:31 -0400
-
-fail2ban (0.8.11-1) unstable; urgency=low
-
-  * retroactive for 0.8.9: by default iptables-* actions do not simply
-    DROP packets from offending IP but rather reject with
-    icmp-port-unreachable.  If DROP behaviour is preferable, provide
-    config/action.d/iptables-blocktype.local with [Init] section defining
-    blocktype = DROP or override action definition to provide
-    blocktype=DROP option in jail.local
-  * Many failregex's were tight-up in this release which could
-    theoretically effect operation in comparison to previous release(s).
-
- -- Yaroslav Halchenko <debian@onerussian.com>  Sat, 16 Nov 2013 22:27:50 -0500
-
-fail2ban (0.8.4-3) unstable; urgency=low
-
-  * Jail named-refused-udp is unsafe and opens possibility for easy DoS,
-    thus discouraged to be used, and commented out (see #583364 for more
-    information).
-
- -- Yaroslav Halchenko <debian@onerussian.com>  Mon, 28 Jun 2010 22:12:22 -0400
-
-fail2ban (0.7.1-0.2) unstable; urgency=low
-
-  fail2ban 0.7 is a complete rewrite of the 0.6 version, and if you
-  customized any of provided configuration or startup files
-  (/etc/default/fail2ban, /etc/fail2ban.conf, /etc/init.d/fail2ban),
-  please read further. The configuration scheme has changed upstream:
-  0.7 ignores /etc/fail2ban.conf and instead uses a split configuration
-  under /etc/fail2ban/. To retain your customizations, for example to
-  monitor anything other than sshd, you will need to set them under that
-  new directory; use *.local files for customizations. Please see
-  /usr/share/doc/fail2ban/README.Debian.gz and
-  http://fail2ban.sourceforge.net for further description of new
-  configuration scheme. Detailed documentation is under development (see
-  #400416).  When you are satisfied with the new settings, please delete
-  /etc/fail2ban.conf to avoid confusion.
-
-  Fail2ban 0.7 uses client/server architecture and fail2ban-client is to
-  substitute fail2ban command to provide an interface between the user and
-  fail2ban-server. That is why some command line parameters present in
-  fail2ban 0.6 are invalid in fail2ban-client. Such change affects
-  /etc/default/fail2ban; you should review that file if you customized it.
-  Please enable sections as directed in README.Debian.gz mentioned above.
-  You must use newly shipped init.d/fail2ban, or otherwise fail2ban will
-  not start.
-
-  This note was rewritten in release 0.7.5-2 to clarify its meaning.
-
- -- Yaroslav Halchenko <debian@onerussian.com>  Sat,  9 Dec 2006 18:24:36 -0500
-
-fail2ban (0.6.0-4) unstable; urgency=low
-
-  In this version the new section ApacheAttacks was introduced to ban IPs
-  which are found to run some known attack on the host. For now it captures
-  just awstats and mambo related attacks. To make this feature work, the bug of
-  wrongly specified timeregexp for Apache's access.log file was fixed.
-  Besides that group of log files has changed to be adm, and now they are
-  readable by the group.
-
- -- Yaroslav Halchenko <debian@onerussian.com>  Fri, 10 Feb 2006 13:05:07 -0500

debian/README.Debian

@@ -1,251 +0,0 @@
-fail2ban (>=0.7.0) for Debian
------------------------------
-
-This package is ~99% identical to the upstream version. Few features
-could have been added but not yet propagated into upstream version and
-some modifications might be Debian-specific. Debian specific jail.conf
-file is shipped. Original upstream file is available from
-/usr/share/doc/fail2ban/examples/jail.conf
-
-Currently, the major difference with upstream: python libraries are
-placed under /usr/share/fail2ban instead of /usr/lib/fail2ban to
-comply with policy regarding architecture independent resources.
-
-fail2ban supports both nftables and iptables.
-
-Shorewall and startup sequence (#847728)
-----------------------------------------
-
-If you are using systemd, create a
-/etc/systemd/system/fail2ban.service.d/override.conf with contents:
-
-[Unit]
-Requires=shorewall.service
-After=shorewall.service
-
-go guarantee a proper sequence of startup/shutdown (shorewall should
-be started before fail2ban, and stopped after).  Similar settings
-could be adopted for other firewall solutions.
-
-
-Upgrade from 0.6 versions:
--------------------------
-
-* New Config Files Format:
-
-If you had introduced your own sections in /etc/fail2ban.conf, you
-would need manually to convert them into a new format. At minimum you
-need to create /etc/fail2ban/filter.d/NAME.local (leave .conf files
-for me and upstream please to avoid any conflicts -- introduce your
-changes in .local) with failregex in [Definition] section. And provide
-appropriate jail definition in /etc/fail2ban/jail.local
-
-
-* Enabled Sections:
-
-Only handling of ssh files is enabled by default. If you want to use
-fail2ban with apache, please enable apache section manually in
-/etc/fail2ban/jail.local by including next lines:
-
-[apache]
-enabled = true
-
-NOTE: -e command line parameter is non existent in 0.7.x
-
-
-* Interpolations vs actions/filters parameters:
-
-For details see #398739 or wait for a closure of #400416
-
-Every pair of .conf and then .local (if exists) files is read
-separately from any other configuration file, so interpolations cannot
-penetrate from jail.* into actions.d/*. To overcome this, it is
-necessary to create a PARAMETER which can be substituted in actions
-[Definition] section, if it is also defined in the [Init] section of
-that file and is used in place of necessary allocation as <PARAMETER>
-tag. Parameters can be specified in the definitions within
-jail.{conf,local}. For instance, 1 lengthy example, where the same
-name "fwchain" is used both as interpolation (in jail.local) and as a
-parameter (in iptables-flex.local) (from #398739)
-
-==> /etc/fail2ban/jail.local <==
-[DEFAULT]
-action = iptables-flex[name=%(__name__)s, port=%(port)s, fwchain=%(fwchain)s, post_start_commands=%(post_start_commands)s, pre_end_commands=%(pre_end_commands)s]
-fwchain = INPUT
-[ssh]
-fwchain = ssh-tarpit
-==> /etc/fail2ban/action.d/iptables-flex.local <==
-[Definition]
-actionstart = iptables -N fail2ban-<name>
-               iptables -I <fwchain> -m state --state NEW -p <protocol> --dport <port> -j fail2ban-<name>
-               iptables -I <fwchain> -j <whitelist>
-actionstop  = iptables -D <fwchain> -j <whitelist>
-               iptables -D <fwchain> -m state --state NEW -p <protocol> --dport <port> -j fail2ban-<name>
-               iptables -F fail2ban-<name>
-               iptables -X fail2ban-<name>
-actioncheck = iptables -n -L <fwchain> | grep -q fail2ban-<name>
-actionban   = iptables -I fail2ban-<name> 1 -s <ip> -j DROP
-actionunban = iptables -D fail2ban-<name> -s <ip> -j DROP
-[Init]
-whitelist = ssh-whitelist
-fwchain = INPUT
-name = default
-port = ssh
-protocol = tcp
-
-
-* Multiport banning: Comment for #373592, #545971
-
-iptables-multiport action is now default banaction (file jail.conf, to
-be customized within jail.local). Therefore assure that you have built
-multiport module if you use custom kernel.
-
-If you would like to ban all ports for that host, just redefine
-fwban/fwunban commands to don't have --dport %(port)s statement at
-all, or use shorewall, where actionban bans whole IP.
-
-* Blocking of NEW connections only
-Comment for the wishlist #350746.
-
-It might be benefitial in some cases to ban only new connections. For
-that just use iptables-new action instead of default banaction
-
-/etc/fail2ban/jail.local:
-
-[DEFAULT]
-banaction=iptables-new
-
-(you can override banaction within interesting for you section).
- Also you can redefine the whole action parameter if you like.
-
-
-* Interaction with ipmasq
-  Comment to #461417
-
-Although fail2ban should detect and recreate missing chains if the external
-command wipes out iptables, it is better to explicitly to force-reload
-fail2ban. For this reason there is examples/ipmasq-ZZZzzz|fail2ban.rul file is
-shipped along to be installed under name ZZZzzz|fail2ban.rul within
-/etc/ipmasq.
-
-* Interaction with logrotate with custom logtarget
-  Comment to #631917
-
-if you use an alternative logtarget (e.g. SYSLOG) thus not using
-/var/log/fail2ban.log you should divert logrotate configuration into
-a disabled state, e.g.
-
-sudo dpkg-divert --rename --divert \
-	 /etc/logrotate.d/fail2ban.disabled /etc/logrotate.d/fail2ban
-
-
-Troubleshooting:
----------------
-
-* Updated failregex:
-
-To resolve the security bug #330827 [1] failregex expressions must
-provide a named group (?P<host>...) as a placeholder of the abuser's
-host. Alternative tag (since 0.7.5) can be "<HOST>". The naming of the
-group was introduced to capture possible future generalizations of
-failregex to provide even more information.
-
-[1] http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=330827
-
-You might benefit from using fail2ban-regex command shipped along to
-construct and debug your failregex statements.
-
-* "Interpolations" in the config file:
-
-Since version 0.6.0-3 to reduce duplication, thus to improve
-readability of the config file, interpolations provided by the module
-ConfigParser are used. If you had custom sections defined before, you
-might benefit from updating config file and adding appropriate
-information for the new sections.
-
-N.B. If you have some nice additional sections defined, I would really
-appreciate if you share them with me or upstream author, so they could
-be eventually included in the fail2ban package for general use by the
-rest of the community.
-
-
-* Mailing:
-
-Since actions.d/mail*.conf commands rely on presence of "mail"
-command, mailx package (or another package providing mailx
-functionality such as mailutils) is required if those actions are
-activated in jail.{conf,local}.
-
-
-* Dirty exit:
-
-If firewall rules gets cleaned out before fail2ban exits (like was
-happening with firestarter), errors get reported during the exit of
-fail2ban, but they are "safe" and can be ignored.
-
-
-** SSHD Configuration Specific Problems
-
-* Ban "Not allowed" attempts:
-
-Make sure that you have
-ChallengeResponseAuthentication no
-PasswordAuthentication yes
-
-Details from the bug report #350980 [2]
-
-[2]  http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=350980
-
-
-* Not caught attempts to login as root
-
-On the boxes running older versions of openssh (e.g. sarge
-distribution) in the case when PermitRootLogin is set to something
-else than "yes" and iff AllowUsers is active, failed root logins do
-not confirm to the standard logging message -- they omit the source
-IP, thus allowing attack to persist since such messages are not caught
-by fail2ban.
-
-
-* Bantime:
-
-An IP is banned for "bantime" not since the last failed login attempt
-from the IP, but rather since the moment when failed login was
-detected by fail2ban. Thus, if fail2ban gets [re]started, any IP which
-had enough of failed logins with durations less than "findtime" between
-them prior to the [re]start moment, will be banned for
-"bantime" since [re]start moment, not since the last failed login
-time.
-
-* Findtime:
-
-"Findtime" option of a jail actually defines a duration to reset the
-counter of failed login attempts, if no new attempt was detected within
-that time frame (i.e.  within "findtime").
-
-See
-http://www.fail2ban.org/wiki/index.php/MANUAL_0_8#Jail_Options
-for more information on jail options.
-
-
-* Syslog entries can be 'forged' by a regular user
-
-From
-http://www.fail2ban.org/wiki/index.php/MANUAL_0_8#Possibility_of_DOS_attack_by_a_local_user
-
-Especially on systems which provide ssh/CGI/PHP services to unknown
-users it is possible to block other users from ssh and probably other
-access as a unprivileged user may issue:
-
-logger -p auth.warning -t 'sshd[123]' 'Illegal user user1 from 1.2.3.4'
-
-N.B. chmod o-x /usr/bin/logger should provide at least obfuscation
-solution
-
-Or the malicious user may write via PHP's openlog()/syslog() to syslog.
-
-P.S. Anyone is welcome to recommend proper security solution to this
-issue, such as an alternative to sysklogd which allows better control
-over users logging to specific facilities (such as AUTH)
-
- -- Yaroslav Halchenko <debian@onerussian.com>, Mon, 22 Jan 2018 10:37:00 -0500

debian/TODO

@@ -1,10 +0,0 @@
-* completions installation
-
-W: fail2ban: package-installs-into-obsolete-dir etc/bash_completion.d/ : ^etc/bash_completion.d/ -> usr/share/bash-completion/completions (see also https://bugs.debian.org/776954)
-W: fail2ban: package-installs-into-obsolete-dir etc/bash_completion.d/fail2ban : ^etc/bash_completion.d/ -> usr/share/bash-completion/completions (see also https://bugs.debian.org/776954)
-
-* Find proper answer to "Syslog entries can be 'forged' by a regular
-   user" mentioned in README.Debian
-
- -- Yaroslav O. Halchenko <debian@onerussian.com>  Wed,  6 Dec 2006 22:14:26 -0500
-

debian/backports/00list.sarge-backports

@@ -1 +0,0 @@
-nopycentral.patch

debian/backports/nopycentral.patch

@@ -1,40 +0,0 @@
-diff -x '*~' -x .svn -Naur trunk/debian/control trunk.backports/debian/control
---- trunk/debian/control	2006-10-23 00:57:02.000000000 -0400
-+++ trunk.backports/debian/control	2006-12-04 08:45:25.000000000 -0500
-@@ -4,13 +4,13 @@
- Maintainer: Yaroslav Halchenko <debian@onerussian.com>
- Uploaders: Barak Pearlmutter <bap@debian.org>
- Build-Depends: debhelper (>= 5.0.37.2), dpatch
--Build-Depends-Indep: python, python-dev, help2man, python-central (>= 0.5.6)
-+Build-Depends-Indep: python, python2.4, python2.4-dev, help2man
- XS-Python-Version: current, >= 2.4
- Standards-Version: 3.7.2
- 
- Package: fail2ban
- Architecture: all
--Depends: ${python:Depends}, iptables, lsb-base (>=2.0-7)
-+Depends: python2.4, iptables, lsb-base (>=2.0-7)
- Suggests: python-gamin
- XB-Python-Version: ${python:Versions}
- Description: bans IPs that cause multiple authentication errors
-diff -x '*~' -x .svn -Naur trunk/debian/rules trunk.backports/debian/rules
---- trunk/debian/rules	2006-11-11 21:19:14.000000000 -0500
-+++ trunk.backports/debian/rules	2006-12-04 08:45:45.000000000 -0500
-@@ -39,7 +39,7 @@
- 	dh_installdirs
- 
- 	# Add here commands to install the package into debian/fail2ban.
--	python setup.py install --root=$(DESTDIR) --no-compile
-+	python2.4 setup.py install --root=$(DESTDIR) --no-compile
- 	#X Evil - must be removed after Debian switches over to 2.4, now
- 	#  distutils.setup will override the enterpreter line to /usr/bin/python
- 	install fail2ban-server fail2ban-client $(DESTDIR)/usr/bin
-@@ -62,7 +62,7 @@
- 	dh_installlogrotate
- 	dh_installinit -- defaults 99
- 	dh_installman man/*.1
--	dh_pycentral
-+	dh_python
- 	dh_link
- 	dh_compress
- 	dh_fixperms

debian/control

@@ -1,48 +0,0 @@
-Source: fail2ban
-Section: net
-Priority: optional
-Maintainer: Debian Python Team <team+python@tracker.debian.org>
-Uploaders: Yaroslav Halchenko <debian@onerussian.com>,
- Sylvestre Ledru <sylvestre@debian.org>
-Build-Depends:
- debhelper-compat (= 9)
- , debhelper (>= 9.20160709)
- , dh-python
- , python3
- , python3-setuptools
- , python3-pyinotify
- , sqlite3
- , 2to3
-Homepage: http://www.fail2ban.org
-Vcs-Git: https://salsa.debian.org/python-team/packages/fail2ban.git
-Vcs-Browser: https://salsa.debian.org/python-team/packages/fail2ban
-Standards-Version: 4.4.1
-
-
-Package: fail2ban
-Architecture: all
-Depends: ${python3:Depends}, ${misc:Depends}, lsb-base (>=2.0-7)
-Recommends: nftables | iptables, whois, python3-pyinotify, python3-systemd
-Suggests: mailx, system-log-daemon, monit, sqlite3
-Description: ban hosts that cause multiple authentication errors
- Fail2ban monitors log files (e.g. /var/log/auth.log,
- /var/log/apache/access.log) and temporarily or persistently bans
- failure-prone addresses by updating existing firewall rules.  Fail2ban
- allows easy specification of different actions to be taken such as to ban
- an IP using iptables or hostsdeny rules, or simply to send a notification
- email.
- .
- By default, it comes with filter expressions for various services
- (sshd, apache, proftpd, sasl, etc.) but configuration can be
- easily extended for monitoring any other text file.  All filters and
- actions are given in the config files, thus fail2ban can be adopted
- to be used with a variety of files and firewalls.  Following recommends
- are listed:
- .
-  - iptables/nftables -- default installation uses iptables for banning.
-    nftables is also supported. You most probably need it
-  - whois -- used by a number of *mail-whois* actions to send notification
-    emails with whois information about attacker hosts. Unless you will use
-    those you don't need whois
-  - python3-pyinotify -- unless you monitor services logs via systemd, you
-    need pyinotify for efficient monitoring for log files changes

debian/copyright

@@ -1,31 +0,0 @@
-This package was originally debianized by Yaroslav Halchenko
-<debian@onerussian.com> on Mon Jul  4 14:41:34 HST 2005
-
-It was downloaded from http://www.sourceforge.net/projects/fail2ban
-
-Author: Cyril Jaquier: <cyril.jaquier@fail2ban.org>
-        http://fail2ban.sourceforge.net
-
-Copyright: 2004-2009 Cyril Jaquier
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program; if not, write to the
-Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston,
-MA 02110-1301, USA.
-
-On Debian systems, the complete text of the GNU General Public
-License, version 2, can be found in /usr/share/common-licenses/GPL-2.
-
-The Debian packaging is (C) 2006-2011, Yaroslav Halchenko <debian@onerussian.com>
-and is licensed under the GPL, see above.
-

debian/debian-files/jail.d_defaults-debian.conf

@@ -1,2 +0,0 @@
-[sshd]
-enabled = true

debian/docs

@@ -1,3 +0,0 @@
-README.md
-TODO
-doc/run-rootless.txt

debian/fail2ban.default

@@ -1,39 +0,0 @@
-# This file is part of Fail2Ban.
-#
-# Fail2Ban is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# Fail2Ban is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Fail2Ban; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
-#
-# Author: Cyril Jaquier
-# 
-# $Revision$
-
-# Command line options for Fail2Ban. Refer to "fail2ban-client -h" for
-# valid options.
-FAIL2BAN_OPTS=""
-
-# Run fail2ban as a different user. If not set, fail2ban
-# will run as root.
-#
-# The user is not created automatically.
-# The user can be created e.g. with
-#    useradd --system --no-create-home --home-dir / --groups adm fail2ban
-# Log files are readable by group adm by default. Adding the fail2ban
-# user to this group allows it to read the logfiles.
-#
-# Another manual step that needs to be taken is to allow write access
-# for fail2ban user to fail2ban log files. The /etc/init.d/fail2ban
-# script will change the ownership when starting fail2ban. Logrotate
-# needs to be configured separately, see /etc/logrotate.d/fail2ban.
-#
-# FAIL2BAN_USER="fail2ban"

debian/fail2ban.logrotate

@@ -1,19 +0,0 @@
-/var/log/fail2ban.log {
-
-    weekly
-    rotate 4
-    compress
-    # Do not rotate if empty
-    notifempty
-
-    delaycompress
-    missingok
-    postrotate
-	fail2ban-client flushlogs 1>/dev/null
-    endscript
-
-    # If fail2ban runs as non-root it still needs to have write access
-    # to logfiles.
-    # create 640 fail2ban adm
-    create 640 root adm
-}

debian/gbp.conf

@@ -1,18 +0,0 @@
-[DEFAULT]
-# the default branch for upstream sources:
-upstream-branch = upstream
-# the default branch for the debian patch:
-debian-branch = debian-releases/experimental
-# use pristine-tar
-# pristine-tar = True
-# the default tag formats used:
-upstream-tag = %(version)s
-debian-tag = debian/%(version)s
-
-
-# Options only affecting git-buildpackage
-[git-buildpackage]
-# use this for more svn-buildpackage like behaviour:
-export-dir = ../build-area/
-tarball-dir = ../tarballs/
-

debian/patches/0002-ENH-verify-that-use_stock_cfg-was-not-provided-while.patch

@@ -1,25 +0,0 @@
-From d1afbb566f0304487b5d578b4aacef8e647ee74b Mon Sep 17 00:00:00 2001
-From: Yaroslav Halchenko <debian@onerussian.com>
-Date: Sun, 21 Jan 2018 20:16:43 -0500
-Subject: [PATCH 2/4] ENH: verify that use_stock_cfg was not provided while
- overriding it
-
-Just found this possibly confusing to outside programmer aspect
-so decided to make it more explicit
----
- fail2ban/tests/fail2banclienttestcase.py | 2 ++
- 1 file changed, 2 insertions(+)
-
-Index: fail2ban/fail2ban/tests/fail2banclienttestcase.py
-===================================================================
---- fail2ban.orig/fail2ban/tests/fail2banclienttestcase.py
-+++ fail2ban/fail2ban/tests/fail2banclienttestcase.py
-@@ -173,6 +173,8 @@ def _start_params(tmp, use_stock=False,
- 			"""Filters list of 'files' to contain only directories (under dir)"""
- 			return [f for f in files if isdir(pjoin(dir, f))]
- 		shutil.copytree(STOCK_CONF_DIR, cfg, ignore=ig_dirs)
-+		assert use_stock_cfg is None, \
-+			"We are about to overload use_stock_cfg from the one provided %s" % repr(use_stock_cfg)
- 		if use_stock_cfg is None: use_stock_cfg = ('action.d', 'filter.d')
- 		# replace fail2ban params (database with memory):
- 		r = re.compile(r'^dbfile\s*=')

debian/patches/deb_init_paths

@@ -1,13 +0,0 @@
-Index: fail2ban/files/debian-initd
-===================================================================
---- fail2ban.orig/files/debian-initd
-+++ fail2ban/files/debian-initd
-@@ -28,7 +28,7 @@ NAME="fail2ban"
- 
- # fail2ban-client is not a daemon itself but starts a daemon and
- # loads its with configuration
--DAEMON="/usr/local/bin/$NAME-client"
-+DAEMON="/usr/bin/$NAME-client"
- SCRIPTNAME="/etc/init.d/$NAME"
- 
- # Ad-hoc way to parse out socket file name

debian/patches/deb_manpages_reportbug

@@ -1,30 +0,0 @@
-From: Yaroslav Halchenko <debian@onerussian.com>
-Date: Fri, 8 Feb 2008 00:40:57 -0500
-Subject: tune ups in upstream manpages to direct users to use reportbug
-
-Index: fail2ban/man/fail2ban-client.1
-===================================================================
---- fail2ban.orig/man/fail2ban-client.1
-+++ fail2ban/man/fail2ban-client.1
-@@ -470,7 +470,7 @@ the action <ACT> for <JAIL>
- .SH FILES
- \fI/etc/fail2ban/*\fR
- .SH "REPORTING BUGS"
--Report bugs to https://github.com/fail2ban/fail2ban/issues
-+Report bugs via Debian bug tracking system \fIhttp://www.debian.org/Bugs/\fR .
- .SH "SEE ALSO"
- .br 
- fail2ban-server(1)
-Index: fail2ban/man/fail2ban-server.1
-===================================================================
---- fail2ban.orig/man/fail2ban-server.1
-+++ fail2ban/man/fail2ban-server.1
-@@ -69,7 +69,7 @@ display this help message
- \fB\-V\fR, \fB\-\-version\fR
- print the version (\fB\-V\fR returns machine\-readable short format)
- .SH "REPORTING BUGS"
--Report bugs to https://github.com/fail2ban/fail2ban/issues
-+Report bugs via Debian bug tracking system \fIhttp://www.debian.org/Bugs/\fR .
- .SH "SEE ALSO"
- .br 
- fail2ban-client(1)

debian/patches/deb_no_iptables_service

@@ -1,30 +0,0 @@
-From: Yaroslav Halchenko <debian@onerussian.com>
-Subject: Remove all non-provided .service's within PartOf of fail2ban.service
-
-  As reported and corroborated in the bug report, this causes inability
-  of firewalld to restart.
-  Correct solution would involve making systemd smarter and tune up
-  of involved .service files.
-  Since Debian ATM doesn't provide any of those ({ip{,6}tables,ipset}.service)
-  files, it should be safe and generic enough to just prune them from PartOf
-
-  Thanks  Joe Cooper <swelljoe@gmail.com> and Sunil Mohan Adapa <sunil@medhas.org>
-  for the reports and nagging ;)
-
-Origin: Fedora, Debian
-Bug-Debian: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=871993
-Last-Update: 2018-04-04
-
-Index: fail2ban/files/fail2ban.service.in
-===================================================================
---- fail2ban.orig/files/fail2ban.service.in
-+++ fail2ban/files/fail2ban.service.in
-@@ -2,7 +2,7 @@
- Description=Fail2Ban Service
- Documentation=man:fail2ban(1)
- After=network.target iptables.service firewalld.service ip6tables.service ipset.service nftables.service
--PartOf=iptables.service firewalld.service ip6tables.service ipset.service nftables.service
-+PartOf=firewalld.service
- 
- [Service]
- Type=simple

debian/patches/deb_path_to_common

@@ -1,22 +0,0 @@
---- a/fail2ban/tests/config/filter.d/zzz-generic-example.conf
-+++ b/fail2ban/tests/config/filter.d/zzz-generic-example.conf
-@@ -8,7 +8,7 @@
- # Read common prefixes. If any customizations available -- read them from
- # common.local.  common.conf is a symlink to the original common.conf and
- # should be copied (dereferenced) during installation
--before = ../../../../config/filter.d/common.conf
-+before = ../../../../../../../config/filter.d/common.conf
- 
- [Definition]
- 
---- a/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf
-+++ b/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf
-@@ -5,7 +5,7 @@
- 
- # Read common prefixes. If any customizations available -- read them from
- # common.local
--before = ../../../../config/filter.d/common.conf
-+before = ../../../../../../../config/filter.d/common.conf
- 
- [DEFAULT]
- 

debian/patches/no-python-user.diff

@@ -1,12 +0,0 @@
-Index: fail2ban/files/fail2ban.service.in
-===================================================================
---- fail2ban.orig/files/fail2ban.service.in
-+++ fail2ban/files/fail2ban.service.in
-@@ -15,6 +15,7 @@ ExecReload=@BINDIR@/fail2ban-client relo
- PIDFile=/run/fail2ban/fail2ban.pid
- Restart=on-failure
- RestartPreventExitStatus=0 255
-+Environment="PYTHONNOUSERSITE=yes"
- 
- [Install]
- WantedBy=multi-user.target

debian/patches/python3-test-suite.diff

@@ -1,10 +0,0 @@
-Index: fail2ban/bin/fail2ban-testcases
-===================================================================
---- fail2ban.orig/bin/fail2ban-testcases
-+++ fail2ban/bin/fail2ban-testcases
-@@ -1,4 +1,4 @@
--#!/usr/bin/env python
-+#!/usr/bin/env python3
- # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*-
- # vi: set ft=python sts=4 ts=4 sw=4 noet :
- """Script to run Fail2Ban tests battery

debian/patches/roundcude-update.diff

@@ -1,11 +0,0 @@
---- fail2ban-0.11.1.orig/config/filter.d/roundcube-auth.conf
-+++ fail2ban-0.11.1/config/filter.d/roundcube-auth.conf
-@@ -18,7 +18,7 @@ prefregex = ^\s*(\[\])?(%(__hostname)s\s
- failregex = ^(?:FAILED login|Login failed) for <F-USER>.*</F-USER> from <HOST>(?:(?:\([^\)]*\))?\. (?:(?! from ).)*(?: user=(?P=user))? in \S+\.php on line \d+ \(\S+ \S+\))?$
-             ^(?:<[\w]+> )?Failed login for <F-USER>.*</F-USER> from <HOST> in session \w+( \(error: \d\))?$
- 
--ignoreregex = 
-+ignoreregex = Could not connect to .* Connection refused
- 
- journalmatch = SYSLOG_IDENTIFIER=roundcube
- 

debian/patches/series

@@ -1,8 +0,0 @@
-deb_path_to_common
-deb_init_paths
-deb_manpages_reportbug
-0002-ENH-verify-that-use_stock_cfg-was-not-provided-while.patch
-deb_no_iptables_service
-python3-test-suite.diff
-no-python-user.diff
-roundcude-update.diff

debian/postinst

@@ -1,98 +0,0 @@
-#! /bin/sh
-# postinst script for fail2ban
-#
-# see: dh_installdeb(1)
-
-set -e
-
-# summary of how this script can be called:
-#        * <postinst> `configure' <most-recently-configured-version>
-#        * <old-postinst> `abort-upgrade' <new version>
-#        * <conflictor's-postinst> `abort-remove' `in-favour' <package>
-#          <new-version>
-#        * <deconfigured's-postinst> `abort-deconfigure' `in-favour'
-#          <failed-install-package> <version> `removing'
-#          <conflicting-package> <version>
-# for details, see http://www.debian.org/doc/debian-policy/ or
-# the debian-policy package
-#
-preversion=$2
-
-case "$1" in
-    configure)
-       # To fix the bug in generated by previous version files permissions
-       # also closes #352053
-
-       LOG=/var/log/fail2ban.log
-       touch $LOG
-       chown root:adm ${LOG}*
-       chmod 640 ${LOG}*
-        
-       # Note regarding changed configuration file
-       # Note regarding changed configuration file
-		if [ ! -z $preversion ]; then
-			if dpkg --compare-versions $preversion lt 0.7.1-1; then
-			cat <<EOF
-WARNING!
-  
-  Fail2ban 0.7 is a complete rewrite of the 0.6 version, and if you
-  customized any of provided configuration or startup files
-  (/etc/default/fail2ban, /etc/fail2ban.conf, /etc/init.d/fail2ban), please
-  read relevant entry in /usr/share/doc/fail2ban/NEWS.Debian.gz.
-  
-EOF
-			fi
-			if dpkg --compare-versions $preversion lt 0.5.4-5.14; then
-			cat <<EOF
-WARNING!
-
- Configuration file /etc/fail2ban.conf, failregex configuration
- parameter specifically, were changed in 0.5.4-5 to close reported
- security breach, and in 0.5.4-5.14 to close few other bugs.
-
-updating from <0.5.4-5
- Unless configuration file (or corresponding failregex'es) gets updated,
- security breach is not closed and corresponding warning will be reported
- by the fail2ban (in the log files).
-
-updating from <0.5.4-5.14
- Bugs #329163, #331695 dealing with changed iptables rules
- outside of fail2ban were fixed in 0.5.4-5.14, and require upgrade of the
- configuration file (fwcheck option was introduced) to take full
- advantage of the problem solution (otherwise some problems might
- persist)
-
- Please review the configuration file and make appropriate changes.
-ENJOY!
-
-EOF
-			fi
-		fi
-		;;
-
-    abort-upgrade|abort-remove|abort-deconfigure)
-
-    ;;
-
-    *)
-        echo "postinst called with unknown argument \`$1'" >&2
-        exit 1
-    ;;
-esac
-
-if dpkg-maintscript-helper supports mv_conffile 2>/dev/null; then
-    dpkg-maintscript-helper mv_conffile /etc/fail2ban/action.d/firewall-cmd-direct-new.conf /etc/fail2ban/action.d/firewallcmd-new.conf 0.8.13-1~ -- "$@"
-    dpkg-maintscript-helper mv_conffile /etc/fail2ban/filter.d/lighttpd-fastcgi.conf /etc/fail2ban/filter.d/suhosin.conf 0.8.13-1~ -- "$@"
-    dpkg-maintscript-helper mv_conffile /etc/fail2ban/filter.d/sasl.conf /etc/fail2ban/filter.d/postfix-sasl.conf 0.8.13-1~ -- "$@"
-    dpkg-maintscript-helper mv_conffile /etc/fail2ban/filter.d/couriersmtp.conf /etc/fail2ban/filter.d/courier-smtp.conf 0.9.0-1~ -- "$@"
-    dpkg-maintscript-helper mv_conffile /etc/fail2ban/filter.d/courierlogin.conf /etc/fail2ban/filter.d/courier-auth.conf 0.9.0-1~ -- "$@"
-fi
-
-# dh_installdeb will replace this with shell code automatically
-# generated by other debhelper scripts.
-
-#DEBHELPER#
-
-exit 0
-
-

debian/postrm

@@ -1,52 +0,0 @@
-#! /bin/sh
-# postrm script for fail2ban
-#
-# see: dh_installdeb(1)
-
-set -e
-
-# summary of how this script can be called:
-#        * <postrm> `remove'
-#        * <postrm> `purge'
-#        * <old-postrm> `upgrade' <new-version>
-#        * <new-postrm> `failed-upgrade' <old-version>
-#        * <new-postrm> `abort-install'
-#        * <new-postrm> `abort-install' <old-version>
-#        * <new-postrm> `abort-upgrade' <old-version>
-#        * <disappearer's-postrm> `disappear' <r>overwrit>r> <new-version>
-# for details, see /usr/doc/packaging-manual/
-
-
-case "$1" in
-       purge|disappear)
-       				
-		# Remove configuration
-                rm -f /etc/fail2ban.conf
-		
-		# Remove logs
-		rm -f /var/log/fail2ban*
-		
-		# Remove sqlite db
-		rm -f /var/lib/fail2ban/fail2ban.sqlite3
-		;;
-       remove|upgrade|failed-upgrade|abort-install|abort-upgrade)
-		# nothing
-		# We may not delete the user fail2ban, as there may be
-		# files owned by it in /var/log/ and /etc/.
-        	;;
-esac
-
-if dpkg-maintscript-helper supports mv_conffile 2>/dev/null; then
-    dpkg-maintscript-helper mv_conffile /etc/fail2ban/action.d/firewall-cmd-direct-new.conf /etc/fail2ban/action.d/firewallcmd-new.conf 0.8.13-1~ -- "$@"
-    dpkg-maintscript-helper mv_conffile /etc/fail2ban/filter.d/lighttpd-fastcgi.conf /etc/fail2ban/filter.d/suhosin.conf 0.8.13-1~ -- "$@"
-    dpkg-maintscript-helper mv_conffile /etc/fail2ban/filter.d/sasl.conf /etc/fail2ban/filter.d/postfix-sasl.conf 0.8.13-1~ -- "$@"
-    dpkg-maintscript-helper mv_conffile /etc/fail2ban/filter.d/couriersmtp.conf /etc/fail2ban/filter.d/courier-smtp.conf 0.9.0-1~ -- "$@"
-    dpkg-maintscript-helper mv_conffile /etc/fail2ban/filter.d/courierlogin.conf /etc/fail2ban/filter.d/courier-auth.conf 0.9.0-1~ -- "$@"
-fi
-
-# dh_installdeb will replace this with shell code automatically
-# generated by other debhelper scripts.
-
-#DEBHELPER#
-
-

debian/preinst

@@ -1,15 +0,0 @@
-#!/bin/sh
-
-set -e
-
-if dpkg-maintscript-helper supports mv_conffile 2>/dev/null; then
-    dpkg-maintscript-helper mv_conffile /etc/fail2ban/action.d/firewall-cmd-direct-new.conf /etc/fail2ban/action.d/firewallcmd-new.conf 0.8.13-1~ -- "$@"
-    dpkg-maintscript-helper mv_conffile /etc/fail2ban/filter.d/lighttpd-fastcgi.conf /etc/fail2ban/filter.d/suhosin.conf 0.8.13-1~ -- "$@"
-    dpkg-maintscript-helper mv_conffile /etc/fail2ban/filter.d/sasl.conf /etc/fail2ban/filter.d/postfix-sasl.conf 0.8.13-1~ -- "$@"
-    dpkg-maintscript-helper mv_conffile /etc/fail2ban/filter.d/couriersmtp.conf /etc/fail2ban/filter.d/courier-smtp.conf 0.9.0-1~ -- "$@"
-    dpkg-maintscript-helper mv_conffile /etc/fail2ban/filter.d/courierlogin.conf /etc/fail2ban/filter.d/courier-auth.conf 0.9.0-1~ -- "$@"
-fi
-
-#DEBHELPER#
-
-exit 0

debian/rules

@@ -1,69 +0,0 @@
-#!/usr/bin/make -f
-# -*- makefile -*-
-# Sample debian/rules that uses debhelper.
-# This file was originally written by Joey Hess and Craig Small.
-# As a special exception, when this file is copied by dh-make into a
-# dh-make output file, you may use that output file without restriction.
-# This special exception was added by Craig Small in version 0.37 of dh-make.
-
-# Uncomment this to turn on verbose mode.
-export DH_VERBOSE=1
-
-export PYBUILD_DISABLE_python2=1
-
-%:
-	dh $@ --with python3,systemd --buildsystem pybuild
-
-DESTDIR=$(CURDIR)/debian/fail2ban
-PYVERSION=$(shell py3versions -dv)
-
-override_dh_clean:
-	-rm -rf fail2ban.egg-info
-	-rm -f debian/fail2ban.init
-	dh_clean
-	: # auto generated
-	-rm -f bin/fail2ban-python
-
-override_dh_auto_configure:
-	LANG=C ./fail2ban-2to3
-	dh_auto_configure
-
-override_dh_install:
-	rm -f $(DESTDIR)/usr/share/doc/fail2ban/README.Solaris
-	find "$(DESTDIR)/etc/fail2ban/" -type f -name 'paths-*.conf' -not -name 'paths-common.conf' -not -name 'paths-debian.conf' -delete
-	: # Remove explicitly created /var/run/fail2ban
-	: # just to please lintian since init file will
-	: # take care about it anyways
-	rm -rf $(DESTDIR)/var/run/ $(DESTDIR)/run/
-	: # Install monit configuration
-	install -d $(DESTDIR)/etc/monit/conf-available
-	install -d $(DESTDIR)/etc/monit/monitrc.d
-	install -m 644 files/monit/fail2ban $(DESTDIR)/etc/monit/monitrc.d/fail2ban
-	: # Install bash completion
-	install -d $(DESTDIR)/usr/share/bash-completion/completions
-	install -m 644 files/bash-completion $(DESTDIR)/usr/share/bash-completion/completions/fail2ban
-	: # Install systemd files
-	install -d $(DESTDIR)/lib/systemd/system
-	install -d $(DESTDIR)/usr/lib/tmpfiles.d
-	install -m 644 build/fail2ban.service $(DESTDIR)/lib/systemd/system
-	install -m 644 files/fail2ban-tmpfiles.conf $(DESTDIR)/usr/lib/tmpfiles.d
-	install -d $(DESTDIR)/lib/systemd/system
-	: # Install default jail enabler
-	install -m 644 debian/debian-files/jail.d_defaults-debian.conf $(DESTDIR)/etc/fail2ban/jail.d/defaults-debian.conf
-	dh_install
-
-override_dh_auto_test:
-ifeq (,$(filter nocheck,$(DEB_BUILD_OPTIONS)))
-# fails for now (4 tests), accept it for now
-	FAIL2BAN_CONFIG_DIR="$(CURDIR)/config" LC_ALL=C.UTF-8 bin/fail2ban-testcases --verbosity=2 --no-network || true
-endif
-
-override_dh_installexamples:
-	dh_installexamples files/ipmasq-* files/nagios files/cacti
-
-override_dh_installinit:
-	cp -p files/debian-initd debian/fail2ban.init
-	dh_installinit -- defaults 99
-
-override_dh_installman:
-	dh_installman man/*.[15]

debian/source/format

@@ -1 +0,0 @@
-3.0 (quilt)

debian/upstream/metadata

@@ -1,4 +0,0 @@
-Bug-Database: https://github.com/fail2ban/fail2ban/issues
-Bug-Submit: https://github.com/fail2ban/fail2ban/issues/new
-Repository: https://github.com/fail2ban/fail2ban.git
-Repository-Browse: https://github.com/fail2ban/fail2ban

debian/watch

@@ -1,6 +0,0 @@
-# watch control file for uscan
-# Run the "uscan" command to check for upstream updates and more.
-# Site		Directory		Pattern			Version	Script
-version=3
-opts="filenamemangle=s/.*\/(.*)/fail2ban-$1\.tar\.gz/" \
- https://github.com/fail2ban/fail2ban/tags .*archive/(\d[\d\.]+).tar.gz

fail2ban/client/actionreader.py

@@ -38,28 +38,31 @@ class ActionReader(DefinitionInitConfigReader):
 
 	_configOpts = {
 		"actionstart": ["string", None],
-		"actionstart_on_demand": ["string", None],
+		"actionstart_on_demand": ["bool", None],
 		"actionstop": ["string", None],
 		"actionflush": ["string", None],
 		"actionreload": ["string", None],
 		"actioncheck": ["string", None],
 		"actionrepair": ["string", None],
-		"actionrepair_on_unban": ["string", None],
+		"actionrepair_on_unban": ["bool", None],
 		"actionban": ["string", None],
-		"actionprolong": ["string", None],
 		"actionreban": ["string", None],
 		"actionunban": ["string", None],
-		"norestored": ["string", None],
+		"norestored": ["bool", None],
 	}
 
 	def __init__(self, file_, jailName, initOpts, **kwargs):
+		# always supply jail name as name parameter if not specified in options:
+		n = initOpts.get("name")
+		if n is None:
+			initOpts["name"] = n = jailName
 		actname = initOpts.get("actname")
 		if actname is None:
 			actname = file_
+			# ensure we've unique action name per jail:
+			if n != jailName:
+				actname += n[len(jailName):] if n.startswith(jailName) else '-' + n
 			initOpts["actname"] = actname
-		# always supply jail name as name parameter if not specified in options:
-		if initOpts.get("name") is None:
-			initOpts["name"] = jailName
 		self._name = actname
 		DefinitionInitConfigReader.__init__(
 			self, file_, jailName, initOpts, **kwargs)
@@ -80,11 +83,6 @@ class ActionReader(DefinitionInitConfigReader):
 	def convert(self):
 		opts = self.getCombined(
 			ignore=CommandAction._escapedTags | set(('timeout', 'bantime')))
-		# type-convert only after combined (otherwise boolean converting prevents substitution):
-		for o in ('norestored', 'actionstart_on_demand', 'actionrepair_on_unban'):
-			if opts.get(o):
-				opts[o] = self._convert_to_boolean(opts[o])
-		
 		# stream-convert:
 		head = ["set", self._jailName]
 		stream = list()

fail2ban/client/beautifier.py

@@ -180,12 +180,6 @@ class Beautifier:
 					msg = "The jail %s action %s has the following " \
 						"methods:\n" % (inC[1], inC[3])
 					msg += ", ".join(response)
-			elif inC[2] == "banip" and inC[0] == "get":
-				if isinstance(response, list):
-					sep = " " if len(inC) <= 3 else inC[3]
-					if sep == "--with-time":
-						sep = "\n"
-					msg = sep.join(response)
 		except Exception:
 			logSys.warning("Beautifier error. Please report the error")
 			logSys.error("Beautify %r with %r failed", response, self.__inputCmd,

fail2ban/client/configparserinc.py

@@ -29,7 +29,7 @@ import re
 import sys
 from ..helpers import getLogger
 
-if sys.version_info >= (3,2):
+if sys.version_info >= (3,): # pragma: 2.x no cover
 
 	# SafeConfigParser deprecated from Python 3.2 (renamed to ConfigParser)
 	from configparser import ConfigParser as SafeConfigParser, BasicInterpolation, \
@@ -61,7 +61,7 @@ if sys.version_info >= (3,2):
 				return super(BasicInterpolationWithName, self)._interpolate_some(
 					parser, option, accum, rest, section, map, *args, **kwargs)
 
-else: # pragma: no cover
+else: # pragma: 3.x no cover
 	from ConfigParser import SafeConfigParser, \
 		InterpolationMissingOptionError, NoOptionError, NoSectionError
 
@@ -372,7 +372,8 @@ after = 1.conf
 						s2 = alls.get(n)
 						if isinstance(s2, dict):
 							# save previous known values, for possible using in local interpolations later:
-							self.merge_section('KNOWN/'+n, s2, '')
+							self.merge_section('KNOWN/'+n, 
+								dict(filter(lambda i: i[0] in s, s2.iteritems())), '')
 							# merge section
 							s2.update(s)
 						else:

fail2ban/client/configreader.py

@@ -34,6 +34,30 @@ from ..helpers import getLogger, _as_bool, _merge_dicts, substituteRecursiveTags
 # Gets the instance of the logger.
 logSys = getLogger(__name__)
 
+CONVERTER = {
+	"bool": _as_bool,
+	"int": int,
+}
+def _OptionsTemplateGen(options):
+	"""Iterator over the options template with default options.
+	
+	Each options entry is composed of an array or tuple with:
+		[[type, name, ?default?], ...]
+	Or it is a dict:
+		{name: [type, default], ...}
+	"""
+	if isinstance(options, (list,tuple)):
+		for optname in options:
+			if len(optname) > 2:
+				opttype, optname, optvalue = optname
+			else:
+				(opttype, optname), optvalue = optname, None
+			yield opttype, optname, optvalue
+	else:
+		for optname in options:
+			opttype, optvalue = options[optname]
+			yield opttype, optname, optvalue
+
 
 class ConfigReader():
 	"""Generic config reader class.
@@ -120,6 +144,10 @@ class ConfigReader():
 		except AttributeError:
 			return False
 
+	def has_option(self, sec, opt, withDefault=True):
+		return self._cfg.has_option(sec, opt) if withDefault \
+			else opt in self._cfg._sections.get(sec, {})
+
 	def merge_defaults(self, d):
 		self._cfg.get_defaults().update(d)
 
@@ -224,31 +252,22 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes):
 	# Or it is a dict:
 	#  {name: [type, default], ...}
 	
-	def getOptions(self, sec, options, pOptions=None, shouldExist=False):
+	def getOptions(self, sec, options, pOptions=None, shouldExist=False, convert=True):
 		values = dict()
 		if pOptions is None:
 			pOptions = {}
 		# Get only specified options:
-		for optname in options:
-			if isinstance(options, (list,tuple)):
-				if len(optname) > 2:
-					opttype, optname, optvalue = optname
-				else:
-					(opttype, optname), optvalue = optname, None
-			else:
-				opttype, optvalue = options[optname]
+		for opttype, optname, optvalue in _OptionsTemplateGen(options):
 			if optname in pOptions:
 				continue
 			try:
-				if opttype == "bool":
-					v = self.getboolean(sec, optname)
-					if v is None: continue
-				elif opttype == "int":
-					v = self.getint(sec, optname)
-					if v is None: continue
-				else:
-					v = self.get(sec, optname, vars=pOptions)
+				v = self.get(sec, optname, vars=pOptions)
 				values[optname] = v
+				if convert:
+					conv = CONVERTER.get(opttype)
+					if conv:
+						if v is None: continue
+						values[optname] = conv(v)
 			except NoSectionError as e:
 				if shouldExist:
 					raise
@@ -261,8 +280,8 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes):
 					logSys.warning("'%s' not defined in '%s'. Using default one: %r"
 								% (optname, sec, optvalue))
 					values[optname] = optvalue
-				elif logSys.getEffectiveLevel() <= logLevel:
-					logSys.log(logLevel, "Non essential option '%s' not defined in '%s'.", optname, sec)
+				# elif logSys.getEffectiveLevel() <= logLevel:
+				# 	logSys.log(logLevel, "Non essential option '%s' not defined in '%s'.", optname, sec)
 			except ValueError:
 				logSys.warning("Wrong value for '" + optname + "' in '" + sec +
 							"'. Using default one: '" + repr(optvalue) + "'")
@@ -320,8 +339,9 @@ class DefinitionInitConfigReader(ConfigReader):
 			pOpts = dict()
 		if self._initOpts:
 			pOpts = _merge_dicts(pOpts, self._initOpts)
+		# type-convert only in combined (otherwise int/bool converting prevents substitution):
 		self._opts = ConfigReader.getOptions(
-			self, "Definition", self._configOpts, pOpts)
+			self, "Definition", self._configOpts, pOpts, convert=False)
 		self._pOpts = pOpts
 		if self.has_section("Init"):
 			# get only own options (without options from default):
@@ -342,10 +362,21 @@ class DefinitionInitConfigReader(ConfigReader):
 				if opt == '__name__' or opt in self._opts: continue
 				self._opts[opt] = self.get("Definition", opt)
 
+	def convertOptions(self, opts, configOpts):
+		"""Convert interpolated combined options to expected type.
+		"""
+		for opttype, optname, optvalue in _OptionsTemplateGen(configOpts):
+			conv = CONVERTER.get(opttype)
+			if conv:
+				v = opts.get(optname)
+				if v is None: continue
+				try:
+					opts[optname] = conv(v)
+				except ValueError:
+					logSys.warning("Wrong %s value %r for %r. Using default one: %r",
+						opttype, v, optname, optvalue)
+					opts[optname] = optvalue
 
-	def _convert_to_boolean(self, value):
-		return _as_bool(value)
-	
 	def getCombOption(self, optname):
 		"""Get combined definition option (as string) using pre-set and init
 		options as preselection (values with higher precedence as specified in section).
@@ -380,6 +411,8 @@ class DefinitionInitConfigReader(ConfigReader):
 			ignore=ignore, addrepl=self.getCombOption)
 		if not opts:
 			raise ValueError('recursive tag definitions unable to be resolved')
+		# convert options after all interpolations:
+		self.convertOptions(opts, self._configOpts)
 		return opts
 	
 	def convert(self):

fail2ban/client/csocket.py

@@ -48,7 +48,8 @@ class CSocket:
 	def send(self, msg, nonblocking=False, timeout=None):
 		# Convert every list member to string
 		obj = dumps(map(CSocket.convert, msg), HIGHEST_PROTOCOL)
-		self.__csock.send(obj + CSPROTO.END)
+		self.__csock.send(obj)
+		self.__csock.send(CSPROTO.END)
 		return self.receive(self.__csock, nonblocking, timeout)
 
 	def settimeout(self, timeout):
@@ -81,9 +82,12 @@ class CSocket:
 		msg = CSPROTO.EMPTY
 		if nonblocking: sock.setblocking(0)
 		if timeout: sock.settimeout(timeout)
-		while msg.rfind(CSPROTO.END) == -1:
-			chunk = sock.recv(512)
-			if chunk in ('', b''): # python 3.x may return b'' instead of ''
-				raise RuntimeError("socket connection broken")
+		bufsize = 1024
+		while msg.rfind(CSPROTO.END, -32) == -1:
+			chunk = sock.recv(bufsize)
+			if not len(chunk):
+				raise socket.error(104, 'Connection reset by peer')
+			if chunk == CSPROTO.END: break
 			msg = msg + chunk
+			if bufsize < 32768: bufsize <<= 1
 		return loads(msg)

fail2ban/client/fail2banclient.py

@@ -168,19 +168,6 @@ class Fail2banClient(Fail2banCmdLine, Thread):
 		if not ret:
 			return None
 
-		# verify that directory for the socket file exists
-		socket_dir = os.path.dirname(self._conf["socket"])
-		if not os.path.exists(socket_dir):
-			logSys.error(
-				"There is no directory %s to contain the socket file %s."
-				% (socket_dir, self._conf["socket"]))
-			return None
-		if not os.access(socket_dir, os.W_OK | os.X_OK): # pragma: no cover
-			logSys.error(
-				"Directory %s exists but not accessible for writing"
-				% (socket_dir,))
-			return None
-
 		# Check already running
 		if not self._conf["force"] and os.path.exists(self._conf["socket"]):
 			logSys.error("Fail2ban seems to be in unexpected state (not running but the socket exists)")

fail2ban/client/fail2bancmdline.py

@@ -27,15 +27,20 @@ import sys
 
 from ..version import version, normVersion
 from ..protocol import printFormatted
-from ..helpers import getLogger, str2LogLevel, getVerbosityFormat
+from ..helpers import getLogger, str2LogLevel, getVerbosityFormat, BrokenPipeError
 
 # Gets the instance of the logger.
 logSys = getLogger("fail2ban")
 
 def output(s): # pragma: no cover
-	print(s)
+	try:
+		print(s)
+	except (BrokenPipeError, IOError) as e: # pragma: no cover
+		if e.errno != 32: # closed / broken pipe
+			raise
 
-CONFIG_PARAMS = ("socket", "pidfile", "logtarget", "loglevel", "syslogsocket",)
+# Config parameters required to start fail2ban which can be also set via command line (overwrite fail2ban.conf),
+CONFIG_PARAMS = ("socket", "pidfile", "logtarget", "loglevel", "syslogsocket")
 # Used to signal - we are in test cases (ex: prevents change logging params, log capturing, etc)
 PRODUCTION = True
 
@@ -94,9 +99,10 @@ class Fail2banCmdLine():
 		output("and bans the corresponding IP addresses using firewall rules.")
 		output("")
 		output("Options:")
-		output("    -c <DIR>                configuration directory")
-		output("    -s <FILE>               socket path")
-		output("    -p <FILE>               pidfile path")
+		output("    -c, --conf <DIR>        configuration directory")
+		output("    -s, --socket <FILE>     socket path")
+		output("    -p, --pidfile <FILE>    pidfile path")
+		output("    --pname <NAME>          name of the process (main thread) to identify instance (default fail2ban-server)")
 		output("    --loglevel <LEVEL>      logging level")
 		output("    --logtarget <TARGET>    logging target, use file-name or stdout, stderr, syslog or sysout.")
 		output("    --syslogsocket auto|<FILE>")
@@ -129,17 +135,15 @@ class Fail2banCmdLine():
 		"""
 		for opt in optList:
 			o = opt[0]
-			if o == "-c":
+			if o in ("-c", "--conf"):
 				self._conf["conf"] = opt[1]
-			elif o == "-s":
+			elif o in ("-s", "--socket"):
 				self._conf["socket"] = opt[1]
-			elif o == "-p":
+			elif o in ("-p", "--pidfile"):
 				self._conf["pidfile"] = opt[1]
-			elif o.startswith("--log") or o.startswith("--sys"):
-				self._conf[ o[2:] ] = opt[1]
-			elif o in ["-d", "--dp", "--dump-pretty"]:
+			elif o in ("-d", "--dp", "--dump-pretty"):
 				self._conf["dump"] = True if o == "-d" else 2
-			elif o == "-t" or o == "--test":
+			elif o in ("-t", "--test"):
 				self.cleanConfOnly = True
 				self._conf["test"] = True
 			elif o == "-v":
@@ -163,12 +167,14 @@ class Fail2banCmdLine():
 				from ..server.mytime import MyTime
 				output(MyTime.str2seconds(opt[1]))
 				return True
-			elif o in ["-h", "--help"]:
+			elif o in ("-h", "--help"):
 				self.dispUsage()
 				return True
-			elif o in ["-V", "--version"]:
+			elif o in ("-V", "--version"):
 				self.dispVersion(o == "-V")
 				return True
+			elif o.startswith("--"): # other long named params (see also resetConf)
+				self._conf[ o[2:] ] = opt[1]
 		return None
 
 	def initCmdLine(self, argv):
@@ -185,7 +191,8 @@ class Fail2banCmdLine():
 			try:
 				cmdOpts = 'hc:s:p:xfbdtviqV'
 				cmdLongOpts = ['loglevel=', 'logtarget=', 'syslogsocket=', 'test', 'async',
-					'timeout=', 'str2sec=', 'help', 'version', 'dp', '--dump-pretty']
+					'conf=', 'pidfile=', 'pname=', 'socket=',
+					'timeout=', 'str2sec=', 'help', 'version', 'dp', 'dump-pretty']
 				optList, self._args = getopt.getopt(self._argv[1:], cmdOpts, cmdLongOpts)
 			except getopt.GetoptError:
 				self.dispUsage()
@@ -227,7 +234,8 @@ class Fail2banCmdLine():
 					if not conf:
 						self.configurator.readEarly()
 						conf = self.configurator.getEarlyOptions()
-					self._conf[o] = conf[o]
+					if o in conf:
+						self._conf[o] = conf[o]
 
 			logSys.info("Using socket file %s", self._conf["socket"])
 
@@ -304,18 +312,24 @@ class Fail2banCmdLine():
 	# since method is also exposed in API via globally bound variable
 	@staticmethod
 	def _exit(code=0):
-		if hasattr(os, '_exit') and os._exit:
-			os._exit(code)
-		else:
-			sys.exit(code)
+		# implicit flush without to produce broken pipe error (32):
+		sys.stderr.close()
+		try:
+			sys.stdout.flush()
+			# exit:
+			if hasattr(sys, 'exit') and sys.exit:
+				sys.exit(code)
+			else:
+				os._exit(code)
+		except (BrokenPipeError, IOError) as e: # pragma: no cover
+			if e.errno != 32: # closed / broken pipe
+				raise
 
 	@staticmethod
 	def exit(code=0):
 		logSys.debug("Exit with code %s", code)
 		# because of possible buffered output in python, we should flush it before exit:
 		logging.shutdown()
-		sys.stdout.flush()
-		sys.stderr.flush()
 		# exit
 		Fail2banCmdLine._exit(code)
 

fail2ban/client/fail2banreader.py

@@ -53,6 +53,7 @@ class Fail2banReader(ConfigReader):
 		opts = [["string", "loglevel", "INFO" ],
 				["string", "logtarget", "STDERR"],
 				["string", "syslogsocket", "auto"],
+				["string", "allowipv6", "auto"],
 				["string", "dbfile", "/var/lib/fail2ban/fail2ban.sqlite3"],
 				["int",    "dbmaxmatches", None],
 				["string", "dbpurgeage", "1d"]]
@@ -74,6 +75,7 @@ class Fail2banReader(ConfigReader):
 		# Also dbfile should be set before all other database options.
 		# So adding order indices into items, to be stripped after sorting, upon return
 		order = {"thread":0, "syslogsocket":11, "loglevel":12, "logtarget":13,
+			"allowipv6": 14,
 			"dbfile":50, "dbmaxmatches":51, "dbpurgeage":51}
 		stream = list()
 		for opt in self.__opts:

fail2ban/client/fail2banregex.py

@@ -21,7 +21,6 @@ Fail2Ban  reads log file that contains password failure report
 and bans the corresponding IP addresses using firewall rules.
 
 This tools can test regular expressions for "fail2ban".
-
 """
 
 __author__ = "Fail2Ban Developers"
@@ -36,11 +35,11 @@ __license__ = "GPL"
 
 import getopt
 import logging
+import re
 import os
 import shlex
 import sys
 import time
-import time
 import urllib
 from optparse import OptionParser, Option
 
@@ -53,7 +52,7 @@ except ImportError:
 
 from ..version import version, normVersion
 from .filterreader import FilterReader
-from ..server.filter import Filter, FileContainer
+from ..server.filter import Filter, FileContainer, MyTime
 from ..server.failregex import Regex, RegexException
 
 from ..helpers import str2LogLevel, getVerbosityFormat, FormatterWithTraceBack, getLogger, \
@@ -109,19 +108,22 @@ class _f2bOptParser(OptionParser):
 	def format_help(self, *args, **kwargs):
 		""" Overwritten format helper with full ussage."""
 		self.usage = ''
-		return "Usage: " + usage() + __doc__ + """
+		return "Usage: " + usage() + "\n" + __doc__ + """
 LOG:
-    string                  a string representing a log line
-    filename                path to a log file (/var/log/auth.log)
-    "systemd-journal"       search systemd journal (systemd-python required)
+  string                a string representing a log line
+  filename              path to a log file (/var/log/auth.log)
+  systemd-journal       search systemd journal (systemd-python required),
+                        optionally with backend parameters, see `man jail.conf`
+                        for usage and examples (systemd-journal[journalflags=1]).
 
 REGEX:
-    string                  a string representing a 'failregex'
-    filename                path to a filter file (filter.d/sshd.conf)
+  string                a string representing a 'failregex'
+  filter                name of filter, optionally with options (sshd[mode=aggressive])
+  filename              path to a filter file (filter.d/sshd.conf)
 
 IGNOREREGEX:
-    string                  a string representing an 'ignoreregex'
-    filename                path to a filter file (filter.d/sshd.conf)
+  string                a string representing an 'ignoreregex'
+  filename              path to a filter file (filter.d/sshd.conf)
 \n""" + OptionParser.format_help(self, *args, **kwargs) + """\n
 Report bugs to https://github.com/fail2ban/fail2ban/issues\n
 """ + __copyright__ + "\n"
@@ -252,6 +254,8 @@ class Fail2banRegex(object):
 
 		self.share_config=dict()
 		self._filter = Filter(None)
+		self._prefREMatched = 0
+		self._prefREGroups = list()
 		self._ignoreregex = list()
 		self._failregex = list()
 		self._time_elapsed = None
@@ -265,13 +269,21 @@ class Fail2banRegex(object):
 			self.setJournalMatch(shlex.split(opts.journalmatch))
 		if opts.timezone:
 			self._filter.setLogTimeZone(opts.timezone)
+		self._filter.checkFindTime = False
+		if True: # not opts.out:
+			MyTime.setAlternateNow(0); # accept every date (years from 19xx up to end of current century, '%ExY' and 'Exy' patterns)
+			from ..server.strptime import _updateTimeRE
+			_updateTimeRE()
 		if opts.datepattern:
 			self.setDatePattern(opts.datepattern)
 		if opts.usedns:
 			self._filter.setUseDns(opts.usedns)
 		self._filter.returnRawHost = opts.raw
-		self._filter.checkFindTime = False
 		self._filter.checkAllRegex = opts.checkAllRegex and not opts.out
+		# ignore pending (without ID/IP), added to matches if it hits later (if ID/IP can be retreved)
+		self._filter.ignorePending = bool(opts.out)
+		# callback to increment ignored RE's by index (during process):
+		self._filter.onIgnoreRegex = self._onIgnoreRegex
 		self._backend = 'auto'
 
 	def output(self, line):
@@ -288,8 +300,8 @@ class Fail2banRegex(object):
 			self._filter.setDatePattern(pattern)
 			self._datepattern_set = True
 			if pattern is not None:
-				self.output( "Use      datepattern : %s" % (
-					self._filter.getDatePattern()[1], ) )
+				self.output( "Use      datepattern : %s : %s" % (
+					pattern, self._filter.getDatePattern()[1], ) )
 
 	def setMaxLines(self, v):
 		if not self._maxlines_set:
@@ -318,26 +330,33 @@ class Fail2banRegex(object):
 		regex = regextype + 'regex'
 		# try to check - we've case filter?[options...]?:
 		basedir = self._opts.config
+		fltName = value
 		fltFile = None
 		fltOpt = {}
 		if regextype == 'fail':
-			fltName, fltOpt = extractOptions(value)
-			if fltName is not None:
-				if "." in fltName[~5:]:
-					tryNames = (fltName,)
-				else:
-					tryNames = (fltName, fltName + '.conf', fltName + '.local')
-				for fltFile in tryNames:
-					if not "/" in fltFile:
-						if os.path.basename(basedir) == 'filter.d':
-							fltFile = os.path.join(basedir, fltFile)
-						else:
-							fltFile = os.path.join(basedir, 'filter.d', fltFile)
+			if re.search(r'^/{0,3}[\w/_\-.]+(?:\[.*\])?$', value):
+				try:
+					fltName, fltOpt = extractOptions(value)
+					if "." in fltName[~5:]:
+						tryNames = (fltName,)
 					else:
-						basedir = os.path.dirname(fltFile)
-					if os.path.isfile(fltFile):
-						break
-					fltFile = None
+						tryNames = (fltName, fltName + '.conf', fltName + '.local')
+					for fltFile in tryNames:
+						if not "/" in fltFile:
+							if os.path.basename(basedir) == 'filter.d':
+								fltFile = os.path.join(basedir, fltFile)
+							else:
+								fltFile = os.path.join(basedir, 'filter.d', fltFile)
+						else:
+							basedir = os.path.dirname(fltFile)
+						if os.path.isfile(fltFile):
+							break
+						fltFile = None
+				except Exception as e:
+					output("ERROR: Wrong filter name or options: %s" % (str(e),))
+					output("       while parsing: %s" % (value,))
+					if self._verbose: raise(e)
+					return False
 		# if it is filter file:
 		if fltFile is not None:
 			if (basedir == self._opts.config
@@ -372,11 +391,8 @@ class Fail2banRegex(object):
 			if not ret:
 				output( "ERROR: failed to load filter %s" % value )
 				return False
-			# overwrite default logtype (considering that the filter could specify this too in Definition/Init sections):
-			if not fltOpt.get('logtype'):
-				reader.merge_defaults({
-					'logtype': ['file','journal'][int(self._backend.startswith("systemd"))]
-				})
+			# set backend-related options (logtype):
+			reader.applyAutoOptions(self._backend)
 			# get, interpolate and convert options:
 			reader.getOptions(None)
 			# show real options if expected:
@@ -436,71 +452,140 @@ class Fail2banRegex(object):
 					'add%sRegex' % regextype.title())(regex.getFailRegex())
 		return True
 
-	def testIgnoreRegex(self, line):
-		found = False
-		try:
-			ret = self._filter.ignoreLine([(line, "", "")])
-			if ret is not None:
-				found = True
-				regex = self._ignoreregex[ret].inc()
-		except RegexException as e: # pragma: no cover
-			output( 'ERROR: %s' % e )
-			return False
-		return found
+	def _onIgnoreRegex(self, idx, ignoreRegex):
+		self._lineIgnored = True
+		self._ignoreregex[idx].inc()
 
 	def testRegex(self, line, date=None):
 		orgLineBuffer = self._filter._Filter__lineBuffer
+		# duplicate line buffer (list can be changed inplace during processLine):
+		if self._filter.getMaxLines() > 1:
+			orgLineBuffer = orgLineBuffer[:]
 		fullBuffer = len(orgLineBuffer) >= self._filter.getMaxLines()
-		is_ignored = False
+		is_ignored = self._lineIgnored = False
 		try:
 			found = self._filter.processLine(line, date)
 			lines = []
-			line = self._filter.processedLine()
 			ret = []
 			for match in found:
-				# Append True/False flag depending if line was matched by
-				# more than one regex
-				match.append(len(ret)>1)
-				regex = self._failregex[match[0]]
-				regex.inc()
-				regex.appendIP(match)
+				if not self._opts.out:
+					# Append True/False flag depending if line was matched by
+					# more than one regex
+					match.append(len(ret)>1)
+					regex = self._failregex[match[0]]
+					regex.inc()
+					regex.appendIP(match)
 				if not match[3].get('nofail'):
 					ret.append(match)
 				else:
 					is_ignored = True
+			if self._opts.out: # (formated) output - don't need stats:
+				return None, ret, None
+			# prefregex stats:
+			if self._filter.prefRegex:
+				pre = self._filter.prefRegex
+				if pre.hasMatched():
+					self._prefREMatched += 1
+					if self._verbose:
+						if len(self._prefREGroups) < self._maxlines:
+							self._prefREGroups.append(pre.getGroups())
+						else:
+							if len(self._prefREGroups) == self._maxlines:
+								self._prefREGroups.append('...')
 		except RegexException as e: # pragma: no cover
 			output( 'ERROR: %s' % e )
-			return False
-		for bufLine in orgLineBuffer[int(fullBuffer):]:
-			if bufLine not in self._filter._Filter__lineBuffer:
-				try:
-					self._line_stats.missed_lines.pop(
-						self._line_stats.missed_lines.index("".join(bufLine)))
-					if self._debuggex:
-						self._line_stats.missed_lines_timeextracted.pop(
-							self._line_stats.missed_lines_timeextracted.index(
-								"".join(bufLine[::2])))
-				except ValueError:
-					pass
-				# if buffering - add also another lines from match:
-				if self._print_all_matched:
-					if not self._debuggex:
-						self._line_stats.matched_lines.append("".join(bufLine))
-					else:
-						lines.append(bufLine[0] + bufLine[2])
-				self._line_stats.matched += 1
-				self._line_stats.missed -= 1
+			return None, 0, None
+		if self._filter.getMaxLines() > 1:
+			for bufLine in orgLineBuffer[int(fullBuffer):]:
+				if bufLine not in self._filter._Filter__lineBuffer:
+					try:
+						self._line_stats.missed_lines.pop(
+							self._line_stats.missed_lines.index("".join(bufLine)))
+						if self._debuggex:
+							self._line_stats.missed_lines_timeextracted.pop(
+								self._line_stats.missed_lines_timeextracted.index(
+									"".join(bufLine[::2])))
+					except ValueError:
+						pass
+					# if buffering - add also another lines from match:
+					if self._print_all_matched:
+						if not self._debuggex:
+							self._line_stats.matched_lines.append("".join(bufLine))
+						else:
+							lines.append(bufLine[0] + bufLine[2])
+					self._line_stats.matched += 1
+					self._line_stats.missed -= 1
 		if lines: # pre-lines parsed in multiline mode (buffering)
-			lines.append(line)
+			lines.append(self._filter.processedLine())
 			line = "\n".join(lines)
-		return line, ret, is_ignored
+		return line, ret, (is_ignored or self._lineIgnored)
+
+	def _prepaireOutput(self):
+		"""Prepares output- and fetch-function corresponding given '--out' option (format)"""
+		ofmt = self._opts.out
+		if ofmt in ('id', 'ip'):
+			def _out(ret):
+				for r in ret:
+					output(r[1])
+		elif ofmt == 'msg':
+			def _out(ret):
+				for r in ret:
+					for r in r[3].get('matches'):
+						if not isinstance(r, basestring):
+							r = ''.join(r for r in r)
+						output(r)
+		elif ofmt == 'row':
+			def _out(ret):
+				for r in ret:
+					output('[%r,\t%r,\t%r],' % (r[1],r[2],dict((k,v) for k, v in r[3].iteritems() if k != 'matches')))
+		elif '<' not in ofmt:
+			def _out(ret):
+				for r in ret:
+					output(r[3].get(ofmt))
+		else: # extended format with tags substitution:
+			from ..server.actions import Actions, CommandAction, BanTicket
+			def _escOut(t, v):
+				# use safe escape (avoid inject on pseudo tag "\x00msg\x00"):
+				if t not in ('msg',):
+					return v.replace('\x00', '\\x00')
+				return v
+			def _out(ret):
+				rows = []
+				wrap = {'NL':0}
+				for r in ret:
+					ticket = BanTicket(r[1], time=r[2], data=r[3])
+					aInfo = Actions.ActionInfo(ticket)
+					# if msg tag is used - output if single line (otherwise let it as is to wrap multilines later):
+					def _get_msg(self):
+						if not wrap['NL'] and len(r[3].get('matches', [])) <= 1:
+							return self['matches']
+						else: # pseudo tag for future replacement:
+							wrap['NL'] = 1
+							return "\x00msg\x00"
+					aInfo['msg'] = _get_msg
+					# not recursive interpolation (use safe escape):
+					v = CommandAction.replaceDynamicTags(ofmt, aInfo, escapeVal=_escOut)
+					if wrap['NL']: # contains multiline tags (msg):
+						rows.append((r, v))
+						continue
+					output(v)
+				# wrap multiline tag (msg) interpolations to single line:
+				for r, v in rows:
+					for r in r[3].get('matches'):
+						if not isinstance(r, basestring):
+							r = ''.join(r for r in r)
+						r = v.replace("\x00msg\x00", r)
+						output(r)
+		return _out
+
 
 	def process(self, test_lines):
 		t0 = time.time()
+		if self._opts.out: # get out function
+			out = self._prepaireOutput()
 		for line in test_lines:
 			if isinstance(line, tuple):
-				line_datetimestripped, ret, is_ignored = self.testRegex(
-					line[0], line[1])
+				line_datetimestripped, ret, is_ignored = self.testRegex(line[0], line[1])
 				line = "".join(line[0])
 			else:
 				line = line.rstrip('\r\n')
@@ -508,8 +593,10 @@ class Fail2banRegex(object):
 					# skip comment and empty lines
 					continue
 				line_datetimestripped, ret, is_ignored = self.testRegex(line)
-			if not is_ignored:
-				is_ignored = self.testIgnoreRegex(line_datetimestripped)
+
+			if self._opts.out: # (formated) output:
+				if len(ret) > 0 and not is_ignored: out(ret)
+				continue
 
 			if is_ignored:
 				self._line_stats.ignored += 1
@@ -517,42 +604,25 @@ class Fail2banRegex(object):
 					self._line_stats.ignored_lines.append(line)
 					if self._debuggex:
 						self._line_stats.ignored_lines_timeextracted.append(line_datetimestripped)
-
-			if len(ret) > 0:
-				assert(not is_ignored)
-				if self._opts.out:
-					if self._opts.out in ('id', 'ip'):
-						for ret in ret:
-							output(ret[1])
-					elif self._opts.out == 'msg':
-						for ret in ret:
-							output('\n'.join(map(lambda v:''.join(v for v in v), ret[3].get('matches'))))
-					elif self._opts.out == 'row':
-						for ret in ret:
-							output('[%r,\t%r,\t%r],' % (ret[1],ret[2],dict((k,v) for k, v in ret[3].iteritems() if k != 'matches')))
-					else:
-						for ret in ret:
-							output(ret[3].get(self._opts.out))
-					continue
+			elif len(ret) > 0:
 				self._line_stats.matched += 1
 				if self._print_all_matched:
 					self._line_stats.matched_lines.append(line)
 					if self._debuggex:
 						self._line_stats.matched_lines_timeextracted.append(line_datetimestripped)
 			else:
-				if not is_ignored:
-					self._line_stats.missed += 1
-					if not self._print_no_missed and (self._print_all_missed or self._line_stats.missed <= self._maxlines + 1):
-						self._line_stats.missed_lines.append(line)
-						if self._debuggex:
-							self._line_stats.missed_lines_timeextracted.append(line_datetimestripped)
+				self._line_stats.missed += 1
+				if not self._print_no_missed and (self._print_all_missed or self._line_stats.missed <= self._maxlines + 1):
+					self._line_stats.missed_lines.append(line)
+					if self._debuggex:
+						self._line_stats.missed_lines_timeextracted.append(line_datetimestripped)
 			self._line_stats.tested += 1
 
 		self._time_elapsed = time.time() - t0
 
 	def printLines(self, ltype):
 		lstats = self._line_stats
-		assert(self._line_stats.missed == lstats.tested - (lstats.matched + lstats.ignored))
+		assert(lstats.missed == lstats.tested - (lstats.matched + lstats.ignored))
 		lines = lstats[ltype]
 		l = lstats[ltype + '_lines']
 		multiline = self._filter.getMaxLines() > 1
@@ -610,7 +680,18 @@ class Fail2banRegex(object):
 			pprint_list(out, " #) [# of hits] regular expression")
 			return total
 
-		# Print title
+		# Print prefregex:
+		if self._filter.prefRegex:
+			#self._filter.prefRegex.hasMatched()
+			pre = self._filter.prefRegex 
+			out = [pre.getRegex()]
+			if self._verbose:
+				for grp in self._prefREGroups:
+					out.append("    %s" % (grp,))
+			output( "\n%s: %d total" % ("Prefregex", self._prefREMatched) )
+			pprint_list(out)
+
+		# Print regex's:
 		total = print_failregexes("Failregex", self._failregex)
 		_ = print_failregexes("Ignoreregex", self._ignoreregex)
 
@@ -689,10 +770,10 @@ class Fail2banRegex(object):
 			test_lines = journal_lines_gen(flt, myjournal)
 		else:
 			# if single line parsing (without buffering)
-			if self._filter.getMaxLines() <= 1:
+			if self._filter.getMaxLines() <= 1 and '\n' not in cmd_log:
 				self.output( "Use      single line : %s" % shortstr(cmd_log.replace("\n", r"\n")) )
 				test_lines = [ cmd_log ]
-			else: # multi line parsing (with buffering)
+			else: # multi line parsing (with and without buffering)
 				test_lines = cmd_log.split("\n")
 				self.output( "Use      multi line : %s line(s)" % len(test_lines) )
 				for i, l in enumerate(test_lines):
@@ -712,6 +793,7 @@ class Fail2banRegex(object):
 
 
 def exec_command_line(*args):
+	logging.exitOnIOError = True
 	parser = get_opt_parser()
 	(opts, args) = parser.parse_args(*args)
 	errors = []

fail2ban/client/filterreader.py

@@ -53,6 +53,14 @@ class FilterReader(DefinitionInitConfigReader):
 	def getFile(self):
 		return self.__file
 
+	def applyAutoOptions(self, backend):
+		# set init option to backend-related logtype, considering
+		# that the filter settings may be overwritten in its local:
+		if (not self._initOpts.get('logtype') and 
+		    not self.has_option('Definition', 'logtype', False)
+		  ):
+			self._initOpts['logtype'] = ['file','journal'][int(backend.startswith("systemd"))]
+
 	def convert(self):
 		stream = list()
 		opts = self.getCombined()

fail2ban/client/jailreader.py

@@ -98,13 +98,6 @@ class JailReader(ConfigReader):
 		"maxmatches": ["int", None],
 		"findtime": ["string", None],
 		"bantime": ["string", None],
-		"bantime.increment": ["bool", None],
-		"bantime.factor": ["string", None],
-		"bantime.formula": ["string", None],
-		"bantime.multipliers": ["string", None],
-		"bantime.maxtime": ["string", None],
-		"bantime.rndtime": ["string", None],
-		"bantime.overalljails": ["bool", None],
 		"ignorecommand": ["string", None],
 		"ignoreself": ["bool", None],
 		"ignoreip": ["string", None],
@@ -140,20 +133,18 @@ class JailReader(ConfigReader):
 			# Read filter
 			flt = self.__opts["filter"]
 			if flt:
-				filterName, filterOpt = extractOptions(flt)
-				if not filterName:
-					raise JailDefError("Invalid filter definition %r" % flt)
+				try:
+					filterName, filterOpt = extractOptions(flt)
+				except ValueError as e:
+					raise JailDefError("Invalid filter definition %r: %s" % (flt, e))
 				self.__filter = FilterReader(
 					filterName, self.__name, filterOpt, 
 					share_config=self.share_config, basedir=self.getBaseDir())
 				ret = self.__filter.read()
 				if not ret:
 					raise JailDefError("Unable to read the filter %r" % filterName)
-				if not filterOpt.get('logtype'):
-					# overwrite default logtype backend-related (considering that the filter settings may be overwritten):
-					self.__filter.merge_defaults({
-						'logtype': ['file','journal'][int(self.__opts.get('backend', '').startswith("systemd"))]
-					})
+				# set backend-related options (logtype):
+				self.__filter.applyAutoOptions(self.__opts.get('backend', ''))
 				# merge options from filter as 'known/...' (all options unfiltered):
 				self.__filter.getOptions(self.__opts, all=True)
 				ConfigReader.merge_section(self, self.__name, self.__filter.getCombined(), 'known/')
@@ -177,10 +168,10 @@ class JailReader(ConfigReader):
 					if not act:			  # skip empty actions
 						continue
 					# join with previous line if needed (consider possible new-line):
-					actName, actOpt = extractOptions(act)
-					prevln = ''
-					if not actName:
-						raise JailDefError("Invalid action definition %r" % act)
+					try:
+						actName, actOpt = extractOptions(act)
+					except ValueError as e:
+						raise JailDefError("Invalid action definition %r: %s" % (act, e))
 					if actName.endswith(".py"):
 						self.__actions.append([
 							"set",

fail2ban/helpers.py

@@ -208,6 +208,26 @@ class FormatterWithTraceBack(logging.Formatter):
 		return logging.Formatter.format(self, record)
 
 
+logging.exitOnIOError = False
+def __stopOnIOError(logSys=None, logHndlr=None): # pragma: no cover
+	if logSys and len(logSys.handlers):
+		logSys.removeHandler(logSys.handlers[0])
+	if logHndlr:
+		logHndlr.close = lambda: None
+	logging.StreamHandler.flush = lambda self: None
+	#sys.excepthook = lambda *args: None
+	if logging.exitOnIOError:
+		try:
+			sys.stderr.close()
+		except:
+			pass
+		sys.exit(0)
+
+try:
+	BrokenPipeError = BrokenPipeError
+except NameError: # pragma: 3.x no cover
+	BrokenPipeError = IOError
+
 __origLog = logging.Logger._log
 def __safeLog(self, level, msg, args, **kwargs):
 	"""Safe log inject to avoid possible errors by unsafe log-handlers, 
@@ -223,6 +243,10 @@ def __safeLog(self, level, msg, args, **kwargs):
 	try:
 		# if isEnabledFor(level) already called...
 		__origLog(self, level, msg, args, **kwargs)
+	except (BrokenPipeError, IOError) as e: # pragma: no cover
+		if e.errno == 32: # closed / broken pipe
+			__stopOnIOError(self)
+		raise
 	except Exception as e: # pragma: no cover - unreachable if log-handler safe in this python-version
 		try:
 			for args in (
@@ -237,6 +261,18 @@ def __safeLog(self, level, msg, args, **kwargs):
 			pass
 logging.Logger._log = __safeLog
 
+__origLogFlush = logging.StreamHandler.flush
+def __safeLogFlush(self):
+	"""Safe flush inject stopping endless logging on closed streams (redirected pipe).
+	"""
+	try:
+		__origLogFlush(self)
+	except (BrokenPipeError, IOError) as e: # pragma: no cover
+		if e.errno == 32: # closed / broken pipe
+			__stopOnIOError(None, self)
+		raise
+logging.StreamHandler.flush = __safeLogFlush
+
 def getLogger(name):
 	"""Get logging.Logger instance with Fail2Ban logger name convention
 	"""
@@ -267,7 +303,7 @@ def getVerbosityFormat(verbosity, fmt=' %(message)s', addtime=True, padding=True
 			if addtime:
 				fmt = ' %(asctime)-15s' + fmt
 	else: # default (not verbose):
-		fmt = "%(name)-23.23s [%(process)d]: %(levelname)-7s" + fmt
+		fmt = "%(name)-24s[%(process)d]: %(levelname)-7s" + fmt
 		if addtime:
 			fmt = "%(asctime)s " + fmt
 	# remove padding if not needed:
@@ -291,7 +327,7 @@ def splitwords(s):
 	"""
 	if not s:
 		return []
-	return filter(bool, map(str.strip, re.split('[ ,\n]+', s)))
+	return filter(bool, map(lambda v: v.strip(), re.split('[ ,\n]+', s)))
 
 if sys.version_info >= (3,5):
 	eval(compile(r'''if 1:
@@ -335,21 +371,27 @@ OPTION_CRE = re.compile(r"^([^\[]+)(?:\[(.*)\])?\s*$", re.DOTALL)
 # since v0.10 separator extended with `]\s*[` for support of multiple option groups, syntax 
 # `action = act[p1=...][p2=...]`
 OPTION_EXTRACT_CRE = re.compile(
-	r'([\w\-_\.]+)=(?:"([^"]*)"|\'([^\']*)\'|([^,\]]*))(?:,|\]\s*\[|$)', re.DOTALL)
+	r'\s*([\w\-_\.]+)=(?:"([^"]*)"|\'([^\']*)\'|([^,\]]*))(?:,|\]\s*\[|$|(?P<wrngA>.+))|,?\s*$|(?P<wrngB>.+)', re.DOTALL)
 # split by new-line considering possible new-lines within options [...]:
 OPTION_SPLIT_CRE = re.compile(
-	r'(?:[^\[\n]+(?:\s*\[\s*(?:[\w\-_\.]+=(?:"[^"]*"|\'[^\']*\'|[^,\]]*)\s*(?:,|\]\s*\[)?\s*)*\])?\s*|[^\n]+)(?=\n\s*|$)', re.DOTALL)
+	r'(?:[^\[\s]+(?:\s*\[\s*(?:[\w\-_\.]+=(?:"[^"]*"|\'[^\']*\'|[^,\]]*)\s*(?:,|\]\s*\[)?\s*)*\])?\s*|\S+)(?=\n\s*|\s+|$)', re.DOTALL)
 
 def extractOptions(option):
 	match = OPTION_CRE.match(option)
 	if not match:
-		# TODO proper error handling
-		return None, None
+		raise ValueError("unexpected option syntax")
 	option_name, optstr = match.groups()
 	option_opts = dict()
 	if optstr:
 		for optmatch in OPTION_EXTRACT_CRE.finditer(optstr):
+			if optmatch.group("wrngA"):
+				raise ValueError("unexpected syntax at %d after option %r: %s" % (
+					optmatch.start("wrngA"), optmatch.group(1), optmatch.group("wrngA")[0:25]))
+			if optmatch.group("wrngB"):
+				raise ValueError("expected option, wrong syntax at %d: %s" % (
+					optmatch.start("wrngB"), optmatch.group("wrngB")[0:25]))
 			opt = optmatch.group(1)
+			if not opt: continue
 			value = [
 				val for val in optmatch.group(2,3,4) if val is not None][0]
 			option_opts[opt.strip()] = value.strip()
@@ -363,8 +405,8 @@ def splitWithOptions(option):
 # tags (<tag>) in tagged options.
 #
 
-# max tag replacement count:
-MAX_TAG_REPLACE_COUNT = 10
+# max tag replacement count (considering tag X in tag Y repeat):
+MAX_TAG_REPLACE_COUNT = 25
 
 # compiled RE for tag name (replacement name) 
 TAG_CRE = re.compile(r'<([^ <>]+)>')
@@ -398,6 +440,7 @@ def substituteRecursiveTags(inptags, conditional='',
 	done = set()
 	noRecRepl = hasattr(tags, "getRawItem")
 	# repeat substitution while embedded-recursive (repFlag is True)
+	repCounts = {}
 	while True:
 		repFlag = False
 		# substitute each value:
@@ -409,7 +452,7 @@ def substituteRecursiveTags(inptags, conditional='',
 			value = orgval = uni_string(tags[tag])
 			# search and replace all tags within value, that can be interpolated using other tags:
 			m = tre_search(value)
-			refCounts = {}
+			rplc = repCounts.get(tag, {})
 			#logSys.log(5, 'TAG: %s, value: %s' % (tag, value))
 			while m:
 				# found replacement tag:
@@ -419,13 +462,13 @@ def substituteRecursiveTags(inptags, conditional='',
 					m = tre_search(value, m.end())
 					continue
 				#logSys.log(5, 'found: %s' % rtag)
-				if rtag == tag or refCounts.get(rtag, 1) > MAX_TAG_REPLACE_COUNT:
+				if rtag == tag or rplc.get(rtag, 1) > MAX_TAG_REPLACE_COUNT:
 					# recursive definitions are bad
 					#logSys.log(5, 'recursion fail tag: %s value: %s' % (tag, value) )
 					raise ValueError(
 						"properties contain self referencing definitions "
 						"and cannot be resolved, fail tag: %s, found: %s in %s, value: %s" % 
-						(tag, rtag, refCounts, value))
+						(tag, rtag, rplc, value))
 				repl = None
 				if conditional:
 					repl = tags.get(rtag + '?' + conditional)
@@ -445,7 +488,7 @@ def substituteRecursiveTags(inptags, conditional='',
 				value = value.replace('<%s>' % rtag, repl)
 				#logSys.log(5, 'value now: %s' % value)
 				# increment reference count:
-				refCounts[rtag] = refCounts.get(rtag, 0) + 1
+				rplc[rtag] = rplc.get(rtag, 0) + 1
 				# the next match for replace:
 				m = tre_search(value, m.start())
 			#logSys.log(5, 'TAG: %s, newvalue: %s' % (tag, value))
@@ -453,6 +496,7 @@ def substituteRecursiveTags(inptags, conditional='',
 			if orgval != value:
 				# check still contains any tag - should be repeated (possible embedded-recursive substitution):
 				if tre_search(value):
+					repCounts[tag] = rplc
 					repFlag = True
 				# copy return tags dict to prevent modifying of inptags:
 				if id(tags) == id(inptags):

fail2ban/protocol.py

@@ -55,6 +55,8 @@ protocol = [
 ["stop", "stops all jails and terminate the server"], 
 ["unban --all", "unbans all IP addresses (in all jails and database)"],
 ["unban <IP> ... <IP>", "unbans <IP> (in all jails and database)"],
+["banned", "return jails with banned IPs as dictionary"],
+["banned <IP> ... <IP>]", "return list(s) of jails where given IP(s) are banned"],
 ["status", "gets the current status of the server"], 
 ["ping", "tests if the server is alive"],
 ["echo", "for internal usage, returns back and outputs a given string"],
@@ -120,6 +122,8 @@ protocol = [
 ["set <JAIL> action <ACT> <PROPERTY> <VALUE>", "sets the <VALUE> of <PROPERTY> for the action <ACT> for <JAIL>"],
 ["set <JAIL> action <ACT> <METHOD>[ <JSONKWARGS>]", "calls the <METHOD> with <JSONKWARGS> for the action <ACT> for <JAIL>"],
 ['', "JAIL INFORMATION", ""],
+["get <JAIL> banned", "return banned IPs of <JAIL>"],
+["get <JAIL> banned <IP> ... <IP>]", "return 1 if IP is banned in <JAIL> otherwise 0, or a list of 1/0 for multiple IPs"],
 ["get <JAIL> logpath", "gets the list of the monitored files for <JAIL>"],
 ["get <JAIL> logencoding", "gets the encoding of the log files for <JAIL>"],
 ["get <JAIL> journalmatch", "gets the journal filter match for <JAIL>"],
@@ -132,7 +136,6 @@ protocol = [
 ["get <JAIL> bantime", "gets the time a host is banned for <JAIL>"],
 ["get <JAIL> datepattern", "gets the patern used to match date/times for <JAIL>"],
 ["get <JAIL> usedns", "gets the usedns setting for <JAIL>"],
-["get <JAIL> banip [<SEP>|--with-time]", "gets the list of of banned IP addresses for <JAIL>. Optionally the separator character ('<SEP>', default is space) or the option '--with-time' (printing the times of ban) may be specified. The IPs are ordered by end of ban."],
 ["get <JAIL> maxretry", "gets the number of failures allowed for <JAIL>"],
 ["get <JAIL> maxmatches", "gets the max number of matches stored in memory per ticket in <JAIL>"], 
 ["get <JAIL> maxlines", "gets the number of lines to buffer for <JAIL>"],

fail2ban/server/action.py

@@ -30,7 +30,10 @@ import tempfile
 import threading
 import time
 from abc import ABCMeta
-from collections import MutableMapping
+try:
+	from collections.abc import MutableMapping
+except ImportError:
+	from collections import MutableMapping
 
 from .failregex import mapTag2Opt
 from .ipdns import DNSUtils
@@ -262,10 +265,6 @@ class ActionBase(object):
 		"""
 		return self.ban(aInfo)
 
-	@property
-	def _prolongable(self): # pragma: no cover - abstract
-		return False
-
 	def unban(self, aInfo): # pragma: no cover - abstract
 		"""Executed when a ban expires.
 
@@ -278,11 +277,6 @@ class ActionBase(object):
 		pass
 
 
-WRAP_CMD_PARAMS = {
-	'timeout': 'str2seconds',
-	'bantime': 'ignore',
-}
-
 class CommandAction(ActionBase):
 	"""A action which executes OS shell commands.
 
@@ -355,10 +349,7 @@ class CommandAction(ActionBase):
 	def __setattr__(self, name, value):
 		if not name.startswith('_') and not self.__init and not callable(value):
 			# special case for some parameters:
-			wrp = WRAP_CMD_PARAMS.get(name)
-			if wrp == 'ignore': # ignore (filter) dynamic parameters
-				return
-			elif wrp == 'str2seconds':
+			if name in ('timeout', 'bantime'):
 				value = MyTime.str2seconds(value)
 			# parameters changed - clear properties and substitution cache:
 			self.__properties = None
@@ -404,10 +395,13 @@ class CommandAction(ActionBase):
 	def _getOperation(self, tag, family):
 		# replace operation tag (interpolate all values), be sure family is enclosed as conditional value
 		# (as lambda in addrepl so only if not overwritten in action):
-		return self.replaceTag(tag, self._properties,
+		cmd = self.replaceTag(tag, self._properties,
 			conditional=('family='+family if family else ''),
-			addrepl=(lambda tag:family if tag == 'family' else None),
 			cache=self.__substCache)
+		if '<' not in cmd or not family: return cmd
+		# replace family as dynamic tags, important - don't cache, no recursion and auto-escape here:
+		cmd = self.replaceDynamicTags(cmd, {'family':family})
+		return cmd
 
 	def _operationExecuted(self, tag, family, *args):
 		""" Get, set or delete command of operation considering family.
@@ -552,26 +546,6 @@ class CommandAction(ActionBase):
 			raise RuntimeError("Error banning %(ip)s" % aInfo)
 		self.__started[family] = self.__started.get(family, 0) | 3; # started and contains items
 
-	@property
-	def _prolongable(self):
-		return (hasattr(self, 'actionprolong') and self.actionprolong 
-			and not str(self.actionprolong).isspace())
-	
-	def prolong(self, aInfo):
-		"""Executes the "actionprolong" command.
-
-		Replaces the tags in the action command with actions properties
-		and ban information, and executes the resulting command.
-
-		Parameters
-		----------
-		aInfo : dict
-			Dictionary which includes information in relation to
-			the ban.
-		"""
-		if not self._processCmd('<actionprolong>', aInfo):
-			raise RuntimeError("Error prolonging %(ip)s" % aInfo)
-
 	def unban(self, aInfo):
 		"""Executes the "actionunban" command.
 
@@ -681,10 +655,8 @@ class CommandAction(ActionBase):
 					ret &= False
 		return ret
 
-	ESCAPE_CRE = re.compile(r"""[\\#&;`|*?~<>^()\[\]{}$'"\n\r]""")
-	
-	@classmethod
-	def escapeTag(cls, value):
+	@staticmethod
+	def escapeTag(value):
 		"""Escape characters which may be used for command injection.
 
 		Parameters
@@ -701,15 +673,12 @@ class CommandAction(ActionBase):
 		-----
 		The following characters are escaped::
 
-			\\#&;`|*?~<>^()[]{}$'"\n\r
+			\\#&;`|*?~<>^()[]{}$'"
 
 		"""
-		_map2c = {'\n': 'n', '\r': 'r'}
-		def substChar(m):
-			c = m.group()
-			return '\\' + _map2c.get(c, c)
-		
-		value = cls.ESCAPE_CRE.sub(substChar, value)
+		for c in '\\#&;`|*?~<>^()[]{}$\'"':
+			if c in value:
+				value = value.replace(c, '\\' + c)
 		return value
 
 	@classmethod
@@ -806,7 +775,7 @@ class CommandAction(ActionBase):
 	ESCAPE_VN_CRE = re.compile(r"\W")
 
 	@classmethod
-	def replaceDynamicTags(cls, realCmd, aInfo):
+	def replaceDynamicTags(cls, realCmd, aInfo, escapeVal=None):
 		"""Replaces dynamical tags in `query` with property values.
 
 		**Important**
@@ -831,16 +800,17 @@ class CommandAction(ActionBase):
 		# array for escaped vars:
 		varsDict = dict()
 
-		def escapeVal(tag, value):
-			# if the value should be escaped:
-			if cls.ESCAPE_CRE.search(value):
-				# That one needs to be escaped since its content is
-				# out of our control
-				tag = 'f2bV_%s' % cls.ESCAPE_VN_CRE.sub('_', tag)
-				varsDict[tag] = value # add variable
-				value = '$'+tag	# replacement as variable
-			# replacement for tag:
-			return value
+		if not escapeVal:
+			def escapeVal(tag, value):
+				# if the value should be escaped:
+				if cls.ESCAPE_CRE.search(value):
+					# That one needs to be escaped since its content is
+					# out of our control
+					tag = 'f2bV_%s' % cls.ESCAPE_VN_CRE.sub('_', tag)
+					varsDict[tag] = value # add variable
+					value = '$'+tag	# replacement as variable
+				# replacement for tag:
+				return value
 
 		# additional replacement as calling map:
 		ADD_REPL_TAGS_CM = CallingMap(ADD_REPL_TAGS)
@@ -1008,8 +978,7 @@ class CommandAction(ActionBase):
 		RuntimeError
 			If command execution times out.
 		"""
-		if logSys.getEffectiveLevel() < logging.DEBUG:
-			logSys.log(9, realCmd)
+		logSys.debug(realCmd)
 		if not realCmd:
 			logSys.debug("Nothing to do")
 			return True

fail2ban/server/actions.py

@@ -28,7 +28,10 @@ import logging
 import os
 import sys
 import time
-from collections import Mapping
+try:
+	from collections.abc import Mapping
+except ImportError:
+	from collections import Mapping
 try:
 	from collections import OrderedDict
 except ImportError:
@@ -39,7 +42,6 @@ from .ipdns import IPAddr
 from .jailthread import JailThread
 from .action import ActionBase, CommandAction, CallingMap
 from .mytime import MyTime
-from .observer import Observers
 from .utils import Utils
 from ..helpers import getLogger
 
@@ -211,15 +213,13 @@ class Actions(JailThread, Mapping):
 	def getBanTime(self):
 		return self.__banManager.getBanTime()
 
-	def getBanList(self, withTime=False):
-		"""Returns the list of banned IP addresses.
-
-		Returns
-		-------
-		list
-			The list of banned IP addresses.
-		"""
-		return self.__banManager.getBanList(ordered=True, withTime=withTime)
+	def getBanned(self, ids):
+		lst = self.__banManager.getBanList()
+		if not ids:
+			return lst
+		if len(ids) == 1:
+			return 1 if ids[0] in lst else 0
+		return map(lambda ip: 1 if ip in lst else 0, ids)
 
 	def addBannedIP(self, ip):
 		"""Ban an IP or list of IPs."""
@@ -254,7 +254,7 @@ class Actions(JailThread, Mapping):
 		if ip is None:
 			return self.__flushBan(db)
 		# Multiple IPs:
-		if isinstance(ip, list):
+		if isinstance(ip, (list, tuple)):
 			missed = []
 			cnt = 0
 			for i in ip:
@@ -276,6 +276,14 @@ class Actions(JailThread, Mapping):
 			# Unban the IP.
 			self.__unBan(ticket)
 		else:
+			# Multiple IPs by subnet or dns:
+			if not isinstance(ip, IPAddr):
+				ipa = IPAddr(ip)
+				if not ipa.isSingle: # subnet (mask/cidr) or raw (may be dns/hostname):
+					ips = filter(ipa.contains, self.__banManager.getBanList())
+					if ips:
+						return self.removeBannedIP(ips, db, ifexists)
+			# not found:
 			msg = "%s is not banned" % ip
 			logSys.log(logging.MSG, msg)
 			if ifexists:
@@ -322,23 +330,33 @@ class Actions(JailThread, Mapping):
 					self._jail.name, name, e,
 					exc_info=logSys.getEffectiveLevel()<=logging.DEBUG)
 		while self.active:
-			if self.idle:
-				logSys.debug("Actions: enter idle mode")
-				Utils.wait_for(lambda: not self.active or not self.idle,
-					lambda: False, self.sleeptime)
-				logSys.debug("Actions: leave idle mode")
-				continue
-			# wait for ban (stop if gets inactive):
-			bancnt = Utils.wait_for(lambda: not self.active or self.__checkBan(), self.sleeptime)
-			cnt += bancnt
-			# unban if nothing is banned not later than banned tickets >= banPrecedence
-			if not bancnt or cnt >= self.banPrecedence:
-				if self.active:
-					# let shrink the ban list faster
-					bancnt *= 2
-					self.__checkUnBan(bancnt if bancnt and bancnt < self.unbanMaxCount else self.unbanMaxCount)
-				cnt = 0
-		
+			try:
+				if self.idle:
+					logSys.debug("Actions: enter idle mode")
+					Utils.wait_for(lambda: not self.active or not self.idle,
+						lambda: False, self.sleeptime)
+					logSys.debug("Actions: leave idle mode")
+					continue
+				# wait for ban (stop if gets inactive, pending ban or unban):
+				bancnt = 0
+				wt = min(self.sleeptime, self.__banManager._nextUnbanTime - MyTime.time())
+				logSys.log(5, "Actions: wait for pending tickets %s (default %s)", wt, self.sleeptime)
+				if Utils.wait_for(lambda: not self.active or self._jail.hasFailTickets, wt):
+					bancnt = self.__checkBan()
+					cnt += bancnt
+				# unban if nothing is banned not later than banned tickets >= banPrecedence
+				if not bancnt or cnt >= self.banPrecedence:
+					if self.active:
+						# let shrink the ban list faster
+						bancnt *= 2
+						logSys.log(5, "Actions: check-unban %s, bancnt %s, max: %s", bancnt if bancnt and bancnt < self.unbanMaxCount else self.unbanMaxCount, bancnt, self.unbanMaxCount)
+						self.__checkUnBan(bancnt if bancnt and bancnt < self.unbanMaxCount else self.unbanMaxCount)
+					cnt = 0
+			except Exception as e: # pragma: no cover
+				logSys.error("[%s] unhandled error in actions thread: %s",
+					self._jail.name, e,
+					exc_info=logSys.getEffectiveLevel()<=logging.DEBUG)
+
 		self.__flushBan(stop=True)
 		self.stopActions()
 		return True
@@ -355,8 +373,6 @@ class Actions(JailThread, Mapping):
 			"fid":			lambda self: self.__ticket.getID(),
 			"failures":	lambda self: self.__ticket.getAttempt(),
 			"time":			lambda self: self.__ticket.getTime(),
-			"bantime":  lambda self: self._getBanTime(),
-			"bancount":  lambda self: self.__ticket.getBanCount(),
 			"matches":	lambda self: "\n".join(self.__ticket.getMatches()),
 			# to bypass actions, that should not be executed for restored tickets
 			"restored":	lambda self: (1 if self.__ticket.restored else 0),
@@ -383,11 +399,6 @@ class Actions(JailThread, Mapping):
 		def copy(self): # pragma: no cover
 			return self.__class__(self.__ticket, self.__jail, self.immutable, self.data.copy())
 
-		def _getBanTime(self):
-			btime = self.__ticket.getBanTime()
-			if btime is None: btime = self.__jail.actions.getBanTime()
-			return int(btime)
-
 		def _mi4ip(self, overalljails=False):
 			"""Gets bans merged once, a helper for lambda(s), prevents stop of executing action by any exception inside.
 
@@ -461,22 +472,16 @@ class Actions(JailThread, Mapping):
 			tickets = self.__getFailTickets(self.banPrecedence)
 		rebanacts = None
 		for ticket in tickets:
-
-			bTicket = BanTicket.wrap(ticket)
-			btime = ticket.getBanTime(self.__banManager.getBanTime())
+			bTicket = BanManager.createBanTicket(ticket)
 			ip = bTicket.getIP()
 			aInfo = self.__getActionInfo(bTicket)
 			reason = {}
 			if self.__banManager.addBanTicket(bTicket, reason=reason):
 				cnt += 1
-				# report ticket to observer, to check time should be increased and hereafter observer writes ban to database (asynchronous)
-				if Observers.Main is not None and not bTicket.restored:
-					Observers.Main.add('banFound', bTicket, self._jail, btime)
 				logSys.notice("[%s] %sBan %s", self._jail.name, ('' if not bTicket.restored else 'Restore '), ip)
-				# do actions :
 				for name, action in self._actions.iteritems():
 					try:
-						if ticket.restored and getattr(action, 'norestored', False):
+						if bTicket.restored and getattr(action, 'norestored', False):
 							continue
 						if not aInfo.immutable: aInfo.reset()
 						action.ban(aInfo)
@@ -491,10 +496,7 @@ class Actions(JailThread, Mapping):
 				if self.banEpoch: # be sure tickets always have the same ban epoch (default 0):
 					bTicket.banEpoch = self.banEpoch
 			else:
-				if reason.get('expired', 0):
-					logSys.info('[%s] Ignore %s, expired bantime', self._jail.name, ip)
-					continue
-				bTicket = reason.get('ticket', bTicket)
+				bTicket = reason['ticket']
 				# if already banned (otherwise still process some action)
 				if bTicket.banned:
 					# compare time of failure occurrence with time ticket was really banned:
@@ -522,6 +524,12 @@ class Actions(JailThread, Mapping):
 						cnt += self.__reBan(bTicket, actions=rebanacts)
 				else: # pragma: no cover - unexpected: ticket is not banned for some reasons - reban using all actions:
 					cnt += self.__reBan(bTicket)
+			# add ban to database:
+			if not bTicket.restored and self._jail.database is not None:
+				# ignore too old (repeated and ignored) tickets,
+				# [todo] replace it with inOperation later (once it gets back-ported):
+				if not reason and bTicket.getTime() >= MyTime.time() - 60:
+					self._jail.database.addBan(self._jail, bTicket)
 		if cnt:
 			logSys.debug("Banned %s / %s, %s ticket(s) in %r", cnt, 
 				self.__banManager.getBanTotal(), self.__banManager.size(), self._jail.name)
@@ -561,29 +569,6 @@ class Actions(JailThread, Mapping):
 			ticket.banEpoch = self.banEpoch
 		return 1
 
-	def _prolongBan(self, ticket):
-		# prevent to prolong ticket that was removed in-between,
-		# if it in ban list - ban time already prolonged (and it stays there):
-		if not self.__banManager._inBanList(ticket): return
-		# do actions :
-		aInfo = None
-		for name, action in self._actions.iteritems():
-			try:
-				if ticket.restored and getattr(action, 'norestored', False):
-					continue
-				if not action._prolongable:
-					continue
-				if aInfo is None:
-					aInfo = self.__getActionInfo(ticket)
-				if not aInfo.immutable: aInfo.reset()
-				action.prolong(aInfo)
-			except Exception as e:
-				logSys.error(
-					"Failed to execute ban jail '%s' action '%s' "
-					"info '%r': %s",
-					self._jail.name, name, aInfo, e,
-					exc_info=logSys.getEffectiveLevel()<=logging.DEBUG)
-
 	def __checkUnBan(self, maxCount=None):
 		"""Check for IP address to unban.
 
@@ -687,13 +672,19 @@ class Actions(JailThread, Mapping):
 		"""Status of current and total ban counts and current banned IP list.
 		"""
 		# TODO: Allow this list to be printed as 'status' output
-		supported_flavors = ["basic", "cymru"]
+		supported_flavors = ["short", "basic", "cymru"]
 		if flavor is None or flavor not in supported_flavors:
 			logSys.warning("Unsupported extended jail status flavor %r. Supported: %s" % (flavor, supported_flavors))
 		# Always print this information (basic)
-		ret = [("Currently banned", self.__banManager.size()),
-			   ("Total banned", self.__banManager.getBanTotal()),
-			   ("Banned IP list", self.__banManager.getBanList())]
+		if flavor != "short":
+			banned = self.__banManager.getBanList()
+			cnt = len(banned)
+		else:
+			cnt = self.__banManager.size()
+		ret = [("Currently banned", cnt),
+			   ("Total banned", self.__banManager.getBanTotal())]
+		if flavor != "short":
+			ret += [("Banned IP list", banned)]
 		if flavor == "cymru":
 			cymru_info = self.__banManager.getBanListExtendedCymruInfo()
 			ret += \

fail2ban/server/banmanager.py

@@ -57,7 +57,7 @@ class BanManager:
 		## Total number of banned IP address
 		self.__banTotal = 0
 		## The time for next unban process (for performance and load reasons):
-		self.__nextUnbanTime = BanTicket.MAX_TIME
+		self._nextUnbanTime = BanTicket.MAX_TIME
 	
 	##
 	# Set the ban time.
@@ -66,8 +66,7 @@ class BanManager:
 	# @param value the time
 	
 	def setBanTime(self, value):
-		with self.__lock:
-			self.__banTime = int(value)
+		self.__banTime = int(value)
 	
 	##
 	# Get the ban time.
@@ -76,8 +75,7 @@ class BanManager:
 	# @return the time
 	
 	def getBanTime(self):
-		with self.__lock:
-			return self.__banTime
+		return self.__banTime
 	
 	##
 	# Set the total number of banned address.
@@ -85,8 +83,7 @@ class BanManager:
 	# @param value total number
 	
 	def setBanTotal(self, value):
-		with self.__lock:
-			self.__banTotal = value
+		self.__banTotal = value
 	
 	##
 	# Get the total number of banned address.
@@ -94,30 +91,15 @@ class BanManager:
 	# @return the total number
 	
 	def getBanTotal(self):
-		with self.__lock:
-			return self.__banTotal
+		return self.__banTotal
 
 	##
 	# Returns a copy of the IP list.
 	#
 	# @return IP list
 	
-	def getBanList(self, ordered=False, withTime=False):
-		with self.__lock:
-			if not ordered:
-				return self.__banList.keys()
-			lst = []
-			for ticket in self.__banList.itervalues():
-				eob = ticket.getEndOfBanTime(self.__banTime)
-				lst.append((ticket,eob))
-			lst.sort(key=lambda t: t[1])
-			t2s = MyTime.time2str
-			if withTime:
-				return ['%s \t%s + %d = %s' % (
-						t[0].getID(), 
-						t2s(t[0].getTime()), t[0].getBanTime(self.__banTime), t2s(t[1])
-					) for t in lst]
-			return [t[0].getID() for t in lst]
+	def getBanList(self):
+		return list(self.__banList.keys())
 
 	##
 	# Returns a iterator to ban list (used in reload, so idle).
@@ -125,8 +107,8 @@ class BanManager:
 	# @return ban list iterator
 	
 	def __iter__(self):
-		with self.__lock:
-			return self.__banList.itervalues()
+		# ensure iterator is safe - traverse over the list in snapshot created within lock (GIL):
+		return iter(list(self.__banList.values()))
 
 	##
 	# Returns normalized value
@@ -262,6 +244,21 @@ class BanManager:
 			logSys.exception(e)
 			return []
 
+	##
+	# Create a ban ticket.
+	#
+	# Create a BanTicket from a FailTicket. The timestamp of the BanTicket
+	# is the current time. This is a static method.
+	# @param ticket the FailTicket
+	# @return a BanTicket
+	
+	@staticmethod
+	def createBanTicket(ticket):
+		# we should always use correct time to calculate correct end time (ban time is variable now, 
+		# + possible double banning by restore from database and from log file)
+		# so use as lastTime always time from ticket.
+		return BanTicket(ticket=ticket)
+	
 	##
 	# Add a ban ticket.
 	#
@@ -271,9 +268,6 @@ class BanManager:
 	
 	def addBanTicket(self, ticket, reason={}):
 		eob = ticket.getEndOfBanTime(self.__banTime)
-		if eob < MyTime.time():
-			reason['expired'] = 1
-			return False
 		with self.__lock:
 			# check already banned
 			fid = ticket.getID()
@@ -295,10 +289,9 @@ class BanManager:
 			# not yet banned - add new one:
 			self.__banList[fid] = ticket
 			self.__banTotal += 1
-			ticket.incrBanCount()
 			# correct next unban time:
-			if self.__nextUnbanTime > eob:
-				self.__nextUnbanTime = eob
+			if self._nextUnbanTime > eob:
+				self._nextUnbanTime = eob
 			return True
 
 	##
@@ -329,12 +322,8 @@ class BanManager:
 	
 	def unBanList(self, time, maxCount=0x7fffffff):
 		with self.__lock:
-			# Permanent banning
-			if self.__banTime < 0:
-				return list()
-
 			# Check next unban time:
-			nextUnbanTime = self.__nextUnbanTime
+			nextUnbanTime = self._nextUnbanTime
 			if nextUnbanTime > time:
 				return list()
 
@@ -347,12 +336,12 @@ class BanManager:
 				if time > eob:
 					unBanList[fid] = ticket
 					if len(unBanList) >= maxCount: # stop search cycle, so reset back the next check time
-						nextUnbanTime = self.__nextUnbanTime
+						nextUnbanTime = self._nextUnbanTime
 						break
 				elif nextUnbanTime > eob:
 					nextUnbanTime = eob
 
-			self.__nextUnbanTime = nextUnbanTime
+			self._nextUnbanTime = nextUnbanTime
 			# Removes tickets.
 			if len(unBanList):
 				if len(unBanList) / 2.0 <= len(self.__banList) / 3.0:

fail2ban/server/database.py

@@ -138,7 +138,7 @@ class Fail2BanDb(object):
 	filename
 	purgeage
 	"""
-	__version__ = 4
+	__version__ = 2
 	# Note all SCRIPTS strings must end in ';' for py26 compatibility
 	_CREATE_SCRIPTS = (
 		 ('fail2banDb', "CREATE TABLE IF NOT EXISTS fail2banDb(version INTEGER);")
@@ -166,36 +166,21 @@ class Fail2BanDb(object):
 			"jail TEXT NOT NULL, " \
 			"ip TEXT, " \
 			"timeofban INTEGER NOT NULL, " \
-			"bantime INTEGER NOT NULL, " \
-			"bancount INTEGER NOT NULL default 1, " \
 			"data JSON, " \
 			"FOREIGN KEY(jail) REFERENCES jails(name) " \
 			");" \
 			"CREATE INDEX IF NOT EXISTS bans_jail_timeofban_ip ON bans(jail, timeofban);" \
 			"CREATE INDEX IF NOT EXISTS bans_jail_ip ON bans(jail, ip);" \
 			"CREATE INDEX IF NOT EXISTS bans_ip ON bans(ip);")
-		,('bips', "CREATE TABLE IF NOT EXISTS bips(" \
-			"ip TEXT NOT NULL, " \
-			"jail TEXT NOT NULL, " \
-			"timeofban INTEGER NOT NULL, " \
-			"bantime INTEGER NOT NULL, " \
-			"bancount INTEGER NOT NULL default 1, " \
-			"data JSON, " \
-			"PRIMARY KEY(ip, jail), " \
-			"FOREIGN KEY(jail) REFERENCES jails(name) " \
-			");" \
-			"CREATE INDEX IF NOT EXISTS bips_timeofban ON bips(timeofban);" \
-			"CREATE INDEX IF NOT EXISTS bips_ip ON bips(ip);")
 	)
 	_CREATE_TABS = dict(_CREATE_SCRIPTS)
 
 
-	def __init__(self, filename, purgeAge=24*60*60, outDatedFactor=3):
+	def __init__(self, filename, purgeAge=24*60*60):
 		self.maxMatches = 10
 		self._lock = RLock()
 		self._dbFilename = filename
 		self._purgeAge = purgeAge
-		self._outDatedFactor = outDatedFactor;
 		self._connectDB()
 
 	def _connectDB(self, checkIntegrity=False):
@@ -381,32 +366,13 @@ class Fail2BanDb(object):
 
 			if version < 2 and self._tableExists(cur, "logs"):
 				cur.executescript("BEGIN TRANSACTION;"
-							"CREATE TEMPORARY TABLE logs_temp AS SELECT * FROM logs;"
-							"DROP TABLE logs;"
-							"%s;"
-							"INSERT INTO logs SELECT * from logs_temp;"
-							"DROP TABLE logs_temp;"
-							"UPDATE fail2banDb SET version = 2;"
-							"COMMIT;" % Fail2BanDb._CREATE_TABS['logs'])
-
-			if version < 3 and self._tableExists(cur, "bans"):
-				# set ban-time to -2 (note it means rather unknown, as persistent, will be fixed by restore):
-				cur.executescript("BEGIN TRANSACTION;"
-							"CREATE TEMPORARY TABLE bans_temp AS SELECT jail, ip, timeofban, -2 as bantime, 1 as bancount, data FROM bans;"
-							"DROP TABLE bans;"
-							"%s;\n"
-							"INSERT INTO bans SELECT * from bans_temp;"
-							"DROP TABLE bans_temp;"
-							"COMMIT;" % Fail2BanDb._CREATE_TABS['bans'])
-			if version < 4 and not self._tableExists(cur, "bips"):
-				cur.executescript("BEGIN TRANSACTION;"
-							"%s;\n"
-							"UPDATE fail2banDb SET version = 4;"
-							"COMMIT;" % Fail2BanDb._CREATE_TABS['bips'])
-				if self._tableExists(cur, "bans"):
-					cur.execute(
-							"INSERT OR REPLACE INTO bips(ip, jail, timeofban, bantime, bancount, data)"
-							"  SELECT ip, jail, timeofban, bantime, bancount, data FROM bans order by timeofban")
+						"CREATE TEMPORARY TABLE logs_temp AS SELECT * FROM logs;"
+						"DROP TABLE logs;"
+						"%s;"
+						"INSERT INTO logs SELECT * from logs_temp;"
+						"DROP TABLE logs_temp;"
+						"UPDATE fail2banDb SET version = 2;"
+						"COMMIT;" % Fail2BanDb._CREATE_TABS['logs'])
 
 			cur.execute("SELECT version FROM fail2banDb LIMIT 1")
 			return cur.fetchone()[0]
@@ -489,22 +455,24 @@ class Fail2BanDb(object):
 			If log was already present in database, value of last position
 			in the log file; else `None`
 		"""
+		return self._addLog(cur, jail, container.getFileName(), container.getPos(), container.getHash())
+
+	def _addLog(self, cur, jail, name, pos=0, md5=None):
 		lastLinePos = None
 		cur.execute(
 			"SELECT firstlinemd5, lastfilepos FROM logs "
 				"WHERE jail=? AND path=?",
-			(jail.name, container.getFileName()))
+			(jail.name, name))
 		try:
 			firstLineMD5, lastLinePos = cur.fetchone()
 		except TypeError:
-			firstLineMD5 = False
+			firstLineMD5 = None
 
-		cur.execute(
-				"INSERT OR REPLACE INTO logs(jail, path, firstlinemd5, lastfilepos) "
-					"VALUES(?, ?, ?, ?)",
-				(jail.name, container.getFileName(),
-					container.getHash(), container.getPos()))
-		if container.getHash() != firstLineMD5:
+		if not firstLineMD5 and (pos or md5):
+			cur.execute(
+					"INSERT OR REPLACE INTO logs(jail, path, firstlinemd5, lastfilepos) "
+						"VALUES(?, ?, ?, ?)", (jail.name, name, md5, pos))
+		if md5 is not None and md5 != firstLineMD5:
 			lastLinePos = None
 		return lastLinePos
 
@@ -533,7 +501,7 @@ class Fail2BanDb(object):
 		return set(row[0] for row in cur.fetchmany())
 
 	@commitandrollback
-	def updateLog(self, cur, *args, **kwargs):
+	def updateLog(self, cur, jail, container):
 		"""Updates hash and last position in log file.
 
 		Parameters
@@ -543,14 +511,48 @@ class Fail2BanDb(object):
 		container : FileContainer
 			File container of the log file being updated.
 		"""
-		self._updateLog(cur, *args, **kwargs)
+		self._updateLog(cur, jail, container.getFileName(), container.getPos(), container.getHash())
 
-	def _updateLog(self, cur, jail, container):
+	def _updateLog(self, cur, jail, name, pos, md5):
 		cur.execute(
 			"UPDATE logs SET firstlinemd5=?, lastfilepos=? "
-				"WHERE jail=? AND path=?",
-			(container.getHash(), container.getPos(),
-				jail.name, container.getFileName()))
+				"WHERE jail=? AND path=?", (md5, pos, jail.name, name))
+		# be sure it is set (if not available):
+		if not cur.rowcount:
+			cur.execute(
+					"INSERT OR REPLACE INTO logs(jail, path, firstlinemd5, lastfilepos) "
+						"VALUES(?, ?, ?, ?)", (jail.name, name, md5, pos))
+
+	@commitandrollback
+	def getJournalPos(self, cur, jail, name, time=0, iso=None):
+		"""Get journal position from database.
+
+		Parameters
+		----------
+		jail : Jail
+			Jail of which the journal belongs to.
+		name, time, iso :
+			Journal name (typically systemd-journal) and last known time.
+
+		Returns
+		-------
+		int (or float)
+			Last position (as time) if it was already present in database; else `None`
+		"""
+		return self._addLog(cur, jail, name, time, iso); # no hash, just time as iso
+
+	@commitandrollback
+	def updateJournal(self, cur, jail, name, time, iso):
+		"""Updates last position (as time) of journal.
+
+		Parameters
+		----------
+		jail : Jail
+			Jail of which the journal belongs to.
+		name, time, iso :
+			Journal name (typically systemd-journal) and last known time.
+		"""
+		self._updateLog(cur, jail, name, time, iso); # no hash, just time as iso
 
 	@commitandrollback
 	def addBan(self, cur, jail, ticket):
@@ -583,13 +585,8 @@ class Fail2BanDb(object):
 			data = data.copy()
 			del data['matches']
 		cur.execute(
-			"INSERT INTO bans(jail, ip, timeofban, bantime, bancount, data) VALUES(?, ?, ?, ?, ?, ?)",
-			(jail.name, ip, int(round(ticket.getTime())), ticket.getBanTime(jail.actions.getBanTime()), ticket.getBanCount(),
-				data))
-		cur.execute(
-			"INSERT OR REPLACE INTO bips(ip, jail, timeofban, bantime, bancount, data) VALUES(?, ?, ?, ?, ?, ?)",
-			(ip, jail.name, int(round(ticket.getTime())), ticket.getBanTime(jail.actions.getBanTime()), ticket.getBanCount(),
-				data))
+			"INSERT INTO bans(jail, ip, timeofban, data) VALUES(?, ?, ?, ?)",
+			(jail.name, ip, int(round(ticket.getTime())), data))
 
 	@commitandrollback
 	def delBan(self, cur, jail, *args):
@@ -602,20 +599,16 @@ class Fail2BanDb(object):
 		args : list of IP
 			IPs to be removed, if not given all tickets of jail will be removed.
 		"""
-		query1 = "DELETE FROM bips WHERE jail = ?"
-		query2 = "DELETE FROM bans WHERE jail = ?"
+		query = "DELETE FROM bans WHERE jail = ?"
 		queryArgs = [jail.name];
 		if not len(args):
-			cur.execute(query1, queryArgs);
-			cur.execute(query2, queryArgs);
+			cur.execute(query, queryArgs);
 			return
-		query1 += " AND ip = ?"
-		query2 += " AND ip = ?"
+		query += " AND ip = ?"
 		queryArgs.append('');
 		for ip in args:
 			queryArgs[1] = str(ip);
-			cur.execute(query1, queryArgs);
-			cur.execute(query2, queryArgs);
+			cur.execute(query, queryArgs);
 
 	@commitandrollback
 	def _getBans(self, cur, jail=None, bantime=None, ip=None):
@@ -733,41 +726,18 @@ class Fail2BanDb(object):
 				self._bansMergedCache[cacheKey] = tickets if ip is None else ticket
 			return tickets if ip is None else ticket
 
-	@commitandrollback
-	def getBan(self, cur, ip, jail=None, forbantime=None, overalljails=None, fromtime=None):
-		ip = str(ip)
-		if not overalljails:
-			query = "SELECT bancount, timeofban, bantime FROM bips"
-		else:
-			query = "SELECT sum(bancount), max(timeofban), sum(bantime) FROM bips"
-		query += " WHERE ip = ?"
-		queryArgs = [ip]
-		if not overalljails and jail is not None:
-			query += " AND jail=?"
-			queryArgs.append(jail.name)
-		if forbantime is not None:
-			query += " AND timeofban > ?"
-			queryArgs.append(MyTime.time() - forbantime)
-		if fromtime is not None:
-			query += " AND timeofban > ?"
-			queryArgs.append(fromtime)
-		if overalljails or jail is None:
-			query += " GROUP BY ip ORDER BY timeofban DESC LIMIT 1"
-		cur = self._db.cursor()
-		return cur.execute(query, queryArgs)
-
 	def _getCurrentBans(self, cur, jail = None, ip = None, forbantime=None, fromtime=None):
+		if fromtime is None:
+			fromtime = MyTime.time()
 		queryArgs = []
 		if jail is not None:
-			query = "SELECT ip, timeofban, bantime, bancount, data FROM bips WHERE jail=?"
+			query = "SELECT ip, timeofban, data FROM bans WHERE jail=?"
 			queryArgs.append(jail.name)
 		else:
-			query = "SELECT ip, max(timeofban), bantime, bancount, data FROM bips WHERE 1"
+			query = "SELECT ip, max(timeofban), data FROM bans WHERE 1"
 		if ip is not None:
 			query += " AND ip=?"
 			queryArgs.append(ip)
-		query += " AND (timeofban + bantime > ? OR bantime <= -1)"
-		queryArgs.append(fromtime)
 		if forbantime not in (None, -1): # not specified or persistent (all)
 			query += " AND timeofban > ?"
 			queryArgs.append(fromtime - forbantime)
@@ -778,90 +748,33 @@ class Fail2BanDb(object):
 		cur = self._db.cursor()
 		return cur.execute(query, queryArgs)
 
-	@commitandrollback
-	def getCurrentBans(self, cur, jail=None, ip=None, forbantime=None, fromtime=None,
-		correctBanTime=True, maxmatches=None
-	):
-		"""Reads tickets (with merged info) currently affected from ban from the database.
-		
-		There are all the tickets corresponding parameters jail/ip, forbantime,
-		fromtime (normally now).
-		
-		If correctBanTime specified (default True) it will fix the restored ban-time 
-		(and therefore endOfBan) of the ticket (normally it is ban-time of jail as maximum)
-		for all tickets with ban-time greater (or persistent).
-		"""
-		if fromtime is None:
-			fromtime = MyTime.time()
+	def getCurrentBans(self, jail = None, ip = None, forbantime=None, fromtime=None, maxmatches=None):
 		tickets = []
 		ticket = None
-		if correctBanTime is True:
-			correctBanTime = jail.getMaxBanTime() if jail is not None else None
-			# don't change if persistent allowed:
-			if correctBanTime == -1: correctBanTime = None
 
-		for ticket in self._getCurrentBans(cur, jail=jail, ip=ip, 
-			forbantime=forbantime, fromtime=fromtime
-		):
-			# can produce unpack error (database may return sporadical wrong-empty row):
-			try:
-				banip, timeofban, bantime, bancount, data = ticket
-				# additionally check for empty values:
-				if banip is None or banip == "": # pragma: no cover
-					raise ValueError('unexpected value %r' % (banip,))
-				# if bantime unknown (after upgrade-db from earlier version), just use min known ban-time:
-				if bantime == -2: # todo: remove it in future version
-					bantime = jail.actions.getBanTime() if jail is not None else (
-						correctBanTime if correctBanTime else 600)
-				elif correctBanTime and correctBanTime >= 0:
-					# if persistent ban (or greater as max), use current max-bantime of the jail:
-					if bantime == -1 or bantime > correctBanTime:
-						bantime = correctBanTime
-				# after correction check the end of ban again:
-				if bantime != -1 and timeofban + bantime <= fromtime:
-					# not persistent and too old - ignore it:
-					logSys.debug("ignore ticket (with new max ban-time %r): too old %r <= %r, ticket: %r",
-						bantime, timeofban + bantime, fromtime, ticket)
-					continue
-			except ValueError as e: # pragma: no cover
-				logSys.debug("get current bans: ignore row %r - %s", ticket, e)
-				continue
-			# logSys.debug('restore ticket   %r, %r, %r', banip, timeofban, data)
-			ticket = FailTicket(banip, timeofban, data=data)
-			# filter matches if expected (current count > as maxmatches specified):
-			if maxmatches is None:
-				maxmatches = self.maxMatches
-			if maxmatches:
-				matches = ticket.getMatches()
-				if matches and len(matches) > maxmatches:
-					ticket.setMatches(matches[-maxmatches:])
-			else:
-				ticket.setMatches(None)
-			# logSys.debug('restored ticket: %r', ticket)
-			ticket.setBanTime(bantime)
-			ticket.setBanCount(bancount)
-			if ip is not None: return ticket
-			tickets.append(ticket)
+		with self._lock:
+			results = list(self._getCurrentBans(self._db.cursor(), 
+				jail=jail, ip=ip, forbantime=forbantime, fromtime=fromtime))
+
+		if results:
+			for banip, timeofban, data in results:
+				# logSys.debug('restore ticket   %r, %r, %r', banip, timeofban, data)
+				ticket = FailTicket(banip, timeofban, data=data)
+				# filter matches if expected (current count > as maxmatches specified):
+				if maxmatches is None:
+					maxmatches = self.maxMatches
+				if maxmatches:
+					matches = ticket.getMatches()
+					if matches and len(matches) > maxmatches:
+						ticket.setMatches(matches[-maxmatches:])
+				else:
+					ticket.setMatches(None)
+				# logSys.debug('restored ticket: %r', ticket)
+				if ip is not None: return ticket
+				tickets.append(ticket)
 
 		return tickets
 
-	def _cleanjails(self, cur):
-		"""Remove empty jails jails and log files from database.
-		"""
-		cur.execute(
-			"DELETE FROM jails WHERE enabled = 0 "
-				"AND NOT EXISTS(SELECT * FROM bans WHERE jail = jails.name) "
-				"AND NOT EXISTS(SELECT * FROM bips WHERE jail = jails.name)")
-
-	def _purge_bips(self, cur):
-		"""Purge old bad ips (jails and log files from database).
-		Currently it is timed out IP, whose time since last ban is several times out-dated (outDatedFactor is default 3).
-		Permanent banned ips will be never removed.
-		"""
-		cur.execute(
-			"DELETE FROM bips WHERE timeofban < ? and bantime != -1 and (timeofban + (bantime * ?)) < ?",
-			(int(MyTime.time()) - self._purgeAge, self._outDatedFactor, int(MyTime.time()) - self._purgeAge))
-
 	@commitandrollback
 	def purge(self, cur):
 		"""Purge old bans, jails and log files from database.
@@ -870,6 +783,7 @@ class Fail2BanDb(object):
 		cur.execute(
 			"DELETE FROM bans WHERE timeofban < ?",
 			(MyTime.time() - self._purgeAge, ))
-		self._purge_bips(cur)
-		self._cleanjails(cur)
+		cur.execute(
+			"DELETE FROM jails WHERE enabled = 0 "
+				"AND NOT EXISTS(SELECT * FROM bans WHERE jail = jails.name)")
 

fail2ban/server/datedetector.py

@@ -35,7 +35,7 @@ from ..helpers import getLogger
 # Gets the instance of the logger.
 logSys = getLogger(__name__)
 
-logLevel = 6
+logLevel = 5
 
 RE_DATE_PREMATCH = re.compile(r"(?<!\\)\{DATE\}", re.IGNORECASE)
 DD_patternCache = Utils.Cache(maxCount=1000, maxTime=60*60)
@@ -282,6 +282,8 @@ class DateDetector(object):
 				elif "{DATE}" in key:
 					self.addDefaultTemplate(preMatch=pattern, allDefaults=False)
 					return
+				elif key == "{NONE}":
+					template = _getPatternTemplate('{UNB}^', key)
 				else:
 					template = _getPatternTemplate(pattern, key)
 
@@ -337,65 +339,76 @@ class DateDetector(object):
 		# if no templates specified - default templates should be used:
 		if not len(self.__templates):
 			self.addDefaultTemplate()
-		logSys.log(logLevel-1, "try to match time for line: %.120s", line)
-		match = None
+		log = logSys.log if logSys.getEffectiveLevel() <= logLevel else lambda *args: None
+		log(logLevel-1, "try to match time for line: %.120s", line)
+
 		# first try to use last template with same start/end position:
+		match = None
+		found = None, 0x7fffffff, 0x7fffffff, -1
 		ignoreBySearch = 0x7fffffff
 		i = self.__lastTemplIdx
 		if i < len(self.__templates):
 			ddtempl = self.__templates[i]
 			template = ddtempl.template
 			if template.flags & (DateTemplate.LINE_BEGIN|DateTemplate.LINE_END):
-				if logSys.getEffectiveLevel() <= logLevel-1: # pragma: no cover - very-heavy debug
-					logSys.log(logLevel-1, "  try to match last anchored template #%02i ...", i)
+				log(logLevel-1, "  try to match last anchored template #%02i ...", i)
 				match = template.matchDate(line)
 				ignoreBySearch = i
 			else:
 				distance, endpos = self.__lastPos[0], self.__lastEndPos[0]
-				if logSys.getEffectiveLevel() <= logLevel-1:
-					logSys.log(logLevel-1, "  try to match last template #%02i (from %r to %r): ...%r==%r %s %r==%r...",
-						i, distance, endpos, 
-						line[distance-1:distance], self.__lastPos[1],
-						line[distance:endpos],
-						line[endpos:endpos+1], self.__lastEndPos[1])
-				# check same boundaries left/right, otherwise possible collision/pattern switch:
-				if (line[distance-1:distance] == self.__lastPos[1] and 
-						line[endpos:endpos+1] == self.__lastEndPos[1]
-				):
+				log(logLevel-1, "  try to match last template #%02i (from %r to %r): ...%r==%r %s %r==%r...",
+					i, distance, endpos,
+					line[distance-1:distance], self.__lastPos[1],
+					line[distance:endpos],
+					line[endpos:endpos+1], self.__lastEndPos[2])
+				# check same boundaries left/right, outside fully equal, inside only if not alnum (e. g. bound RE
+				# with space or some special char), otherwise possible collision/pattern switch:
+				if ((
+					line[distance-1:distance] == self.__lastPos[1] or
+						(line[distance:distance+1] == self.__lastPos[2] and not self.__lastPos[2].isalnum())
+					) and (
+					line[endpos:endpos+1] == self.__lastEndPos[2] or
+						(line[endpos-1:endpos] == self.__lastEndPos[1] and not self.__lastEndPos[1].isalnum())
+				)):
+					# search in line part only:
+					log(logLevel-1, "  boundaries are correct, search in part %r", line[distance:endpos])
 					match = template.matchDate(line, distance, endpos)
+				else:
+					log(logLevel-1, "  boundaries show conflict, try whole search")
+					match = template.matchDate(line)
+					ignoreBySearch = i
 			if match:
 				distance = match.start()
 				endpos = match.end()
 				# if different position, possible collision/pattern switch:
 				if (
+					len(self.__templates) == 1 or # single template:
 					template.flags & (DateTemplate.LINE_BEGIN|DateTemplate.LINE_END) or 
 					(distance == self.__lastPos[0] and endpos == self.__lastEndPos[0])
 				):
-					logSys.log(logLevel, "  matched last time template #%02i", i)
+					log(logLevel, "  matched last time template #%02i", i)
 				else:
-					logSys.log(logLevel, "  ** last pattern collision - pattern change, search ...")
+					log(logLevel, "  ** last pattern collision - pattern change, reserve & search ...")
+					found = match, distance, endpos, i; # save current best alternative
 					match = None
 			else:
-				logSys.log(logLevel, "  ** last pattern not found - pattern change, search ...")
+				log(logLevel, "  ** last pattern not found - pattern change, search ...")
 		# search template and better match:
 		if not match:
-			logSys.log(logLevel, " search template (%i) ...", len(self.__templates))
-			found = None, 0x7fffffff, 0x7fffffff, -1
+			log(logLevel, " search template (%i) ...", len(self.__templates))
 			i = 0
 			for ddtempl in self.__templates:
-				if logSys.getEffectiveLevel() <= logLevel-1:
-					logSys.log(logLevel-1, "  try template #%02i: %s", i, ddtempl.name)
 				if i == ignoreBySearch:
 					i += 1
 					continue
+				log(logLevel-1, "  try template #%02i: %s", i, ddtempl.name)
 				template = ddtempl.template
 				match = template.matchDate(line)
 				if match:
 					distance = match.start()
 					endpos = match.end()
-					if logSys.getEffectiveLevel() <= logLevel:
-						logSys.log(logLevel, "  matched time template #%02i (at %r <= %r, %r) %s",
-							i, distance, ddtempl.distance, self.__lastPos[0], template.name)
+					log(logLevel, "  matched time template #%02i (at %r <= %r, %r) %s",
+						i, distance, ddtempl.distance, self.__lastPos[0], template.name)
 					## last (or single) template - fast stop:
 					if i+1 >= len(self.__templates):
 						break
@@ -408,7 +421,7 @@ class DateDetector(object):
 					## [grave] if distance changed, possible date-match was found somewhere 
 					## in body of message, so save this template, and search further:
 					if distance > ddtempl.distance or distance > self.__lastPos[0]:
-						logSys.log(logLevel, "  ** distance collision - pattern change, reserve")
+						log(logLevel, "  ** distance collision - pattern change, reserve")
 						## shortest of both:
 						if distance < found[1]:
 							found = match, distance, endpos, i
@@ -422,7 +435,7 @@ class DateDetector(object):
 			# check other template was found (use this one with shortest distance):
 			if not match and found[0]:
 				match, distance, endpos, i = found
-				logSys.log(logLevel, "  use best time template #%02i", i)
+				log(logLevel, "  use best time template #%02i", i)
 				ddtempl = self.__templates[i]
 				template = ddtempl.template
 		# we've winner, incr hits, set distance, usage, reorder, etc:
@@ -432,8 +445,8 @@ class DateDetector(object):
 			ddtempl.distance = distance
 			if self.__firstUnused == i:
 				self.__firstUnused += 1
-			self.__lastPos = distance, line[distance-1:distance]
-			self.__lastEndPos = endpos, line[endpos:endpos+1]
+			self.__lastPos = distance, line[distance-1:distance], line[distance]
+			self.__lastEndPos = endpos, line[endpos-1], line[endpos:endpos+1]
 			# if not first - try to reorder current template (bubble up), they will be not sorted anymore:
 			if i and i != self.__lastTemplIdx:
 				i = self._reorderTemplate(i)
@@ -442,7 +455,7 @@ class DateDetector(object):
 			return (match, template)
 
 		# not found:
-		logSys.log(logLevel, " no template.")
+		log(logLevel, " no template.")
 		return (None, None)
 
 	@property

fail2ban/server/datetemplate.py

@@ -36,15 +36,16 @@ logSys = getLogger(__name__)
 RE_GROUPED = re.compile(r'(?<!(?:\(\?))(?<!\\)\((?!\?)')
 RE_GROUP = ( re.compile(r'^((?:\(\?\w+\))?\^?(?:\(\?\w+\))?)(.*?)(\$?)$'), r"\1(\2)\3" )
 
+RE_EXLINE_NO_BOUNDS = re.compile(r'^\{UNB\}')
 RE_EXLINE_BOUND_BEG = re.compile(r'^\{\^LN-BEG\}')
-RE_EXSANC_BOUND_BEG = re.compile(r'^\(\?:\^\|\\b\|\\W\)')
+RE_EXSANC_BOUND_BEG = re.compile(r'^\((?:\?:)?\^\|\\b\|\\W\)')
 RE_EXEANC_BOUND_BEG = re.compile(r'\(\?=\\b\|\\W\|\$\)$')
-RE_NO_WRD_BOUND_BEG = re.compile(r'^\(*(?:\(\?\w+\))?(?:\^|\(*\*\*|\(\?:\^)')
+RE_NO_WRD_BOUND_BEG = re.compile(r'^\(*(?:\(\?\w+\))?(?:\^|\(*\*\*|\((?:\?:)?\^)')
 RE_NO_WRD_BOUND_END = re.compile(r'(?<!\\)(?:\$\)?|\\b|\\s|\*\*\)*)$')
 RE_DEL_WRD_BOUNDS = ( re.compile(r'^\(*(?:\(\?\w+\))?\(*\*\*|(?<!\\)\*\*\)*$'), 
 	                    lambda m: m.group().replace('**', '') )
 
-RE_LINE_BOUND_BEG = re.compile(r'^(?:\(\?\w+\))?(?:\^|\(\?:\^(?!\|))')
+RE_LINE_BOUND_BEG = re.compile(r'^(?:\(\?\w+\))?(?:\^|\((?:\?:)?\^(?!\|))')
 RE_LINE_BOUND_END = re.compile(r'(?<![\\\|])(?:\$\)?)$')
 
 RE_ALPHA_PATTERN = re.compile(r'(?<!\%)\%[aAbBpc]')
@@ -119,7 +120,7 @@ class DateTemplate(object):
 		if boundBegin:
 			self.flags |= DateTemplate.WORD_BEGIN if wordBegin != 'start' else DateTemplate.LINE_BEGIN
 			if wordBegin != 'start':
-				regex = r'(?:^|\b|\W)' + regex
+				regex = r'(?=^|\b|\W)' + regex
 			else:
 				regex = r"^(?:\W{0,2})?" + regex
 				if not self.name.startswith('{^LN-BEG}'):
@@ -128,12 +129,14 @@ class DateTemplate(object):
 		if boundEnd:
 			self.flags |= DateTemplate.WORD_END
 			regex += r'(?=\b|\W|$)'
-		if RE_LINE_BOUND_BEG.search(regex): self.flags |= DateTemplate.LINE_BEGIN
-		if RE_LINE_BOUND_END.search(regex): self.flags |= DateTemplate.LINE_END
+		if not (self.flags & DateTemplate.LINE_BEGIN) and RE_LINE_BOUND_BEG.search(regex):
+			self.flags |= DateTemplate.LINE_BEGIN
+		if not (self.flags & DateTemplate.LINE_END) and RE_LINE_BOUND_END.search(regex):
+			self.flags |= DateTemplate.LINE_END
 		# remove possible special pattern "**" in front and end of regex:
 		regex = RE_DEL_WRD_BOUNDS[0].sub(RE_DEL_WRD_BOUNDS[1], regex)
 		self._regex = regex
-		logSys.log(7, '  constructed regex %s', regex)
+		logSys.log(4, '  constructed regex %s', regex)
 		self._cRegex = None
 
 	regex = property(getRegex, setRegex, doc=
@@ -156,6 +159,7 @@ class DateTemplate(object):
 		"""
 		if not self._cRegex:
 			self._compileRegex()
+		logSys.log(4, "   search %s", self.regex)
 		dateMatch = self._cRegex.search(line, *args); # pos, endpos
 		if dateMatch:
 			self.hits += 1
@@ -188,7 +192,7 @@ class DateTemplate(object):
 	def unboundPattern(pattern):
 		return RE_EXEANC_BOUND_BEG.sub('',
 			RE_EXSANC_BOUND_BEG.sub('',
-				RE_EXLINE_BOUND_BEG.sub('', pattern)
+				RE_EXLINE_BOUND_BEG.sub('', RE_EXLINE_NO_BOUNDS.sub('', pattern))
 			)
 		)
 
@@ -297,6 +301,10 @@ class DatePatternRegex(DateTemplate):
 	def setRegex(self, pattern, wordBegin=True, wordEnd=True):
 		# original pattern:
 		self._pattern = pattern
+		# if unbound signalled - reset boundaries left and right:
+		if RE_EXLINE_NO_BOUNDS.search(pattern):
+			pattern = RE_EXLINE_NO_BOUNDS.sub('', pattern)
+			wordBegin = wordEnd = False
 		# if explicit given {^LN-BEG} - remove it from pattern and set 'start' in wordBegin:
 		if wordBegin and RE_EXLINE_BOUND_BEG.search(pattern):
 			pattern = RE_EXLINE_BOUND_BEG.sub('', pattern)