diff --git a/roles/common/handlers/main.yml b/roles/common/handlers/main.yml
index 4d8d07e..25034bc 100644
--- a/roles/common/handlers/main.yml
+++ b/roles/common/handlers/main.yml
@@ -25,3 +25,4 @@
   ansible.builtin.systemd:
     name: fail2ban
     state: restarted
+  when: webserver is defined and webserver == 'nginx'
diff --git a/roles/common/tasks/fail2ban.yml b/roles/common/tasks/fail2ban.yml
index 45dd3a4..9d47270 100644
--- a/roles/common/tasks/fail2ban.yml
+++ b/roles/common/tasks/fail2ban.yml
@@ -1,4 +1,15 @@
 ---
+- name: Install fail2ban
+  when:
+    - ansible_distribution_major_version is version('11', '>=')
+    - webserver is defined and webserver == 'nginx'
+  ansible.builtin.package:
+    name:
+      - fail2ban
+      - python3-systemd
+    state: present
+    cache_valid_time: 3600
+
 - name: Configure fail2ban sshd filter
   ansible.builtin.template:
     src: etc/fail2ban/jail.d/sshd.local.j2
diff --git a/roles/common/tasks/firewall_Debian.yml b/roles/common/tasks/firewall_Debian.yml
index ba28aec..4a436d2 100644
--- a/roles/common/tasks/firewall_Debian.yml
+++ b/roles/common/tasks/firewall_Debian.yml
@@ -6,10 +6,8 @@
       when: ansible_distribution_major_version is version('11', '>=')
       ansible.builtin.package:
         name:
-          - fail2ban
           - libnet-ip-perl # for aggregate-cidr-addresses.pl
           - nftables
-          - python3-systemd
           - curl # for nftables update scripts
         state: present
         cache_valid_time: 3600
@@ -109,8 +107,12 @@
         state: started
         enabled: true
 
+    # We only install fail2ban on systems running nginx. If the host is running
+    # Caddy then there are no logs for us to monitor.
     - ansible.builtin.include_tasks: fail2ban.yml
-      when: ansible_distribution_major_version is version('9', '>=')
+      when:
+        - ansible_distribution_major_version is version('9', '>=')
+        - webserver is defined and webserver == 'nginx'
   tags: firewall
 
 # vim: set sw=2 ts=2:
diff --git a/roles/common/tasks/firewall_Ubuntu.yml b/roles/common/tasks/firewall_Ubuntu.yml
index f30af1e..aa7f5bb 100644
--- a/roles/common/tasks/firewall_Ubuntu.yml
+++ b/roles/common/tasks/firewall_Ubuntu.yml
@@ -3,12 +3,11 @@
 
 - block:
     - name: Install Ubuntu firewall packages
+      when: ansible_distribution_version is version('20.04', '>=')
       ansible.builtin.package:
         name:
-          - fail2ban
           - libnet-ip-perl # for aggregate-cidr-addresses.pl
           - nftables
-          - python3-systemd
           - curl # for nftables update scripts
         state: present
         cache_valid_time: 3600
@@ -107,8 +106,12 @@
         state: started
         enabled: true
 
+    # We only install fail2ban on systems running nginx. If the host is running
+    # Caddy then there are no logs for us to monitor.
     - ansible.builtin.include_tasks: fail2ban.yml
-      when: ansible_distribution_version is version('16.04', '>=')
+      when:
+        - ansible_distribution_version is version('16.04', '>=')
+        - webserver is defined and webserver == 'nginx'
   tags: firewall
 
 # vim: set sw=2 ts=2: