after upgrade to OPNsense 25.1.7 caddy stopped working with reverse_proxy config

[0zzy]

Hello together,
since I updated today morning to the newest version of opnsense, caddy doesn't recognise my config.
Caddyfile in /usr/local/etc/caddy/

Code Select Expand

# DO NOT EDIT THIS FILE -- OPNsense auto-generated file


# caddy_user=root

# Global Options
{
log {
output net unixgram//var/run/caddy/log.sock {
}
format json {
time_format rfc3339
}
}

servers {
protocols h1 h2
trusted_proxies static 192.168.0.0/16 172.16.0.0/12 10.0.0.0/8
}

dynamic_dns {
provider cloudflare LrY7LW4tqarzszMUvTJgXQy4dsC_Q7o97T5HAdL9
domains {
domain.de *
xxxx xxxx
xxxx xxxx
this one works remotely
xxxx xxxx
xxxx xxxx
xxxx xxxx
xxxx xxxx
}
ip_source interface igc0
update_only
}

email xxxx
grace_period 10s
import /usr/local/etc/caddy/caddy.d/*.global
}

# Reverse Proxy Configuration


*.xxxx {
log {
output file /var/log/caddy/access/bb35164d-3b13-4c5c-a47e-6499b75c76da.log {
roll_keep_for 10d
}
}
tls {
issuer acme {
dns cloudflare xxxx
}
}

@258c701d-7862-4552-b894-d961cdbab7e4_zerot3chde {
not client_ip 192.168.0.0/16 172.16.0.0/12 10.0.0.0/8
}
handle @258c701d-7862-4552-b894-d961cdbab7e4_zerot3chde {
abort
}
}

xxxx {
log {
output file /var/log/caddy/access/3d8d56e7-f7e5-49a9-9229-789c65baf88c.log {
roll_keep_for 10d
}
}

@258c701d-7862-4552-b894-d961cdbab7e4_giteazerot3chde {
not client_ip 192.168.0.0/16 172.16.0.0/12 10.0.0.0/8
}
handle @258c701d-7862-4552-b894-d961cdbab7e4_giteazerot3chde {
abort
}

handle {
reverse_proxy xxxx:3000 {
transport http {
}
}
}
}

xxxx {
log {
output file /var/log/caddy/access/d9431ba8-f607-43f7-a888-131e861fe5e6.log {
roll_keep_for 10d
}
}

@258c701d-7862-4552-b894-d961cdbab7e4_str1ctzerot3chde {
not client_ip 192.168.0.0/16 172.16.0.0/12 10.0.0.0/8
}
handle @258c701d-7862-4552-b894-d961cdbab7e4_str1ctzerot3chde {
abort
}

handle {
reverse_proxy xxxx:8000 {
transport http {
}
}
}
}

xxxx (this one works) {
log {
output file /var/log/caddy/access/2b8651a0-b5db-41f6-9d3b-9a8f1109f3e1.log {
roll_keep_for 10d
}
}

@258c701d-7862-4552-b894-d961cdbab7e4_remotelyzerot3chde {
not client_ip 192.168.0.0/16 172.16.0.0/12 10.0.0.0/8
}
handle @258c701d-7862-4552-b894-d961cdbab7e4_remotelyzerot3chde {
abort
}

handle {
reverse_proxy xxxx:5000 {
transport http {
}
}
}
}

xxxx {
log {
output file /var/log/caddy/access/c7f68fcb-0ec2-48f2-a3d0-790c7677b365.log {
roll_keep_for 10d
}
}

@258c701d-7862-4552-b894-d961cdbab7e4_invoicezerot3chde {
not client_ip 192.168.0.0/16 172.16.0.0/12 10.0.0.0/8
}
handle @258c701d-7862-4552-b894-d961cdbab7e4_invoicezerot3chde {
abort
}

handle {
reverse_proxy xxxx:8035 {
}
}
}

xxxx {
log {
output file /var/log/caddy/access/4033eaa0-263f-470f-8e2d-9cd6f42788c5.log {
roll_keep_for 10d
}
}

@258c701d-7862-4552-b894-d961cdbab7e4_jellyfinzerot3chde {
not client_ip 192.168.0.0/16 172.16.0.0/12 10.0.0.0/8
}
handle @258c701d-7862-4552-b894-d961cdbab7e4_jellyfinzerot3chde {
abort
}

handle {
reverse_proxy xxxx:8096 {
}
}
}

xxxx {
log {
output file /var/log/caddy/access/a3475101-3588-49e2-9e20-17143ad58ba9.log {
roll_keep_for 10d
}
}

@258c701d-7862-4552-b894-d961cdbab7e4_librenmszerot3chde {
not client_ip 192.168.0.0/16 172.16.0.0/12 10.0.0.0/8
}
handle @258c701d-7862-4552-b894-d961cdbab7e4_librenmszerot3chde {
abort
}

handle {
reverse_proxy xxxx:80 {
transport http {
}
}
}
}

xxxx {
log {
output file /var/log/caddy/access/d273108a-38a7-432f-a88f-2cba3ee196b6.log {
roll_keep_for 10d
}
}

@258c701d-7862-4552-b894-d961cdbab7e4_authentikzerot3chde {
not client_ip 192.168.0.0/16 172.16.0.0/12 10.0.0.0/8
}
handle @258c701d-7862-4552-b894-d961cdbab7e4_authentikzerot3chde {
abort
}

handle {
reverse_proxy https://xxxx:9443 {
transport http {
tls_insecure_skip_verify
tls_trust_pool file /var/db/caddy/data/caddy/certificates/temp/6694f83be85a0.pem
tls_server_name authentik
}
}
}
}

import /usr/local/etc/caddy/caddy.d/*.conf
the interesting part is, in the last version I put some own configs and a snippet.global in /usr/local/etc/caddy/caddy.d/
like this custom config:

Code Select Expand

test.domain.de {
    reverse_proxy any:anyport
    import encodingOptions
    tls {
        import tlsOptions
    }
    header {
        import HardeningConfig cors ratelimiterOptions reffererConfig https://invoice.zerot3ch.de
    }
    log {
        output file /var/log/caddy/test.log {
            import logOptions
        }
        format json
    }
}
the snippet.global was:

Code Select Expand

(encodingOptions) {
    encode zstd gzip
}

(tlsOptions) {
    protocols tls1.2 tls1.3
    curves x25519 secp384r1
    ciphers TLS_AES_128_GCM_SHA256 TLS_AES_256_GCM_SHA384
}

(HardeningConfig) {
    header Permissions-Policy "interest-cohort=()"
    header Strict-Transport-Security "max-age=31536000; includeSubDomains; preload"
    header X-Content-Type-Options "nosniff"
    header X-Frame-Options "SAMEORIGIN"
    header Server-Timing "*"
    header X-XSS-Protection "1; mode=block"
    header Content-Security-Policy "default-src 'self'; script-src 'self' 'unsafe-inline' 'unsafe-eval'; object-src 'none'; frame-src 'none';"
    header Set-Cookie "my_cookie=value; Path=/; HttpOnly; SameSite=Lax"
}

(cors) {


root@OPNsense:/home/ozzy/caddy # cat snippets.global
(encodingOptions) {
    encode zstd gzip
}

(tlsOptions) {
    protocols tls1.2 tls1.3
    curves x25519 secp384r1
    ciphers TLS_AES_128_GCM_SHA256 TLS_AES_256_GCM_SHA384
}

(HardeningConfig) {
    header Permissions-Policy "interest-cohort=()"
    header Strict-Transport-Security "max-age=31536000; includeSubDomains; preload"
    header X-Content-Type-Options "nosniff"
    header X-Frame-Options "SAMEORIGIN"
    header Server-Timing "*"
    header X-XSS-Protection "1; mode=block"
    header Content-Security-Policy "default-src 'self'; script-src 'self' 'unsafe-inline' 'unsafe-eval'; object-src 'none'; frame-src 'none';"
    header Set-Cookie "my_cookie=value; Path=/; HttpOnly; SameSite=Lax"
}

(cors) {
    @origin header Origin *
    header @origin Access-Control-Allow-Origin "*"
    header @origin Vary Origin
}

(logOptions) {
    roll_size 100MiB
    roll_keep 5
    roll_keep_for 168h
}

(reffererConfig) {
    header Referrer-Policy "strict-origin-when-cross-origin"
}

(ratelimiterOptions) {
    rate_limit {
        zone dynamic {
            key {remote_host}
            rate 10r/m
        }
        zone static {
            key {remote_host}
            rate 20r/m
            window 1m
        }
        zone api {
            key {remote_host}
            rate 5r/m
        }
    }
}
But also after upgrade these settings not working.
I get the following errors:

1.) Error: adapting config using caddyfile: ambiguous site definition:
2.) adapting config using caddyfile: File to import not found: encodingOptions, at /usr/local/etc/caddy/caddy.d/custom.conf:3 import chain ['/usr/local/etc/caddy/Caddyfile:216 (import)'] (but the options are set in snippets.global)
Also may I as why only
import /usr/local/etc/caddy/caddy.d/*.conf
is in the caddyfile but
import /usr/local/etc/caddy/caddy.d/*.global
before doesn't work?

any help is very important, I need my git and some other things back ;(
@Monviech could you help?

[Monviech (Cedrik)]

If you define a custom site block, the same site block cannot exist in the GUI, otherwise the domain exists two times and is ambigous.

The configurations cannot be merged, each item must be unique.

It worked accidentally before because wildcard subdomains were matched inside the scope of the wildcard domain.

But the latest change due to upstream requirements now renders each subdomain as full site block.

So either remove your custom configuration file with the site block, or remove the dupicate domains from the GUI that you already declared in a custom config file.

[0zzy]

I tested out both:
1.) recreate the entries for my sites --> not working
2.) delete the sites (Reverse Proxy & Handlers) and use my own config --> not working also

What I'm doing wrong?

I simply use the Documentation of caddy in the OPNSense Documentation

[Monviech (Cedrik)]

Im not sure what you are doing wrong without having the full configuration file and all imports available without any omissions (besides the cloudflare key).

Please PM them to me so I can take a look if there is an unexpected regression, or if its a configuration error.

[Monviech (Cedrik)]

I've looked at your configuration structure and the issue is that you declare snippets inside the global configuration.

But snippets should be declared outside of the global configuration before the site block. So just put the snippets into a file named "1.conf" so its loaded first.

Please note that I cannot spend more community support on custom configuration structures like these, if its not 100% GUI created its hard to support this.

[0zzy]

No Problem Mate,
I was simply wondered what was going wrong.
and this kind of support is really enough ;)
I would test this out and let a post in the forum if it works.

At the moment I I created every site as a custom config, because there are many options especially for security reasons which I suggest to use in the webui.
But there's not so much as you could define via custom configs.

At least im happy because now it working, not as usual but it works.

[Monviech (Cedrik)]

There might have been upstream changes that validate the caddyfile stricter with the version change to 2.10.0.

I would not use too many of these custom snippets as Caddy strives to be secure by default. Whatever you configure might make it unknowingly worse.

Well, you are up and running again so good job :)

[0zzy]

Monviech,
not really, the problem is that the most homelab things are insecure by default.
mandatory security mechanisms like TLS-SNI, OCSP, Input Validation & Output Escaping, CSRF Protection, Secure Headers etc. are only mediocrely implemented in caddy on opnsense.

With my configuration which I use on a standalone caddy server which is hosted by an cloudprovider, its also the same scenario.

What I like to have is to implement everything which is possible to make my sites as secure as possible.

Some fact here are, the not everything works with every application or site.

To show you what I mean I use Qualys SSL Labs,
one of my sites getting an A+ rating (which is almost okay but not perfect), some getting a T Rating and some getting B rating:

In order A to T:

https://imgur.com/27TaXTm

I work especially at tls /ssl cuipher suites to activate the highest algorithm which is possible:
This isn't what I expect!
https://imgur.com/jUy9oVy

And something like this is worst in my opinion:

Code Select Expand

https://imgur.com/358tipMor

https://imgur.com/KZEts5Y

[Monviech (Cedrik)]

But e.g. they support the latest features in this release, like

https://github.com/caddyserver/caddy/releases

Post-quantum (PQC) key exchange: Caddy now supports the standardized x25519mlkem768 cryptographic group by default.

That is a TLS 1.3 feature, there you cannot freely choose which cipher suites should be used as they are not configurable.
https://github.com/golang/go/issues/29349

Also why "encode zstd gzip" -> https://caddy.community/t/caddy-gzip-and-crime-breach/487

I dont want to say that what you do is wrong, some things just might not have the intended effect.

[0zzy]

as far as I understand encode zstd gzip meins the compression or I'm wrong it hasn't something to do with crypto.

[Patrick M. Hausen]

not really, the problem is that the most homelab things are insecure by default.

That claim intrigued me so I went to check. I run all my publicly reachable service on either FreeBSD jails or Docker on Linux, all with plain text HTTP for the application server and Caddy on OPNsense as the single ingress.

Nextcloud: A+
All others: A

All with Caddy plugin default settings. In my opinion it is not the business of the proxy to apply e.g. security headers. The backend application should do that and clearly e.g. the Nextcloud documentation recommends to configure like this:

Code Select Expand

        add_header Strict-Transport-Security "max-age=15768000; includeSubDomains; preload" always;
        [...]
        add_header Cache-Control                     "public, max-age=15778463$asset_immutable";
        add_header Referrer-Policy                   "no-referrer"       always;
        add_header X-Content-Type-Options            "nosniff"           always;
        add_header X-Frame-Options                   "SAMEORIGIN"        always;
        add_header X-Permitted-Cross-Domain-Policies "none"              always;
        add_header X-Robots-Tag                      "noindex, nofollow" always;
        add_header X-XSS-Protection                  "1; mode=block"     always;

On the application web server, not the proxy. Even when the application web server is plain HTTP.

I checked my "only A" applications in detail and contrary to Nextcloud they did not have HSTS enabled.
After fixing that they, too, all get an A+. You can easily do that in the Caddy plugin if you don't want to do it in the backend configuration as recommended for Nextcloud.

And bingo - all A+, now.

Thanks for that hint :-)

Kind regards,
Patrick

[0zzy]

Patrick please check the dev console also with your entire browser.

A+ isn't everything as I mentioned before.
It means literally that you have a certain "better" basic protection. However, this does not mean that the bar has been reached.

And this is exactly my goal! Get it hard harder as hardest possible hardening also for my sites.

It's a decision everyone should make.

[Patrick M. Hausen]

Patrick please check the dev console also with your entire browser.

Sorry, I don't get that. Of course there are a ton of javascript warnings - but this is entirely up to the application, nothing any reverse proxy could fix.

What specifically should I look for?

Kind regards,
Patrick

[0zzy]

Hey Patrick,

this is named by two technics:

1.) Client Side Security Audit
and
2.) Browser Based Security Inspection.

The Problem with sites like Qualys is, that you get an A+ also if you have misconfigurations which are used by attackers to get into.

What they check:

Qualys SSL Labs scores sites primarily on TLS configuration, not holistic web security. The main factors include:

1.) Supported TLS versions (e.g., only TLS 1.2 and 1.3 = higher score)
2.) Cipher suites (strong, forward-secret ciphers = higher score)
3.) Certificate strength (2048+ bit key, valid chain, no weak signature algos)
4.) HSTS with long duration and preload
5.) No support for insecure renegotiation, compression, or SSLv3
6.) OCSP stapling and other TLS extras

What sites like qualys don't do:

- Web app misconfigurations
- CSP or other headers
- WAF or rate limiting
- Cookie security flags
- Auth schemes / access control
- Firewall rules
- Malware / phishing risks
- Backend exposure / IP leaks

And much more.

So bringing a site ( it doesn't matter if as reverse proxy or a site server config) isn't all and of course isn't secure by default.

You mentioned next cloud, which has a good implementation of a Security Scan.
But it gives you only some of the ways to harden a System.

Its not only about caddy but that would go too far here.

I scrambled my head on how you could do it easily.

I think much easier is to use something like Mozilla HTTP Observatory https://developer.mozilla.org/en-US/observatory.

Here you can check it easier and you would see that an A+ is more an B or something.

[meyergru]

I think Patrick knows the distinction.

It is true that Qualys SSL Labs measures only certain aspects - as the name suggests, it measures the quality of the SSL (or better: TLS) encryption setup, nothing more. BTW: They do not evaluate OCSP any more, because that is vanishing.

They actually say exactly what they evaluate: https://github.com/ssllabs/research/wiki/SSL-Server-Rating-Guide

However, the Mozilla HTTP Observatory is also not a replacement for Qualys, as it only cover security of the HTTP setup, and, for example, not application security aspects.

All of the mentioned tools of the trade just cover one aspect, there is no "one to rule them all" that does the trick.