I have a strange situation here with Nginx and SNI SSL passthrough.
I have multiple OPNsense servers running similar configs for similar stacks on the backend.
Basically setup is as follows:
Stream Server is listening on 443 with "Route With" set to "SNI Upstream mapping" and the "SNI Upstream Mapping" is set to the SNI stack
SNI Based Routing has a single entry for all the domainnames are pointing to the correct "Upstream" entries
Upstream has all the correct entries pointing to the correct "Upstream servers"
Upstream Servers list all the correct backend server with IP:port settings priority 1 with 1000 for the rest of the settings.
Now here is where it gets interesting. Once I restart nginx it works great for anywhere between 10 seconds to 1 minute. but then no matter what I do I cat 404 errors 100% of the time after the initial working phase is over.
there are no errors in the logs and DNS is constant and correct.
Anyone have ANY ideas WTF is going on here?
thx
Here is the actual config.
load_module /usr/local/libexec/nginx/ngx_stream_module.so;
load_module /usr/local/libexec/nginx/ngx_http_naxsi_module.so;
load_module /usr/local/libexec/nginx/ngx_mail_module.so;
load_module /usr/local/libexec/nginx/ngx_http_brotli_filter_module.so;
load_module /usr/local/libexec/nginx/ngx_http_brotli_static_module.so;
load_module /usr/local/libexec/nginx/ngx_http_js_module.so;
load_module /usr/local/libexec/nginx/ngx_http_vhost_traffic_status_module.so;
user www staff;
worker_processes 1;
#error_log /var/log/nginx/error.log;
error_log syslog:server=unix:/var/run/log,facility=local6,nohostname warn;
events {
worker_connections 1024;
}
http {
include mime.types;
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';
log_format main_ext '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for" '
'"$host" sn="$server_name" '
'rt=$request_time '
'ua="$upstream_addr" us="$upstream_status" '
'ut="$upstream_response_time" ul="$upstream_response_length" '
'cs=$upstream_cache_status';
log_format handshake '"$http_user_agent" "$ssl_ciphers" "$ssl_curves"';
log_format anonymized ':: - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';
#tcp_nopush on;
# https intercept detection
js_import /usr/local/opnsense/scripts/nginx/ngx_functions.js;
js_set $tls_intercepted ngx_functions.check_intercept;
# 200M should be big enough for file servers etc.
client_max_body_size 200M;
brotli_static on;
brotli on;
gzip_static on;
gzip on;
server_tokens off;
sendfile Off;
default_type application/octet-stream;
map $http_upgrade $connection_upgrade {
default upgrade;
'' close;
}
# Map used in location.conf for proxy_ssl_name
map $ssl_server_name $upstream_sni_name {
default $ssl_server_name;
'' $host;
}
include http_post/*.conf;
# TODO add when core is ready for allowing nginx to serve the web interface
# include nginx_web.conf;
# UPSTREAM SERVERS
include opnsense_http_vhost_plugins/*.conf;
server {
listen 80;
listen [::]:80;
sendfile On;
server_name ucs-sso-ng.caritas-im-norden.de;
client_header_buffer_size 1k;
large_client_header_buffers 4 8k;
access_log /var/log/nginx/ucs-sso-ng.caritas-im-norden.de.access.log main;
access_log /var/log/nginx/tls_handshake.log handshake;
error_log /var/log/nginx/ucs-sso-ng.caritas-im-norden.de.error.log error;
#include tls.conf;
error_page 403 /opnsense_error_403.html;
error_page 404 /opnsense_error_404.html;
error_page 405 /waf_denied.html;
error_page 500 501 502 503 504 /opnsense_server_error.html;
location = /opnsense_error_403.html {
internal;
root /usr/local/etc/nginx/views;
}
location = /opnsense_error_404.html {
internal;
root /usr/local/etc/nginx/views;
}
location = /opnsense_server_error.html {
internal;
root /usr/local/etc/nginx/views;
}
proxy_intercept_errors off;
# location to ban the host permanently
set $naxsi_extensive_log 0;
location @permanentban {
access_log /var/log/nginx/permanentban.access.log main;
internal;
add_header "Content-Type" "text/plain; charset=UTF-8" always;
return 403 "You got banned permanently from this server.";
}
error_page 418 = @permanentban;
location = /waf_denied.html {
root /usr/local/etc/nginx/views;
access_log /var/log/nginx/waf_denied.access.log main;
}
location ^~ /.well-known/acme-challenge/ {
default_type "text/plain";
proxy_pass http://127.0.0.1:43580;
}
# block based on User Agents defined in global http settings
if ($http_user_agent ~* Python-urllib|Nmap|python-requests|libwww-perl|MJ12bot|Jorgee|fasthttp|libwww|Telesphoreo|A6-Indexer|ltx71|okhttp|ZmEu|sqlmap|LMAO/2.0|l9explore|l9tcpid|Masscan|zgrab|Ronin/2.0|Hakai/2.0|Indy\sLibrary|^Mozilla/[\d\.]+$|Morfeus\sFucking\sScanner|MSIE\s[0-6]\.\d+) {
return 418;
}
location /opnsense-auth-request {
internal;
fastcgi_pass unix:/var/run/php-webgui.socket;
fastcgi_index index.php;
fastcgi_param TLS-Cipher $ssl_cipher;
fastcgi_param TLS-Protocol $ssl_protocol;
fastcgi_param TLS-SNI-Host $ssl_server_name;
fastcgi_param Original-URI $request_uri;
fastcgi_param Original-HOST $host;
fastcgi_param SERVER-UUID "532ac9d7-321f-4ce6-a569-8947c1ee60bc";
fastcgi_param SCRIPT_FILENAME /usr/local/opnsense/scripts/nginx/ngx_auth.php;
fastcgi_intercept_errors on;
include fastcgi_params;
}
if ($scheme != "https") {
return 302 https://$host$request_uri;
}
include 532ac9d7-321f-4ce6-a569-8947c1ee60bc_pre/*.conf;
include 532ac9d7-321f-4ce6-a569-8947c1ee60bc_post/*.conf;
}
}
stream {
# LOG FORMATS
log_format main '$remote_addr [$time_local] '
'$protocol $status $bytes_sent $bytes_received '
'$session_time';
log_format anonymized ':: [$time_local] '
'$protocol $status $bytes_sent $bytes_received '
'$session_time';
# UPSTREAM SERVERS
upstream upstreama2f569d399594042bd3e87d44972480f {
server 10.200.1.2:443 weight=1 max_conns=1000 max_fails=1000 fail_timeout=1000;
}
upstream upstream9f39913216d146b9bba809e04c704161 {
server 10.200.6.2:443 weight=1 max_conns=1000 max_fails=1000 fail_timeout=1000;
}
upstream upstream33d1614fde14477b8286c258dbb57a74 {
server 10.200.1.4:443 weight=1 max_conns=1000 max_fails=1000 fail_timeout=1000;
}
upstream upstream7ec9246b91294af08d0f2dbd5373f412 {
server 10.200.1.5:443 weight=1 max_conns=1000 max_fails=1000 fail_timeout=1000;
}
upstream upstream419f7f5065ef436eae97b504c28d354b {
server 10.200.1.6:443 weight=1 max_conns=1000 max_fails=1000 fail_timeout=1000;
}
# upstream maps
map $ssl_preread_server_name $hostmap41cac9dd9a5f4a89a1e0be6c73445cc6 {
sub1.domain.tld upstream9f39913216d146b9bba809e04c704161;
sub2.domain.tld upstream9f39913216d146b9bba809e04c704161;
sub3.domain.tld upstream9f39913216d146b9bba809e04c704161;
sub4.domain.tld upstreama2f569d399594042bd3e87d44972480f;
sub5.domain.tld upstream33d1614fde14477b8286c258dbb57a74;
sub6.domain.tld upstream33d1614fde14477b8286c258dbb57a74;
sub7.domain.tld upstream33d1614fde14477b8286c258dbb57a74;
sub8.domain.tld upstream419f7f5065ef436eae97b504c28d354b;
sub9.domain.tld upstream7ec9246b91294af08d0f2dbd5373f412;
}
include opnsense_stream_vhost_plugins/*.conf;
# servers
server {
listen 443;
access_log /var/log/nginx/stream_843d8674-ab7a-48d0-b4ed-715b2036b605.access.log main;
error_log /var/log/nginx/stream_843d8674-ab7a-48d0-b4ed-715b2036b605.error.log info;
ssl_preread on;
include 843d8674-ab7a-48d0-b4ed-715b2036b605_pre/*.conf;
proxy_pass $hostmap41cac9dd9a5f4a89a1e0be6c73445cc6;
include 843d8674-ab7a-48d0-b4ed-715b2036b605_post/*.conf;
}
}
# mail {
# }