{ "hunt": { "meta": { "tlp": "clear", "hunt": { "handoff": "promote-to-detection", "trigger": "intel-report", "methodology": "model-assisted", "applicability": "campaign-specific", "justification": "The LiteLLM supply chain compromise allows for unauthenticated RCE and credential theft at scale. Verifying the integrity of Python environments is critical for preventing unauthorized cloud access." }, "name": "LiteLLM Supply Chain Poisoning and Persistence", "type": "investigation", "labels": [ "hunt", "attack.t1195.002", "attack.t1574.013", "attack.t1555", "credential access", "execution", "initial access", "persistence" ], "series": { "slug": "lessons-from-cybersecurity-black-week", "index": 2, "title": "Lessons from Cybersecurity Black Week", "total": 2 }, "related": [ { "hunt": "python-startup-hijack-general", "reason": "This hunt is specific to the LiteLLM incident; a broader hunt for all .pth hijacks is a sibling.", "relation": "out-of-scope-alternative" }, { "hunt": "cybersecurity-black-week-vulnerability-trust-audit", "relation": "follows" } ], "targets": { "hunter": { "name": "Hunt agent", "agent": true }, "analyst": { "name": "Tier-2 analyst", "role": "analyst" }, "endpoint": { "name": "Endpoint telemetry (hb_ surfaces)", "category": "endpoint", "telemetry": [ "endpoint" ] } }, "analysis": "A static rule for package versions is bypassed by renaming. This hunt uses prevalence counting across .pth files and correlates it with specific Python-based file access behavioral patterns, providing a cross-surface signal that simple version checks miss.", "coverage": [ { "stage": "malicious-dependency-installation", "steps": [ "litellm-install-lead", "litellm-inventory-check" ], "status": "covered" }, { "stage": "python-startup-persistence", "steps": [ "pth-persistence-prevalence" ], "status": "covered" }, { "stage": "credential-theft-and-rat-c2", "steps": [ "python-credential-theft" ], "status": "covered" }, { "stage": "rce-vulnerability-exposure", "reason": "Belongs to another part of the 'Lessons from Cybersecurity Black Week' series.", "status": "out_of_scope" }, { "stage": "memory-safety-exploitation-crashes", "reason": "Belongs to another part of the 'Lessons from Cybersecurity Black Week' series.", "status": "out_of_scope" }, { "stage": "compromised-certificate-trust", "reason": "Belongs to another part of the 'Lessons from Cybersecurity Black Week' series.", "status": "out_of_scope" } ], "scenario": { "stages": [ { "name": "Vulnerable Service Exposure", "slug": "rce-vulnerability-exposure", "tactic": "initial-access", "techniques": [ "T1190" ], "observables": [ "PHP versions 5.4 through 8.5.11", "GitLab CE/EE versions 7.0 to 19.5", "Node.js 24.22.0", "Linux kernel KSMBD SMB Direct (ZDI-CAN-33549, ZDI-CAN-33774)", "CVE-2026-109618", "CVE-2026-109619", "CVE-2026-110264", "CVE-2026-111497" ] }, { "name": "Memory Safety Exploitation", "slug": "memory-safety-exploitation-crashes", "tactic": "execution", "techniques": [ "T1203" ], "observables": [ "PHP out-of-memory (OOM) errors", "Node.js Brotli decoder crashes", "GitLab integer overflow during regex compilation", "Application Error 1000", "Exception code c0000005" ] }, { "name": "Supply Chain Poisoning", "slug": "malicious-dependency-installation", "tactic": "initial-access", "techniques": [ "T1195.002" ], "observables": [ "litellm==1.107.0", "Docker tags: v1.107.0, latest, main-latest, main-stable", "Malicious images pushed to official container registries" ] }, { "name": "Python Startup Hijack", "slug": "python-startup-persistence", "tactic": "persistence", "techniques": [ "T1574.013" ], "observables": [ ".pth files created in Python site-packages", "Abuse of Python startup mechanisms" ] }, { "name": "Credential Theft and RAT Activity", "slug": "credential-theft-and-rat-c2", "tactic": "credential-access", "techniques": [ "T1555", "T1071.001" ], "observables": [ "Credential stealer payload", "Remote Access Trojan (RAT) execution", "Outbound C2 connections from Python processes", "Access to sensitive local secret stores" ] }, { "name": "Intermediate CA Trust Compromise", "slug": "compromised-certificate-trust", "tactic": "initial-access", "techniques": [ "T1553.004" ], "observables": [ "eMudhra/emSign intermediate CAs with insecure private keys", "certifi < 2026.10.19", "pip < 26.2.2", "uv < 0.13.4" ] } ], "summary": "A barrage of critical vulnerabilities in October 2026 including PHP2Shell, Node2Shell, and GitLab RCE enabled unauthenticated remote code execution, while a supply-chain compromise of the LiteLLM package facilitated persistent credential theft. Attackers poisoned LiteLLM's PyPI and Docker distributions with a payload that hijacks Python startup via .pth files and establishes a remote access trojan (RAT)." }, "severity": "high", "rationale": "Focus on developers and CI/CD runners where LiteLLM is likely used. ephemeral build agents are high priority.", "guardrails": { "claims": "no_unsupported", "evidence": "citation_required", "telemetry": "untrusted", "missing_data": "not_benign" }, "hypothesis": "An adversary has installed a poisoned version of the LiteLLM library to establish persistence via Python startup files and exfiltrate host credentials.", "parameters": { "scope_hosts": { "from": { "ref": "analyst-defined", "kind": "manual", "observed": "2026-10-28" }, "type": "list[host]", "default": [], "description": "Optional list of hostnames to focus on after the scoping step." }, "lookback_days": { "from": { "ref": "incident-window", "kind": "manual", "observed": "2026-10-21" }, "type": "number", "default": "14", "description": "Days of history to examine for installation and behavior." }, "malicious_version": { "from": { "ref": "https://www.welivesecurity.com/en/business-security/lessons-cybersecurity-black-week/", "kind": "article", "observed": "2026-10-21" }, "type": "string", "default": "1.107.0", "description": "The specific compromised version of LiteLLM." } }, "provenance": { "authors": [ { "org": "huntbase.io", "name": "Huntbase hunt generation" } ], "generated": { "by": "huntbase-hunt-generation", "from": "https://www.welivesecurity.com/en/business-security/lessons-cybersecurity-black-week/", "gates": [ "dry-run", "lint" ], "model": "hb_google/gemini-3-flash-preview" } }, "references": [ { "url": "https://www.welivesecurity.com/en/business-security/lessons-cybersecurity-black-week/", "name": "ESET Research \u2014 Lessons from Cybersecurity Black Week" } ], "blind_spots": [ { "id": "no-process-telemetry", "risk": "A host that installs the package but does not report process history will be missed by the lead query.", "stage": "malicious-dependency-installation", "question": "whether the installation occurred on a host without process logging", "requires": "hb_process_activity with command-line auditing" }, { "id": "insufficient-file-logs", "risk": "If the RAT targets non-standard secret locations, the behavior will be invisible.", "stage": "credential-theft-and-rat-c2", "question": "whether the RAT accessed secrets not in the standard searched paths", "requires": "hb_file_activity with activity_id = 2 (Read) auditing for sensitive directories" } ] }, "name": "LiteLLM Supply Chain Poisoning and Persistence", "description": "This hunt targets the Cybersecurity Black Week compromise of the LiteLLM library (v1.107.0). It follows a gated flow, starting with a lead query that identifies the malicious package installation in process history to capture activity on ephemeral hosts. If the lead is positive, the hunt fans out to find rare Python persistence (.pth files) and sensitive credential access across the fleet. An agent then correlates these signals to confirm a compromise and trigger containment." }, "nodes": [ { "id": "hypothesis", "type": "hypothesis", "label": "Hypothesis", "config": { "tags": [], "series": { "slug": "lessons-from-cybersecurity-black-week", "index": 2, "title": "Lessons from Cybersecurity Black Week", "total": 2 }, "coverage": [ { "stage": "malicious-dependency-installation", "steps": [ "litellm-install-lead", "litellm-inventory-check" ], "status": "covered" }, { "stage": "python-startup-persistence", "steps": [ "pth-persistence-prevalence" ], "status": "covered" }, { "stage": "credential-theft-and-rat-c2", "steps": [ "python-credential-theft" ], "status": "covered" }, { "stage": "rce-vulnerability-exposure", "reason": "Belongs to another part of the 'Lessons from Cybersecurity Black Week' series.", "status": "out_of_scope" }, { "stage": "memory-safety-exploitation-crashes", "reason": "Belongs to another part of the 'Lessons from Cybersecurity Black Week' series.", "status": "out_of_scope" }, { "stage": "compromised-certificate-trust", "reason": "Belongs to another part of the 'Lessons from Cybersecurity Black Week' series.", "status": "out_of_scope" } ], "rationale": "An adversary has installed a poisoned version of the LiteLLM library to establish persistence via Python startup files and exfiltrate host credentials.", "blind_spots": [ { "id": "no-process-telemetry", "risk": "A host that installs the package but does not report process history will be missed by the lead query.", "stage": "malicious-dependency-installation", "question": "whether the installation occurred on a host without process logging", "requires": "hb_process_activity with command-line auditing" }, { "id": "insufficient-file-logs", "risk": "If the RAT targets non-standard secret locations, the behavior will be invisible.", "stage": "credential-theft-and-rat-c2", "question": "whether the RAT accessed secrets not in the standard searched paths", "requires": "hb_file_activity with activity_id = 2 (Read) auditing for sensitive directories" } ], "scoping_notes": "Focus on developers and CI/CD runners where LiteLLM is likely used. ephemeral build agents are high priority.", "beyond_detection": "A static rule for package versions is bypassed by renaming. This hunt uses prevalence counting across .pth files and correlates it with specific Python-based file access behavioral patterns, providing a cross-surface signal that simple version checks miss." } }, { "id": "litellm-install-lead", "type": "query", "label": "Detect malicious LiteLLM installation events", "config": { "dsl": "sqlite", "role": "scoping", "source": "endpoint", "content": "SELECT device_hostname, process_name, process_cmd_line, user_name, time FROM hb_process_activity WHERE (LOWER(process_cmd_line) LIKE '%pip%install%litellm%{{malicious_version}}%' OR LOWER(process_cmd_line) LIKE '%litellm-{{malicious_version}}%') AND time >= datetime('now', '-{{lookback_days}} days')", "surface": "hb_process_activity", "description": "Identify hosts that attempted to install the compromised version of LiteLLM, including ephemeral hosts.", "expected_signal": "Rows showing pip or python commands installing the malicious version. Silence means no recorded installation events in the window." }, "parents": [ { "id": "hypothesis" } ], "primitive_config": { "dsl": "sqlite", "role": "scoping", "label": "Detect malicious LiteLLM installation events", "reads": [ "device_hostname", "process_name", "process_cmd_line", "user_name", "time" ], "source": "hb_process_activity", "target": "endpoint", "content": "SELECT device_hostname, process_name, process_cmd_line, user_name, time FROM hb_process_activity WHERE (LOWER(process_cmd_line) LIKE '%pip%install%litellm%{{malicious_version}}%' OR LOWER(process_cmd_line) LIKE '%litellm-{{malicious_version}}%') AND time >= datetime('now', '-{{lookback_days}} days')", "silence": "not_evidence_of_absence", "expected": "Rows showing pip or python commands installing the malicious version. Silence means no recorded installation events in the window.", "verified": "dry-run", "verified_at": "2026-10-29" } }, { "id": "assess-lead", "type": "analytic", "label": "Assess lead evidence", "config": { "cite": "required", "tools": [ "endpoint" ], "context": [ "litellm-install-lead" ], "objective": "Identify hosts that have LiteLLM version 1.107.0 installation activity in their process logs.", "description": "Read only the lead query to confirm if any host is affected.", "max_iterations": 3, "expected_signal": "A list of hosts with confirmed LiteLLM installation activity.", "success_criteria": "A verdict naming the affected hosts." }, "parents": [ { "id": "litellm-install-lead" } ] }, { "id": "gate-on-install", "type": "checkpoint", "label": "Gate on installation activity", "config": { "fuzzy": true, "judge": "hunter", "question": "the agent confirms at least one host has activity relating to version 1.107.0 of LiteLLM", "condition": "the agent confirms at least one host has activity relating to version 1.107.0 of LiteLLM", "blind_spot": "no-process-telemetry", "confidence": "high", "description": "Open the expensive behavioral queries only when the lead is real.", "checkpoint_type": "mandatory" }, "parents": [ { "id": "assess-lead" } ] }, { "id": "pth-persistence-prevalence", "type": "query", "label": "Rare .pth file persistence", "config": { "dsl": "sqlite", "role": "baseline", "source": "endpoint", "content": "SELECT file_name, file_path, process_name, process_cmd_line, COUNT(DISTINCT device_hostname) as host_count, MIN(time) as first_seen FROM hb_file_activity WHERE LOWER(file_name) LIKE '%.pth' AND (LOWER(file_path) LIKE '%site-packages%' OR LOWER(file_path) LIKE '%dist-packages%') AND ('{{scope_hosts}}' = '' OR instr(',' || '{{scope_hosts}}' || ',', ',' || device_hostname || ',') > 0) AND time >= datetime('now', '-{{lookback_days}} days') GROUP BY file_name, file_path, process_name, process_cmd_line HAVING host_count < 5", "surface": "hb_file_activity", "description": "Stack-count .pth file creations to find rare startup hijacks and attribute them to a process.", "expected_signal": "Rare .pth files created on one or two hosts. Persistent malicious startup files will stand out from common library files." }, "parents": [ { "id": "gate-on-install", "branch": "on_supports" } ], "primitive_config": { "dsl": "sqlite", "role": "baseline", "label": "Rare .pth file persistence", "reads": [ "file_name", "file_path", "device_hostname", "process_name", "process_cmd_line", "time" ], "source": "hb_file_activity", "target": "endpoint", "content": "SELECT file_name, file_path, process_name, process_cmd_line, COUNT(DISTINCT device_hostname) as host_count, MIN(time) as first_seen FROM hb_file_activity WHERE LOWER(file_name) LIKE '%.pth' AND (LOWER(file_path) LIKE '%site-packages%' OR LOWER(file_path) LIKE '%dist-packages%') AND ('{{scope_hosts}}' = '' OR instr(',' || '{{scope_hosts}}' || ',', ',' || device_hostname || ',') > 0) AND time >= datetime('now', '-{{lookback_days}} days') GROUP BY file_name, file_path, process_name, process_cmd_line HAVING host_count < 5", "silence": "not_evidence_of_absence", "baseline": { "window": "{{lookback_days}}d", "compare": "first_seen" }, "expected": "Rare .pth files created on one or two hosts. Persistent malicious startup files will stand out from common library files.", "verified": "dry-run", "prevalence": { "by": "device_hostname", "key": [ "file_name", "file_path" ], "rare_below": 5 }, "verified_at": "2026-10-29" } }, { "id": "litellm-inventory-check", "type": "query", "label": "Check current package inventory", "config": { "dsl": "sqlite", "role": "enrichment", "source": "endpoint", "content": "SELECT device_hostname, package_name, package_version, install_path FROM hb_software_inventory WHERE LOWER(package_name) = 'litellm' AND package_version = '{{malicious_version}}' AND ('{{scope_hosts}}' = '' OR instr(',' || '{{scope_hosts}}' || ',', ',' || device_hostname || ',') > 0)", "surface": "hb_software_inventory", "description": "Confirm if the malicious version is currently installed on any disk in the fleet.", "expected_signal": "A snapshot of hosts with the vulnerable version currently installed." }, "parents": [ { "id": "gate-on-install", "branch": "on_supports" } ], "primitive_config": { "dsl": "sqlite", "role": "enrichment", "label": "Check current package inventory", "reads": [ "device_hostname", "package_name", "package_version", "install_path" ], "source": "hb_software_inventory", "target": "endpoint", "content": "SELECT device_hostname, package_name, package_version, install_path FROM hb_software_inventory WHERE LOWER(package_name) = 'litellm' AND package_version = '{{malicious_version}}' AND ('{{scope_hosts}}' = '' OR instr(',' || '{{scope_hosts}}' || ',', ',' || device_hostname || ',') > 0)", "silence": "not_evidence_of_absence", "expected": "A snapshot of hosts with the vulnerable version currently installed.", "verified": "dry-run", "verified_at": "2026-10-29" } }, { "id": "python-credential-theft", "type": "query", "label": "Python credential file access", "config": { "dsl": "sqlite", "role": "detection-candidate", "source": "endpoint", "content": "SELECT device_hostname, process_name, process_cmd_line, file_path, time FROM hb_file_activity WHERE activity_id = 2 AND LOWER(process_name) LIKE '%python%' AND (LOWER(file_path) LIKE '%/.aws/credentials' OR LOWER(file_path) LIKE '%/.ssh/id_%' OR LOWER(file_path) LIKE '%/config.json' OR LOWER(file_path) LIKE '%/.dockercfg' OR LOWER(file_path) LIKE '%/.npmrc') AND ('{{scope_hosts}}' = '' OR instr(',' || '{{scope_hosts}}' || ',', ',' || device_hostname || ',') > 0) AND time >= datetime('now', '-{{lookback_days}} days')", "surface": "hb_file_activity", "description": "Find Python processes reading sensitive local secret stores.", "expected_signal": "Reads of cloud or SSH credentials by a Python interpreter. Legitimate tools may exist, but those matching the scope hosts are high risk." }, "parents": [ { "id": "gate-on-install", "branch": "on_supports" } ], "primitive_config": { "dsl": "sqlite", "role": "detection-candidate", "label": "Python credential file access", "reads": [ "device_hostname", "process_name", "process_cmd_line", "file_path", "time", "activity_id" ], "source": "hb_file_activity", "target": "endpoint", "content": "SELECT device_hostname, process_name, process_cmd_line, file_path, time FROM hb_file_activity WHERE activity_id = 2 AND LOWER(process_name) LIKE '%python%' AND (LOWER(file_path) LIKE '%/.aws/credentials' OR LOWER(file_path) LIKE '%/.ssh/id_%' OR LOWER(file_path) LIKE '%/config.json' OR LOWER(file_path) LIKE '%/.dockercfg' OR LOWER(file_path) LIKE '%/.npmrc') AND ('{{scope_hosts}}' = '' OR instr(',' || '{{scope_hosts}}' || ',', ',' || device_hostname || ',') > 0) AND time >= datetime('now', '-{{lookback_days}} days')", "silence": "not_evidence_of_absence", "expected": "Reads of cloud or SSH credentials by a Python interpreter. Legitimate tools may exist, but those matching the scope hosts are high risk.", "verified": "dry-run", "verified_at": "2026-10-29" } }, { "id": "final-triage", "type": "analytic", "label": "Triage compromise and impact", "config": { "cite": "required", "tools": [ "endpoint" ], "context": [ "assess-lead", "pth-persistence-prevalence", "litellm-inventory-check", "python-credential-theft" ], "objective": "Determine if the compromised LiteLLM package has achieved persistence or attempted credential theft on the affected hosts.", "description": "A second agent weighs the installation lead with the behavioral evidence.", "max_iterations": 6, "expected_signal": "A per-host verdict citing specific evidence of compromise.", "success_criteria": "A per-host verdict of malicious | suspicious | benign." }, "parents": [ { "id": "pth-persistence-prevalence", "kind": "merge" }, { "id": "litellm-inventory-check", "kind": "merge" }, { "id": "python-credential-theft", "kind": "merge" } ] }, { "id": "route-on-verdict", "type": "checkpoint", "label": "Route on triage verdict", "config": { "fuzzy": true, "judge": "hunter", "question": "the triage verdict is malicious for at least one host, showing both package presence and either persistence (.pth) or credential access", "condition": "the triage verdict is malicious for at least one host, showing both package presence and either persistence (.pth) or credential access", "blind_spot": "insufficient-file-logs", "confidence": "high", "description": "Direct confirmed hits to containment and uncertain ones to review.", "checkpoint_type": "mandatory" }, "parents": [ { "id": "final-triage" } ] }, { "id": "contain-host", "type": "action", "label": "Isolate compromised host", "config": { "target": "endpoint", "description": "Stop the RAT and credential exfiltration immediately.", "instructions": "Isolate the host and initiate the supply-chain incident response protocol, specifically focusing on credential rotation.", "action_approval": "required" }, "parents": [ { "id": "route-on-verdict", "branch": "on_supports" } ] }, { "id": "analyst-review", "type": "task", "label": "Analyst review", "config": { "assignee": "analyst", "description": "Verify behavioral findings and identify exposed secrets.", "instructions": "Review the identified .pth files and the Python processes that read sensitive files. Confirm if LiteLLM was the primary actor and determine which cloud or SSH credentials were leaked." }, "parents": [ { "id": "gate-on-install", "branch": "default" }, { "id": "gate-on-install", "branch": "on_unavailable" }, { "id": "route-on-verdict", "branch": "default" }, { "id": "route-on-verdict", "branch": "on_unavailable" }, { "id": "contain-host" } ] }, { "id": "close-out", "type": "task", "label": "Close out", "config": { "assignee": "analyst", "description": "Final reporting and health check.", "instructions": "Document the absence of malicious LiteLLM versions or the containment of any hits. Ensure all containers and virtual environments are updated to 1.107.1 or higher." }, "parents": [ { "id": "gate-on-install", "branch": "on_refutes" }, { "id": "route-on-verdict", "branch": "on_refutes" }, { "id": "analyst-review" } ] } ] }