{ "hunt": { "meta": { "tlp": "clear", "hunt": { "handoff": "promote-to-detection", "trigger": "intel-report", "methodology": "model-assisted", "applicability": "campaign-specific", "justification": "Exploitation of AI/ML orchestration tools like Ray is an active vector for cryptomining. Detecting the staging and initial reverse shell is the only way to catch this intrusion before the kernel rootkit is loaded and evades standard EDR monitoring." }, "name": "Ray Framework Exploitation and SANSAR Staging", "type": "investigation", "labels": [ "hunt", "attack.t1059.004", "attack.t1059.006", "attack.t1543.002", "attack.t1098.004", "command and control", "defense evasion", "execution", "impact", "initial access", "persistence" ], "series": { "slug": "0-gpu-utilization-how-sansar-hides-linux-cryptomining", "index": 1, "title": "0% GPU utilization: How SANSAR hides Linux cryptomining", "total": 2 }, "related": [ { "hunt": "sansar-c2-and-evasion", "reason": "This hunt focuses on the initial access; the next hunt in the series focuses on the Tor-based C2 and rootkit stealth mechanisms.", "relation": "follows" } ], "targets": { "hunter": { "name": "Hunt agent", "agent": true }, "analyst": { "name": "Tier-2 analyst", "role": "analyst" }, "endpoint": { "name": "Endpoint telemetry (hb_ surfaces)", "category": "endpoint", "telemetry": [ "endpoint" ] } }, "analysis": "A simple rule might catch a Python shell, but this hunt pivots from specific staging paths to rare persistence mechanisms and uses stack-counting to differentiate malicious systemd services from legitimate fleet-wide management, providing context a single rule cannot.", "coverage": [ { "stage": "initial-access-ray-job-agent", "steps": [ "python-reverse-shells" ], "status": "covered" }, { "stage": "execution-reverse-shell", "steps": [ "python-reverse-shells" ], "status": "covered" }, { "stage": "persistence-toolkit-deployment", "steps": [ "staging-path-lead", "rare-systemd-persistence" ], "status": "covered" }, { "stage": "defense-evasion-stealth-mechanisms", "reason": "Belongs to another part of the '0% GPU utilization: How SANSAR hides Linux cryptomining' series.", "status": "out_of_scope" }, { "stage": "c2-tor-and-ethereum", "reason": "Belongs to another part of the '0% GPU utilization: How SANSAR hides Linux cryptomining' series.", "status": "out_of_scope" }, { "stage": "impact-cryptomining", "reason": "Belongs to another part of the '0% GPU utilization: How SANSAR hides Linux cryptomining' series.", "status": "out_of_scope" } ], "scenario": { "stages": [ { "name": "Ray Job Agent Exploitation", "slug": "initial-access-ray-job-agent", "tactic": "initial-access", "techniques": [ "T1059" ], "observables": [ "Unauthenticated access to Ray Job API", "External probes in Ray logs", "Job submission to Ray head node" ] }, { "name": "Python Reverse Shell Execution", "slug": "execution-reverse-shell", "tactic": "execution", "techniques": [ "T1059" ], "observables": [ "nvidia-smi --query-gpu=name", "Python reverse-shell job submission", "Connection to external listener via Python" ] }, { "name": "Persistence and Toolkit Deployment", "slug": "persistence-toolkit-deployment", "tactic": "persistence", "techniques": [ "T1059" ], "observables": [ "Staging under /opt/vadi_/", "Modification of authorized_keys for SSH access", "netns backdoor registered as a systemd service", "acpi_thermal_mgr kernel module built via DKMS", "initramfs loader modification" ] }, { "name": "Stealth and Telemetry Falsification", "slug": "defense-evasion-stealth-mechanisms", "tactic": "defense-evasion", "techniques": [ "T1059" ], "observables": [ "acpi_thermal_mgr.ko loading", "Replacement of libnvidia-ml.so.1 with modified wrapper", "Presence of libnvidia-ml.so.1.backup", "SANSAR anti-debugging TracerPid check in /proc/self/status" ] }, { "name": "Evasive Command and Control", "slug": "c2-tor-and-ethereum", "tactic": "command-and-control", "techniques": [ "T1090.003" ], "observables": [ "efxisnqldelyr5ra25oqfrsut54b34g26e7zrsiuupsoekjof2gho2id.onion:9051", "api.ipify.org", "ifconfig.me", "icanhazip.com", "checkip.amazonaws.com", "Lyrebird transport helper usage (obfs4, WebTunnel)", "Ethereum smart contract calls for netns C2 resolution" ] }, { "name": "Cryptomining Resource Hijacking", "slug": "impact-cryptomining", "tactic": "impact", "techniques": [ "T1496" ], "observables": [ "kurt (XMRig 6.25.0) process activity", "tilki (SRBMiner-MULTI 3.6.9) process activity", "Deployment of GPU mining contexts" ] } ], "summary": "Threat actor REF3364 targets Linux GPU hosts by exploiting unauthenticated Ray job agents to gain root access and persistent shell control. The campaign deploys the SANSAR toolkit, which employs a kernel rootkit to hide its presence and modified NVIDIA libraries to falsify GPU utilization while managing cryptomining and backdoors via Tor and Ethereum-based infrastructure." }, "severity": "high", "rationale": "Start by identifying hosts running Ray (look for processes named 'ray' or 'gcs_server'). The lead query focuses on the unique staging path /opt/vadi_/ to quickly isolate affected nodes before performing heavier behavioral checks.", "guardrails": { "claims": "no_unsupported", "evidence": "citation_required", "telemetry": "untrusted", "missing_data": "not_benign" }, "hypothesis": "An attacker has exploited an unauthenticated Ray Job API to execute a Python reverse shell and stage the SANSAR toolkit under /opt/vadi_/, including persistence via SSH and systemd.", "parameters": { "scope_hosts": { "from": { "ref": "analyst-scoping", "kind": "manual", "observed": "2024-11-12" }, "type": "list[host]", "default": [], "description": "Target hosts identified in the lead query; leave empty to scan the entire estate." }, "staging_path": { "from": { "ref": "REF3364", "kind": "article", "observed": "2024-11-12" }, "type": "path", "default": "/opt/vadi_/", "description": "The reported staging path for the SANSAR toolkit." }, "lookback_days": { "from": { "ref": "default-retention", "kind": "manual", "observed": "2024-11-12" }, "type": "number", "default": "14", "description": "Days of history to examine." } }, "provenance": { "authors": [ { "org": "huntbase.io", "name": "Huntbase hunt generation" } ], "generated": { "by": "huntbase-hunt-generation", "from": "https://www.elastic.co/security-labs/threat-command/sansar-linux-cryptomining-malware", "gates": [ "dry-run", "lint", "critic" ], "model": "hb_google/gemini-3-flash-preview" } }, "references": [ { "url": "https://www.elastic.co/security-labs/threat-command/sansar-linux-cryptomining-malware", "name": "Elastic Security Labs \u2014 0% GPU utilization: How SANSAR hides Linux cryptomining" } ], "blind_spots": [ { "id": "limited-process-telemetry", "risk": "Attackers can submit Python code that executes purely in-memory without spawning observable shell child processes.", "stage": "initial-access-ray-job-agent", "question": "What specifically was submitted via the Ray Jobs API?", "requires": "Deep inspection of Ray job submission logs" }, { "id": "limited-file-telemetry", "risk": "Once the rootkit is loaded, it can manipulate process and file listings, making the presence of the toolkit invisible to standard user-space queries.", "stage": "persistence-toolkit-deployment", "question": "Was the kernel rootkit successfully compiled and loaded?", "requires": "hb_linux_audit for DKMS build monitoring" } ] }, "name": "Ray Framework Exploitation and SANSAR Staging", "description": "This hunt targets the initial access and persistence phases of the REF3364 SANSAR intrusion. It searches for indicators of Ray Job Agent abuse, specifically focusing on suspicious shell processes spawned from Python workloads and the subsequent deployment of the SANSAR toolkit in specific staging directories. The hunt uses a gated flow to first confirm the presence of the unique staging path before initiating a broader search for rare systemd persistence and reverse shell behavior across the estate." }, "nodes": [ { "id": "hypothesis", "type": "hypothesis", "label": "Hypothesis", "config": { "tags": [], "series": { "slug": "0-gpu-utilization-how-sansar-hides-linux-cryptomining", "index": 1, "title": "0% GPU utilization: How SANSAR hides Linux cryptomining", "total": 2 }, "coverage": [ { "stage": "initial-access-ray-job-agent", "steps": [ "python-reverse-shells" ], "status": "covered" }, { "stage": "execution-reverse-shell", "steps": [ "python-reverse-shells" ], "status": "covered" }, { "stage": "persistence-toolkit-deployment", "steps": [ "staging-path-lead", "rare-systemd-persistence" ], "status": "covered" }, { "stage": "defense-evasion-stealth-mechanisms", "reason": "Belongs to another part of the '0% GPU utilization: How SANSAR hides Linux cryptomining' series.", "status": "out_of_scope" }, { "stage": "c2-tor-and-ethereum", "reason": "Belongs to another part of the '0% GPU utilization: How SANSAR hides Linux cryptomining' series.", "status": "out_of_scope" }, { "stage": "impact-cryptomining", "reason": "Belongs to another part of the '0% GPU utilization: How SANSAR hides Linux cryptomining' series.", "status": "out_of_scope" } ], "rationale": "An attacker has exploited an unauthenticated Ray Job API to execute a Python reverse shell and stage the SANSAR toolkit under /opt/vadi_/, including persistence via SSH and systemd.", "blind_spots": [ { "id": "limited-process-telemetry", "risk": "Attackers can submit Python code that executes purely in-memory without spawning observable shell child processes.", "stage": "initial-access-ray-job-agent", "question": "What specifically was submitted via the Ray Jobs API?", "requires": "Deep inspection of Ray job submission logs" }, { "id": "limited-file-telemetry", "risk": "Once the rootkit is loaded, it can manipulate process and file listings, making the presence of the toolkit invisible to standard user-space queries.", "stage": "persistence-toolkit-deployment", "question": "Was the kernel rootkit successfully compiled and loaded?", "requires": "hb_linux_audit for DKMS build monitoring" } ], "scoping_notes": "Start by identifying hosts running Ray (look for processes named 'ray' or 'gcs_server'). The lead query focuses on the unique staging path /opt/vadi_/ to quickly isolate affected nodes before performing heavier behavioral checks.", "beyond_detection": "A simple rule might catch a Python shell, but this hunt pivots from specific staging paths to rare persistence mechanisms and uses stack-counting to differentiate malicious systemd services from legitimate fleet-wide management, providing context a single rule cannot." } }, { "id": "staging-path-lead", "type": "query", "label": "Lead: SANSAR Staging Directory Activity", "config": { "dsl": "sqlite", "role": "scoping", "source": "endpoint", "content": "SELECT device_hostname, process_name, process_cmd_line, user_name, time FROM hb_process_activity WHERE (LOWER(process_path) LIKE '{{staging_path}}%' OR LOWER(process_cmd_line) LIKE '%{{staging_path}}%') AND time >= datetime('now', '-{{lookback_days}} days')", "surface": "hb_process_activity", "description": "Identify any process activity or file access involving the specific staging directory used by the threat actor to establish a lead.", "expected_signal": "Processes executing from or referencing the /opt/vadi_/ directory. Silence suggests this specific campaign staging has not occurred on the estate." }, "parents": [ { "id": "hypothesis" } ], "primitive_config": { "dsl": "sqlite", "role": "scoping", "label": "Lead: SANSAR Staging Directory Activity", "reads": [ "device_hostname", "process_cmd_line", "process_name", "process_path", "time", "user_name" ], "source": "hb_process_activity", "target": "endpoint", "content": "SELECT device_hostname, process_name, process_cmd_line, user_name, time FROM hb_process_activity WHERE (LOWER(process_path) LIKE '{{staging_path}}%' OR LOWER(process_cmd_line) LIKE '%{{staging_path}}%') AND time >= datetime('now', '-{{lookback_days}} days')", "silence": "not_evidence_of_absence", "expected": "Processes executing from or referencing the /opt/vadi_/ directory. Silence suggests this specific campaign staging has not occurred on the estate.", "verified": "dry-run", "verified_at": "2026-11-13" } }, { "id": "evaluate-staging-lead", "type": "analytic", "label": "Evaluate Staging Lead", "config": { "cite": "required", "tools": [ "endpoint" ], "context": [ "staging-path-lead" ], "objective": "Determine if the processes found in the staging path are consistent with the SANSAR orchestrator or related miners.", "description": "Analyze the lead results to determine if they match the REF3364 toolkit deployment patterns.", "max_iterations": 3, "expected_signal": "A high-confidence verdict on whether the activity warrants further investigation.", "success_criteria": "A clear decision to open the gated queries or terminate the hunt." }, "parents": [ { "id": "staging-path-lead" } ] }, { "id": "gate-check", "type": "checkpoint", "label": "Gate: Staging Found", "config": { "fuzzy": true, "judge": "hunter", "question": "The evaluate-staging-lead verdict confirms suspicious activity or unauthorized directory creation", "condition": "The evaluate-staging-lead verdict confirms suspicious activity or unauthorized directory creation", "blind_spot": "limited-process-telemetry", "confidence": "high", "description": "Route the hunt based on the agent's evaluation of the lead.", "checkpoint_type": "mandatory" }, "parents": [ { "id": "evaluate-staging-lead" } ] }, { "id": "rare-systemd-persistence", "type": "query", "label": "Prevalence: Rare Systemd Services", "config": { "dsl": "sqlite", "role": "baseline", "source": "endpoint", "content": "SELECT file_path, COUNT(DISTINCT device_hostname) AS hosts, MIN(time) AS first_seen FROM hb_file_activity WHERE (file_path LIKE '/etc/systemd/system/%' OR file_path LIKE '/usr/lib/systemd/system/%') AND activity_id = 1 AND ('{{scope_hosts}}' = '' OR instr(',' || '{{scope_hosts}}' || ',', ',' || device_hostname || ',') > 0) AND time >= datetime('now', '-{{lookback_days}} days') GROUP BY file_path HAVING hosts <= 3", "surface": "hb_file_activity", "description": "Identify rare systemd service files created on the estate, which may correspond to the netns backdoor.", "expected_signal": "New systemd unit files appearing on only a few hosts. Legit services are usually fleet-wide." }, "parents": [ { "id": "gate-check", "branch": "on_supports" } ], "primitive_config": { "dsl": "sqlite", "role": "baseline", "label": "Prevalence: Rare Systemd Services", "reads": [ "activity_id", "device_hostname", "file_path", "time" ], "source": "hb_file_activity", "target": "endpoint", "content": "SELECT file_path, COUNT(DISTINCT device_hostname) AS hosts, MIN(time) AS first_seen FROM hb_file_activity WHERE (file_path LIKE '/etc/systemd/system/%' OR file_path LIKE '/usr/lib/systemd/system/%') AND activity_id = 1 AND ('{{scope_hosts}}' = '' OR instr(',' || '{{scope_hosts}}' || ',', ',' || device_hostname || ',') > 0) AND time >= datetime('now', '-{{lookback_days}} days') GROUP BY file_path HAVING hosts <= 3", "silence": "not_evidence_of_absence", "baseline": { "window": "{{lookback_days}}d", "compare": "first_seen" }, "expected": "New systemd unit files appearing on only a few hosts. Legit services are usually fleet-wide.", "verified": "dry-run", "prevalence": { "by": "device_hostname", "key": [ "file_path" ], "rare_below": 3 }, "verified_at": "2026-11-13" } }, { "id": "python-reverse-shells", "type": "query", "label": "Behaviour: Python Reverse Shells from Ray", "config": { "dsl": "sqlite", "role": "detection-candidate", "source": "endpoint", "content": "SELECT device_hostname, process_name, process_cmd_line, parent_process_name, user_name, time FROM hb_process_activity WHERE (LOWER(process_cmd_line) LIKE '%python%' AND (LOWER(process_cmd_line) LIKE '%socket%' OR LOWER(process_cmd_line) LIKE '%pty.spawn%')) AND ('{{scope_hosts}}' = '' OR instr(',' || '{{scope_hosts}}' || ',', ',' || device_hostname || ',') > 0) AND time >= datetime('now', '-{{lookback_days}} days')", "surface": "hb_process_activity", "description": "Identify Python processes exhibiting reverse shell syntax that are spawned from a Ray or web service context.", "expected_signal": "Python shells spawned by non-interactive accounts. This confirms the initial access phase of the intrusion." }, "parents": [ { "id": "gate-check", "branch": "on_supports" } ], "primitive_config": { "dsl": "sqlite", "role": "detection-candidate", "label": "Behaviour: Python Reverse Shells from Ray", "reads": [ "device_hostname", "parent_process_name", "process_cmd_line", "process_name", "time", "user_name" ], "source": "hb_process_activity", "target": "endpoint", "content": "SELECT device_hostname, process_name, process_cmd_line, parent_process_name, user_name, time FROM hb_process_activity WHERE (LOWER(process_cmd_line) LIKE '%python%' AND (LOWER(process_cmd_line) LIKE '%socket%' OR LOWER(process_cmd_line) LIKE '%pty.spawn%')) AND ('{{scope_hosts}}' = '' OR instr(',' || '{{scope_hosts}}' || ',', ',' || device_hostname || ',') > 0) AND time >= datetime('now', '-{{lookback_days}} days')", "silence": "not_evidence_of_absence", "expected": "Python shells spawned by non-interactive accounts. This confirms the initial access phase of the intrusion.", "verified": "dry-run", "verified_at": "2026-11-13" } }, { "id": "final-triage", "type": "analytic", "label": "Final Triage and Correlation", "config": { "cite": "required", "tools": [ "endpoint" ], "context": [ "evaluate-staging-lead", "rare-systemd-persistence", "python-reverse-shells" ], "objective": "Determine if the host has been compromised via Ray and whether the SANSAR toolkit is actively persisting and executing shell commands.", "description": "Correlate the staging lead with the persistence and execution evidence to reach a final verdict.", "max_iterations": 6, "expected_signal": "A detailed per-host verdict citing specific process and file artifacts.", "success_criteria": "A verdict of malicious | suspicious | benign per host, citing relevant rows." }, "parents": [ { "id": "rare-systemd-persistence", "kind": "merge" }, { "id": "python-reverse-shells", "kind": "merge" } ] }, { "id": "route-verdict", "type": "checkpoint", "label": "Route Final Verdict", "config": { "fuzzy": true, "judge": "hunter", "question": "the final-triage verdict is malicious for at least one host", "condition": "the final-triage verdict is malicious for at least one host", "blind_spot": "limited-file-telemetry", "confidence": "high", "description": "Determine whether to trigger automated containment or move to manual review.", "checkpoint_type": "mandatory" }, "parents": [ { "id": "final-triage" } ] }, { "id": "isolate-host", "type": "action", "label": "Isolate Compromised Host", "config": { "target": "endpoint", "description": "Contain the threat and stop the execution of shells and mining tools.", "instructions": "Isolate the host, terminate the identified Python reverse shell processes, and preserve the staging directory for forensic analysis.", "action_approval": "required" }, "parents": [ { "id": "route-verdict", "branch": "on_supports" } ] }, { "id": "analyst-review", "type": "task", "label": "Analyst Review", "config": { "assignee": "analyst", "description": "Validate the findings and identify the source of the Ray Job API exposure.", "instructions": "Verify the presence of the /opt/vadi_/ directory and any new SSH keys in authorized_keys. Confirm if the Ray head node dashboard or job agent is reachable from the internet." }, "parents": [ { "id": "gate-check", "branch": "default" }, { "id": "gate-check", "branch": "on_unavailable" }, { "id": "route-verdict", "branch": "default" }, { "id": "route-verdict", "branch": "on_unavailable" }, { "id": "isolate-host" } ] }, { "id": "close-out", "type": "task", "label": "Hunt Close-out", "config": { "assignee": "analyst", "description": "Document the final state of the hunt and findings.", "instructions": "Record whether any compromised hosts were found and note any Ray frameworks that require hardening." }, "parents": [ { "id": "gate-check", "branch": "on_refutes" }, { "id": "route-verdict", "branch": "on_refutes" } ] } ] }