diff --git a/conf/default/integrations.conf.default b/conf/default/integrations.conf.default index 578a4503f01..78bf7e0108e 100644 --- a/conf/default/integrations.conf.default +++ b/conf/default/integrations.conf.default @@ -148,3 +148,93 @@ tight_strings = yes min_length = 5 # Download FLOSS signatures from https://github.com/mandiant/flare-floss/tree/master/sigs sigs_path = data/flare-signatures + +# Code-similarity engines +# Enable too in processing.conf. +# We recommend ensuring local similairty engines such as mcrit are populated with unpacked samples ideally aquired from CAPE payloads/procdumps or are not heavily packed/obfuscated to ensure best matching. +# Also for accurate matching populate with libraries (i.e. import from https://github.com/danielplohmann/mcrit-data dragging and dropping all the files for each into import, +# or importing binaries and then editing them and marking them as libraries once imported). +[similarity] + +# Hard per-job deadline (seconds) so a stuck engine can't stall processing. +timeout = 120 + +# ---- Engines ------------------------------------------------ +# Only listed engines are attempted. Each engine only receives files in the +# formats it supports, so enabling more engines is safe. Currently only MCRIT is supported (https://github.com/danielplohmann/mcrit) +# although the integration allows adding other performative similairty engines later. +mcrit = yes +# Formats sent to MCRIT. MCRIT's SMDA disassembler supports PE, ELF and +# Mach-O (x86/x64/aarch64). Narrow +# this only if your MCRIT server runs an older SMDA: e.g. mcrit_formats = pe. +# As CAPE does not support OSX but only Windows and Linux macho is ommited. +mcrit_formats = pe,elf + +# File sources to submit (default is memory extracted files for unpacked files). +similarity_payloads = yes +similarity_procdump = yes +similarity_dropped = no +similarity_submittedfile = no + +# Skip artifacts already resolved to a known family. Set no to +# submit everything (corpus building / re-checking known files). +only_unidentified = yes + +# ---- Result selection (what is SHOWN) ----------------------- +# Minimum similarity % for a match to appear in the report. +minimum_similarity = 50 +# Cap the number of matches returned/shown per artifact (top N by +# score). Tunable limit on samples returned by each engine. +max_results = 3 +# Include matches that carry a family label. +match_families = yes +# Include matches with no family label (shows sha256 + score). +match_nofamily = yes +# If nothing clears minimum_similarity, still show the single best +# hit flagged low-confidence. Off by default. +best_match = no +# Comma-separated family substrings to suppress (case-insensitive). +# exclude_families = msvc,mingw + +# Ignore matches that are mostly shared library / runtime code (Go runtime, +# OpenSSL, the C runtime, etc.) by scoring on MCRIT's non-library-weighted +# similarity and discarding pure-library matches. This is the robust, +# list-free way to keep only malware-code matches and is preferred over +# maintaining a long exclude_families list. Relies on library code in your +# MCRIT corpus being flagged as library samples (is_library) and/or MCRIT's +# automatic library-function detection. +ignore_library_matches = yes + +# ---- Family classification (CAPE malfamily detections) +# When yes, a family match is promoted into CAPE's detections list (main +# page + report header), acting as a classification engine. Only for +# artifacts with no existing family; each family added at most once per run. +update_malfamily = no +# SEPARATE, higher threshold for promotion. Lets low-confidence matches +# still appear in the similarity table (minimum_similarity) while only +# strong matches set the malware family. +malfamily_minimum_similarity = 80 +# Which matches become the malware family when update_malfamily = yes: +# top - only the single highest-similarity family per artifact (default) +# all - every family whose match clears malfamily_minimum_similarity +# Either way each family is added at most once per analysis. +malfamily_mode = top + +# Concurrent submission +# Submit multiple artifacts simultaneously. max_workers bounds simultaneous jobs. +concurrent_submissions = no +max_workers = 4 + +# ---- MCRIT connection --------------------------------------- +# REST API endpoint - NOT the web UI (5000), the API (8000). +mcrit_host = http://127.0.0.1:8000 +mcrit_apitoken = +mcrit_username = cape +mcrit_poll_interval = 3 + +# Persistence: +# no (default) - transient /query: matched on the worker, never +# stored. Nothing to delete; labelled samples never touched. +# yes - stored via /samples/binary for corpus building. A sha256 +# pre-check reuses an existing sample instead of re-adding. +persist_samples = no diff --git a/conf/default/processing.conf.default b/conf/default/processing.conf.default index 1a9cec3e06d..264fa4335d5 100644 --- a/conf/default/processing.conf.default +++ b/conf/default/processing.conf.default @@ -357,3 +357,7 @@ enabled = no # Enable when using the PolarProxy option during analysis. This will merge the tls.pcap containing # plain-text TLS streams into the task PCAP. enabled = no + +# Code-similarity engine(s). See conf/integrations.conf [similarity]. +[similarity] +enabled = no diff --git a/lib/cuckoo/common/integrations/similarity/__init__.py b/lib/cuckoo/common/integrations/similarity/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/lib/cuckoo/common/integrations/similarity/base.py b/lib/cuckoo/common/integrations/similarity/base.py new file mode 100644 index 00000000000..632dc1e7606 --- /dev/null +++ b/lib/cuckoo/common/integrations/similarity/base.py @@ -0,0 +1,189 @@ +# Copyright (C) 2010-2015 Cuckoo Foundation, 2016-2026 CAPE developers. +# This file is part of CAPE Sandbox - http://www.capesandbox.com +# See the file 'docs/LICENSE' for copying permission. + +"""Code-similarity engine framework — base abstractions. + +Zero external dependencies (stdlib + whatever CAPE already ships). New +engines subclass SimilarityEngine, declare the file formats they accept, +implement available()/analyze(), and register in registry.py. +""" + +import logging +from typing import Dict, List, Optional, Set + +log = logging.getLogger(__name__) + +# Canonical file-format tokens used across the framework. +FMT_PE = "pe" +FMT_ELF = "elf" +FMT_MACHO = "macho" +FMT_UNKNOWN = "unknown" + + +def _as_bool(value) -> bool: + if isinstance(value, bool): + return value + if value is None: + return False + return str(value).strip().lower() in ("1", "yes", "true", "on", "y") + + +def detect_format(type_string: Optional[str], magic: Optional[bytes] = None) -> str: + """Map CAPE's libmagic 'type' string (or raw magic) to a format token. + + Prefers CAPE's reported type (already computed during processing); + falls back to magic-byte sniffing when the type string is absent. + """ + t = (type_string or "").lower() + if "pe32" in t or "ms-dos" in t or "ms windows" in t or "for ms windows" in t: + return FMT_PE + if "elf" in t: + return FMT_ELF + if "mach-o" in t or "mach o" in t: + return FMT_MACHO + if magic: + if magic[:2] == b"MZ": + return FMT_PE + if magic[:4] == b"\x7fELF": + return FMT_ELF + if magic[:4] in (b"\xfe\xed\xfa\xce", b"\xfe\xed\xfa\xcf", + b"\xce\xfa\xed\xfe", b"\xcf\xfa\xed\xfe", b"\xca\xfe\xba\xbe"): + return FMT_MACHO + return FMT_UNKNOWN + + +class MatchRecord: + """Single normalized similarity hit, engine-agnostic.""" + + def __init__( + self, + family: Optional[str], + similarity: float, + sample_sha256: Optional[str] = None, + sample_id=None, + matched_functions: Optional[int] = None, + total_functions: Optional[int] = None, + version: Optional[str] = None, + is_low_confidence: bool = False, + ): + self.family = family or None + self.similarity = round(float(similarity), 2) + self.sample_sha256 = sample_sha256 + self.sample_id = sample_id + self.matched_functions = matched_functions + self.total_functions = total_functions + self.version = version + self.is_low_confidence = is_low_confidence + + @property + def has_family(self) -> bool: + return bool(self.family) + + def to_dict(self) -> Dict: + return { + "family": self.family, + "similarity": self.similarity, + "sample_sha256": self.sample_sha256, + "sample_id": self.sample_id, + "matched_functions": self.matched_functions, + "total_functions": self.total_functions, + "version": self.version, + "low_confidence": self.is_low_confidence, + } + + +class EngineResult: + """Per-file result from one engine for one artifact.""" + + def __init__(self, status: str = "ok", error: Optional[str] = None): + # status: ok | no_match | skipped | timeout | error | disabled + self.status = status + self.error = error + self.matches: List[MatchRecord] = [] + + def to_dict(self) -> Dict: + out = {"status": self.status, "matches": [m.to_dict() for m in self.matches]} + if self.error: + out["error"] = self.error + return out + + +class SimilarityEngine: + """Base class for a code-similarity backend. + + Subclasses set ``supported_formats`` to the set of file formats they + can analyze, and implement available()/analyze(). External packages + must only be imported lazily (inside available()) so a missing optional + dependency cannot crash CAPE's plugin loader. + """ + + name = "base" + # Formats this engine accepts. Override per engine. + # MCRIT -> {FMT_PE} (PE only) + # a multi-format engine -> {FMT_PE, FMT_ELF, FMT_MACHO} + supported_formats: Set[str] = {FMT_PE} + + def __init__(self, options: Dict): + self.options = options or {} + self.minimum_similarity = float(self.options.get("minimum_similarity", 50) or 0) + self.match_families = _as_bool(self.options.get("match_families", True)) + self.match_nofamily = _as_bool(self.options.get("match_nofamily", True)) + self.best_match = _as_bool(self.options.get("best_match", False)) + self.max_results = int(self.options.get("max_results", 3) or 3) + self.timeout = int(self.options.get("timeout", 120) or 120) + excluded = self.options.get("exclude_families", "") or "" + self.exclude_families = [e.strip().lower() for e in excluded.split(",") if e.strip()] + + def accepts_format(self, file_format: str) -> bool: + return file_format in self.supported_formats + + def available(self) -> bool: + return True + + def analyze( + self, + file_path: str, + sha256: str, + filename: str, + source: str, + is_dump: bool = False, + base_addr: Optional[int] = None, + bitness: Optional[int] = None, + ) -> EngineResult: + raise NotImplementedError + + def _is_excluded_family(self, family: Optional[str]) -> bool: + if not family or not self.exclude_families: + return False + return any(token in family.lower() for token in self.exclude_families) + + def _select_matches(self, candidates: List[MatchRecord]) -> List[MatchRecord]: + """Apply shared family/threshold/best-match selection, then cap to max_results.""" + candidates = [c for c in candidates if not self._is_excluded_family(c.family)] + + kept: List[MatchRecord] = [] + for cand in candidates: + if cand.has_family and not self.match_families: + continue + if not cand.has_family and not self.match_nofamily: + continue + if cand.similarity >= self.minimum_similarity: + kept.append(cand) + + if kept: + kept.sort(key=lambda m: m.similarity, reverse=True) + return kept[: self.max_results] if self.max_results > 0 else kept + + # Nothing cleared the threshold: optionally surface the single best hit. + if self.best_match and candidates: + eligible = [ + c for c in candidates + if (c.has_family and self.match_families) or (not c.has_family and self.match_nofamily) + ] + if eligible: + best = max(eligible, key=lambda m: m.similarity) + best.is_low_confidence = True + return [best] + + return [] diff --git a/lib/cuckoo/common/integrations/similarity/mcrit_engine.py b/lib/cuckoo/common/integrations/similarity/mcrit_engine.py new file mode 100644 index 00000000000..d0e854aa566 --- /dev/null +++ b/lib/cuckoo/common/integrations/similarity/mcrit_engine.py @@ -0,0 +1,268 @@ +# Copyright (C) 2010-2015 Cuckoo Foundation, 2016-2026 CAPE developers. +# This file is part of CAPE Sandbox - http://www.capesandbox.com +# See the file 'docs/LICENSE' for copying permission. + +"""MCRIT similarity engine — direct REST API client. + +Uses only ``requests`` (already in CAPE). The ``mcrit`` and ``smda`` Python +packages are NOT required and must NOT be installed (smda conflicts with +CAPE's venv). MCRIT is PE-only, so this engine declares supported_formats +accordingly; non-PE artifacts are filtered out by the dispatcher. + +REST contract (MCRIT 1.4.x), all responses {"status":"successful","data":...}: + GET /version + POST /query/binary body=bytes params=minhash_score=N -> job_id + POST /query/binary/mapped/ body=bytes params=minhash_score=N -> job_id + GET /jobs/ -> job dict (.result set when done) + GET /results/ -> matching result dict + GET /samples/sha256/ -> sample entry or 404 + POST /samples/binary?... body=bytes -> job_id (persist mode) +""" + +import logging +import threading +import time +from typing import List + +from lib.cuckoo.common.integrations.similarity.base import ( + FMT_ELF, FMT_MACHO, FMT_PE, EngineResult, MatchRecord, SimilarityEngine, _as_bool, +) + +log = logging.getLogger(__name__) + + +class McritEngine(SimilarityEngine): + name = "mcrit" + # MCRIT disassembles via SMDA, which now supports PE, ELF and Mach-O + # (x86/x64/aarch64). Formats are config-driven via mcrit_formats so an + # operator on an older SMDA can narrow it; default covers all three. + supported_formats = {FMT_PE, FMT_ELF, FMT_MACHO} + + def __init__(self, options): + super().__init__(options) + # Allow operators to restrict/extend the formats sent to MCRIT. + fmts = (self.options.get("mcrit_formats") or "pe,elf,macho") + parsed = {f.strip().lower() for f in str(fmts).split(",") if f.strip()} + if parsed: + self.supported_formats = parsed + self.host = (self.options.get("mcrit_host") or "http://127.0.0.1:8000").rstrip("/") + self.apitoken = self.options.get("mcrit_apitoken") or None + self.username = self.options.get("mcrit_username") or "cape" + self.persist_samples = _as_bool(self.options.get("persist_samples", False)) + self.poll_interval = int(self.options.get("mcrit_poll_interval", 3) or 3) + self.minhash_threshold = int(self.minimum_similarity) if self.minimum_similarity else None + # When set, score matches by their NON-library weighted similarity so + # matches that are mostly shared library/runtime code (Go, OpenSSL, CRT, + # etc.) fall below the threshold and drop out — leaving malware-code + # matches. Pure-library matches are discarded outright. + self.ignore_library_matches = _as_bool(self.options.get("ignore_library_matches", True)) + # requests.Session is NOT thread-safe. With concurrent_submissions the + # ThreadPoolExecutor calls analyze() from multiple threads, so each + # thread gets its own Session via threading.local() (still keeping + # HTTP keep-alive per thread). _available records that the server + # answered a probe successfully during available(). + self._thread_local = threading.local() + self._available = False + + @property + def _session(self): + """Per-thread requests.Session (thread-safe, keep-alive preserved).""" + session = getattr(self._thread_local, "session", None) + if session is None: + import requests + session = requests.Session() + session.headers.update(self._auth_headers()) + self._thread_local.session = session + return session + + def available(self) -> bool: + try: + import requests # noqa: F401 (ensure dependency present) + except ImportError: + log.warning("MCRIT engine: 'requests' unavailable (unexpected in CAPE venv).") + return False + try: + resp = self._session.get(f"{self.host}/version", timeout=10) + if self._unwrap(resp) is None: + log.warning("MCRIT at %s gave an unexpected /version response.", self.host) + return False + self._available = True + return True + except Exception as err: + log.warning("MCRIT server unreachable at %s: %s", self.host, err) + return False + + def analyze(self, file_path, sha256, filename, source, + is_dump=False, base_addr=None, bitness=None) -> EngineResult: + if not self._available: + return EngineResult(status="disabled") + try: + with open(file_path, "rb") as fh: + binary = fh.read() + except OSError as err: + return EngineResult(status="error", error=f"unreadable: {err}") + if not binary: + return EngineResult(status="skipped", error="empty file") + + try: + if self.persist_samples: + result = self._persist_and_match(binary, sha256, filename, is_dump, base_addr, bitness) + else: + result = self._transient_match(binary, is_dump, base_addr) + except TimeoutError as err: + return EngineResult(status="timeout", error=str(err)) + except Exception as err: + log.exception("MCRIT analysis failed for %s", filename) + return EngineResult(status="error", error=str(err)) + + if result is None: + return EngineResult(status="error", error="no result returned from MCRIT") + + out = EngineResult(status="ok") + out.matches = self._select_matches(self._parse_matches(result)) + if not out.matches: + out.status = "no_match" + return out + + # -- match strategies ------------------------------------------------- + + def _transient_match(self, binary, is_dump, base_addr): + params = self._match_params() + if is_dump and base_addr is not None: + url = f"{self.host}/query/binary/mapped/{base_addr:#x}" + else: + url = f"{self.host}/query/binary" + job_id = self._unwrap(self._session.post(url, data=binary, params=params, timeout=30)) + if not job_id: + raise RuntimeError(f"MCRIT returned no job_id for {url}") + return self._await_result(job_id) + + def _persist_and_match(self, binary, sha256, filename, is_dump, base_addr, bitness): + sample_id = self._get_sample_id_by_sha256(sha256) + if sample_id is None: + job_id = self._submit_binary(binary, filename, is_dump, base_addr, bitness) + if job_id: + self._await_result(job_id) + sample_id = self._get_sample_id_by_sha256(sha256) + if sample_id is None: + raise RuntimeError("sample submitted but sha256 not found in MCRIT") + job_id = self._unwrap(self._session.get( + f"{self.host}/matches/sample/{sample_id}", params=self._match_params(), timeout=30)) + if not job_id: + raise RuntimeError("MCRIT returned no match job_id") + return self._await_result(job_id) + + # -- HTTP helpers ----------------------------------------------------- + + def _auth_headers(self): + h = {} + if self.apitoken: + h["apitoken"] = self.apitoken + if self.username: + h["username"] = self.username + return h + + def _match_params(self): + return {"minhash_score": self.minhash_threshold} if self.minhash_threshold is not None else {} + + @staticmethod + def _unwrap(response): + try: + if response.status_code not in (200, 202): + return None + j = response.json() + if isinstance(j, dict) and j.get("status") == "successful": + return j.get("data") + except Exception: + pass + return None + + def _get_sample_id_by_sha256(self, sha256): + try: + data = self._unwrap(self._session.get(f"{self.host}/samples/sha256/{sha256}", timeout=15)) + if isinstance(data, dict): + return data.get("sample_id") + except Exception: + pass + return None + + def _submit_binary(self, binary, filename, is_dump, base_addr, bitness): + fields = [f"filename={filename}"] + if is_dump: + fields.append("is_dump=1") + if base_addr is not None: + fields.append(f"base_addr={base_addr:#x}") + if bitness in (32, 64): + fields.append(f"bitness={bitness}") + qs = "?" + "&".join(fields) + return self._unwrap(self._session.post(f"{self.host}/samples/binary{qs}", data=binary, timeout=30)) + + # -- job polling (threaded deadline) ---------------------------------- + + def _await_result(self, job_id): + container = {} + + def _poll(): + try: + container["result"] = self._poll_job(job_id) + except Exception as err: + container["error"] = err + + t = threading.Thread(target=_poll, daemon=True) + t.start() + t.join(self.timeout) + if t.is_alive(): + raise TimeoutError(f"MCRIT job {job_id} exceeded {self.timeout}s deadline") + if "error" in container: + raise container["error"] + return container.get("result") + + def _poll_job(self, job_id): + while True: + job = self._unwrap(self._session.get(f"{self.host}/jobs/{job_id}", timeout=15)) + if not isinstance(job, dict): + raise RuntimeError(f"unexpected job response for {job_id}") + if job.get("terminated"): + raise RuntimeError(f"MCRIT job {job_id} terminated") + if job.get("attempts_left") == 0: + raise RuntimeError(f"MCRIT job {job_id} failed (attempts exhausted)") + result_id = job.get("result") + if result_id: + return self._unwrap(self._session.get(f"{self.host}/results/{result_id}", timeout=30)) + time.sleep(self.poll_interval) + + # -- result parsing --------------------------------------------------- + + def _parse_matches(self, result_dict) -> List[MatchRecord]: + records: List[MatchRecord] = [] + if not isinstance(result_dict, dict): + return records + for s in (result_dict.get("matches") or {}).get("samples") or []: + matched = s.get("matched") or {} + percent = matched.get("percent") or {} + functions = matched.get("functions") or {} + combined = functions.get("combined") or 0 + library = functions.get("library") or 0 + + if self.ignore_library_matches: + # Drop matches whose matched functions are entirely library code. + if combined > 0 and library >= combined: + continue + # Score on the non-library weighted similarity so library-heavy + # matches sink below the threshold. Fall back gracefully. + similarity = percent.get("nonlib_score_weighted") + if similarity is None: + similarity = percent.get("score_weighted") or percent.get("unweighted") or 0.0 + else: + similarity = percent.get("score_weighted") or percent.get("unweighted") or 0.0 + + records.append(MatchRecord( + family=s.get("family") or None, + similarity=float(similarity), + sample_sha256=s.get("sha256"), + sample_id=s.get("sample_id"), + matched_functions=combined, + total_functions=s.get("num_functions"), + version=s.get("version") or None, + )) + return records diff --git a/lib/cuckoo/common/integrations/similarity/registry.py b/lib/cuckoo/common/integrations/similarity/registry.py new file mode 100644 index 00000000000..d3c648f0a66 --- /dev/null +++ b/lib/cuckoo/common/integrations/similarity/registry.py @@ -0,0 +1,61 @@ +# Copyright (C) 2010-2015 Cuckoo Foundation, 2016-2026 CAPE developers. +# This file is part of CAPE Sandbox - http://www.capesandbox.com +# See the file 'docs/LICENSE' for copying permission. + +"""Engine registry and dispatcher. + +Engine modules are imported LAZILY inside get_enabled_engines() so that a +missing or broken optional dependency in any engine cannot crash CAPE's +plugin loader when the engine is disabled or its package is absent. + +Adding a new engine (e.g. binlex, Malcat, Intezer): + 1. Implement a SimilarityEngine subclass in _engine.py + (lazy-import any external deps inside available()). + 2. Add an entry to _ENGINE_MODULES below. + 3. Add " = yes/no" to integrations.conf [similarity] section. + Nothing else needs changing. Engines run independently, so you can have + one, several, or all enabled at once; matches merge per artifact and are + tagged with the engine name. +""" + +import importlib +import logging +from typing import Dict, List + +from lib.cuckoo.common.integrations.similarity.base import SimilarityEngine, _as_bool + +log = logging.getLogger(__name__) + +# Map of config key -> dotted "module.ClassName" path. +# Only engines listed here are ever instantiated. Additional engines (binlex, +# Malcat, Intezer, ...) slot in by adding a line here plus a config toggle — +# the dispatcher, result merging and UI need no changes. +_ENGINE_MODULES: Dict[str, str] = { + "mcrit": "lib.cuckoo.common.integrations.similarity.mcrit_engine.McritEngine", + # "binlex": "lib.cuckoo.common.integrations.similarity.binlex_engine.BinlexEngine", +} + + +def _load_engine_class(dotted: str): + module_path, class_name = dotted.rsplit(".", 1) + module = importlib.import_module(module_path) + return getattr(module, class_name) + + +def get_enabled_engines(options: Dict) -> List[SimilarityEngine]: + """Return initialized, available engines whose config key is 'yes'.""" + engines: List[SimilarityEngine] = [] + for engine_name, dotted_path in _ENGINE_MODULES.items(): + if not _as_bool(options.get(engine_name, False)): + continue + try: + cls = _load_engine_class(dotted_path) + engine = cls(options) + except Exception as err: + log.warning("Similarity engine '%s' failed to load: %s", engine_name, err) + continue + if not engine.available(): + log.warning("Similarity engine '%s' is enabled but unavailable; skipping.", engine_name) + continue + engines.append(engine) + return engines diff --git a/modules/processing/similarity.py b/modules/processing/similarity.py new file mode 100644 index 00000000000..3aeadd91e44 --- /dev/null +++ b/modules/processing/similarity.py @@ -0,0 +1,334 @@ +# Copyright (C) 2010-2015 Cuckoo Foundation, 2016-2026 CAPE developers. +# This file is part of CAPE Sandbox - http://www.capesandbox.com +# See the file 'docs/LICENSE' for copying permission. + +"""Code-similarity processing module. + +Runs after CAPE extraction + YARA detection (order=20). Collects artifacts +(submitted file, CAPE payloads, process dumps, dropped files) that did not +already resolve to a known family, filters them to the formats each engine +supports (per engine; MCRIT does PE/ELF/Mach-O), submits them, and +writes results to results["similarity"]. + +Output shape: + results["similarity"] = { + "enabled_engines": ["mcrit"], + "results": {"mcrit": [ {file, sha256, source, status, matches:[...]}, ... ]}, + "by_sha256": {"": [ {engine, ...match...}, ... ]}, # for per-file UI + } + +Optionally promotes a high-confidence top family match into CAPE's own +detections list so similarity acts as a classification engine. +""" + +import logging +import os +from concurrent.futures import ThreadPoolExecutor + +from lib.cuckoo.common.abstracts import Processing +from lib.cuckoo.common.config import Config +from lib.cuckoo.common.integrations.similarity.base import detect_format +from lib.cuckoo.common.integrations.similarity.registry import get_enabled_engines + +log = logging.getLogger(__name__) + + +class Similarity(Processing): + """Submit unidentified artifacts to code-similarity engines.""" + + order = 20 + + def run(self): + self.key = "similarity" + + try: + options = dict(Config("integrations").get("similarity") or {}) + except Exception as err: + log.warning("Could not read [similarity] from integrations.conf: %s", err) + return {} + + engines = get_enabled_engines(options) + if not engines: + return {} + + only_unidentified = _as_bool(options.get("only_unidentified", True)) + sources_enabled = { + "payload": _as_bool(options.get("similarity_payloads", True)), + "procdump": _as_bool(options.get("similarity_procdump", True)), + "dropped": _as_bool(options.get("similarity_dropped", True)), + "submitted": _as_bool(options.get("similarity_submittedfile", False)), + } + concurrent = _as_bool(options.get("concurrent_submissions", False)) + max_workers = int(options.get("max_workers", 4) or 4) + + update_malfamily = _as_bool(options.get("update_malfamily", False)) + malfamily_min_sim = float(options.get("malfamily_minimum_similarity", 80) or 80) + malfamily_mode = (options.get("malfamily_mode", "top") or "top").strip().lower() + if malfamily_mode not in ("top", "all"): + malfamily_mode = "top" + + candidates = [] + if sources_enabled["payload"]: + candidates.extend(self._gather_payloads(only_unidentified)) + if sources_enabled["procdump"]: + candidates.extend(self._gather_procdumps()) + if sources_enabled["dropped"]: + candidates.extend(self._gather_dropped()) + if sources_enabled["submitted"]: + candidates.extend(self._gather_submitted(only_unidentified)) + + # Deduplicate by sha256, keep only files that still exist. + seen, unique = set(), [] + for cand in candidates: + if cand["sha256"] not in seen and os.path.exists(cand["path"]): + seen.add(cand["sha256"]) + unique.append(cand) + + if not unique: + log.info("Similarity: no candidates to submit (only_unidentified=%s). " + "Set only_unidentified=no to include already-identified artifacts.", + only_unidentified) + return {} + + # Build (engine, idx, candidate) tasks, skipping format mismatches so + # e.g. an ELF is never sent to MCRIT (PE-only). + tasks = [] + for engine in engines: + for idx, cand in enumerate(unique): + if engine.accepts_format(cand["format"]): + tasks.append((engine, idx, cand)) + else: + log.debug("Similarity: %s skips %s (format=%s not supported)", + engine.name, cand["filename"], cand["format"]) + + if not tasks: + log.info("Similarity: no artifacts match the enabled engines' supported formats.") + return {} + + log.info("Similarity: submitting %d task(s) to engine(s): %s", + len(tasks), ", ".join(e.name for e in engines)) + + rows_by_engine = {engine.name: [None] * len(unique) for engine in engines} + + if concurrent and len(tasks) > 1 and max_workers > 1: + workers = min(max_workers, len(tasks)) + with ThreadPoolExecutor(max_workers=workers, thread_name_prefix="similarity") as pool: + futures = [pool.submit(self._analyze_one, eng, cand) for eng, _i, cand in tasks] + for (eng, idx, _c), future in zip(tasks, futures): + rows_by_engine[eng.name][idx] = future.result() + else: + for eng, idx, cand in tasks: + rows_by_engine[eng.name][idx] = self._analyze_one(eng, cand) + + output = {"enabled_engines": [e.name for e in engines], "results": {}, "by_sha256": {}} + for engine_name, rows in rows_by_engine.items(): + kept = [r for r in rows if r is not None] + if kept: + output["results"][engine_name] = kept + + if not output["results"]: + return {} + + # Per-file index for the web UI: sha256 -> list of match dicts (with engine tag). + for engine_name, rows in output["results"].items(): + for row in rows: + if not row.get("matches"): + continue + bucket = output["by_sha256"].setdefault(row["sha256"], []) + for match in row["matches"]: + entry = dict(match) + entry["engine"] = engine_name + bucket.append(entry) + + # Annotate each source artifact dict in-place with its matches. Because + # _file_info.html renders the payload/dropped/procdump/target dict + # directly (as its ``file`` variable), this makes matches appear in + # every view — the overview page and the AJAX-loaded Payloads/Dropped/ + # Procdump tabs — without any change to web/analysis/views.py. + for cand in unique: + matches = output["by_sha256"].get(cand["sha256"]) + if matches and isinstance(cand.get("artifact"), dict): + cand["artifact"]["similarity_matches"] = matches + + if update_malfamily: + self._apply_family_classifications(output, malfamily_min_sim, malfamily_mode) + + return output + + # -- per-task analysis ------------------------------------------------ + + def _analyze_one(self, engine, cand): + try: + result = engine.analyze( + file_path=cand["path"], sha256=cand["sha256"], + filename=cand["filename"], source=cand["source"], + is_dump=cand["is_dump"], base_addr=cand["base_addr"], + bitness=cand.get("bitness"), + ) + except Exception as err: + log.exception("Similarity engine '%s' crashed on %s", engine.name, cand["filename"]) + return {"file": cand["filename"], "sha256": cand["sha256"], "source": cand["source"], + "status": "error", "error": str(err), "matches": []} + row = {"file": cand["filename"], "sha256": cand["sha256"], "source": cand["source"]} + row.update(result.to_dict()) + # Keep every real submission attempt visible; drop only skipped. + return row if result.status != "skipped" else None + + # -- candidate collection --------------------------------------------- + + def _gather_payloads(self, only_unidentified): + rows = [] + cape = self.results.get("CAPE", {}) + payloads = cape.get("payloads", []) if isinstance(cape, dict) else (cape if isinstance(cape, list) else []) + for p in payloads: + if not isinstance(p, dict): + continue + if not p.get("path") or not p.get("sha256"): + continue + if only_unidentified and self._payload_identified(p): + continue + rows.append(self._mk(p, p["path"], "payload", is_dump=False)) + return rows + + def _gather_procdumps(self): + rows = [] + for d in (self.results.get("procdump") or []): + if isinstance(d, dict) and d.get("path") and d.get("sha256"): + rows.append(self._mk(d, d["path"], "procdump", is_dump=True, base_addr=self._dump_base(d))) + return rows + + def _gather_dropped(self): + rows = [] + for d in (self.results.get("dropped") or []): + if isinstance(d, dict) and d.get("path") and d.get("sha256"): + rows.append(self._mk(d, d["path"], "dropped", is_dump=False)) + return rows + + def _gather_submitted(self, only_unidentified): + target = self.results.get("target", {}) + f = target.get("file") if isinstance(target, dict) else None + if not isinstance(f, dict) or not f.get("path") or not f.get("sha256"): + return [] + if only_unidentified and self._analysis_has_family(): + return [] + return [self._mk(f, f["path"], "submitted", is_dump=False)] + + def _mk(self, info, path, source, is_dump=False, base_addr=None): + """Build a candidate dict, deriving its format from CAPE's 'type' field. + + ``artifact`` keeps a reference to the original report dict (payload / + dropped / procdump / target file) so matches can be annotated back onto + it for per-file rendering in the web UI. + """ + return { + "path": path, + "sha256": info["sha256"], + "filename": info.get("name") or os.path.basename(path), + "source": source, + "is_dump": is_dump, + "base_addr": base_addr, + "format": detect_format(info.get("type")), + "artifact": info, + } + + # -- family classification -------------------------------------------- + + def _apply_family_classifications(self, output, min_sim, mode="top"): + """Promote family hits (>= min_sim) into results["detections"]. + + mode="top": only the single highest-similarity family per artifact is + promoted (conservative; default). + mode="all": every family whose match clears min_sim is promoted. + + Only artifacts with no existing family detection are considered, and + each unique family is added at most once per analysis so the same + family is never appended repeatedly across multiple matching payloads. + """ + try: + from lib.cuckoo.common.utils import add_family_detection + except ImportError: + log.warning("Similarity: add_family_detection unavailable; update_malfamily skipped.") + return + + already_present = self._existing_detection_families() + added = set() + mode = (mode or "top").strip().lower() + + # Collect candidate (family, similarity, engine, sha256) tuples. + promotions = [] # list of dicts + per_artifact_best = {} # sha256 -> best tuple + + for engine_name, rows in output.get("results", {}).items(): + for row in rows: + sha256 = row.get("sha256", "") + if not row.get("matches") or self._sha_already_identified(sha256): + continue + for m in row["matches"]: + fam, sim = m.get("family"), float(m.get("similarity") or 0) + if not fam or sim < min_sim: + continue + entry = {"family": fam, "similarity": sim, "engine": engine_name, "sha256": sha256} + if mode == "all": + promotions.append(entry) + cur = per_artifact_best.get(sha256) + if cur is None or sim > cur["similarity"]: + per_artifact_best[sha256] = entry + + if mode != "all": + promotions = list(per_artifact_best.values()) + + # Highest-similarity first so the dedup keeps the strongest evidence. + promotions.sort(key=lambda e: e["similarity"], reverse=True) + + for e in promotions: + key = e["family"].lower() + if key in already_present or key in added: + continue # never double-add the same family + log.info("Similarity: classifying %s as '%s' (%.1f%%, via %s)", + e["sha256"][:12], e["family"], e["similarity"], e["engine"]) + add_family_detection(self.results, e["family"], f"Similarity/{e['engine']}", e["sha256"]) + added.add(key) + + def _existing_detection_families(self): + return {(b.get("family") or "").lower() for b in (self.results.get("detections") or [])} + + def _sha_already_identified(self, sha256): + for block in (self.results.get("detections") or []): + for detail in (block.get("details") or []): + if sha256 in detail.values(): + return True + return False + + # -- "already identified?" helpers ------------------------------------ + + def _payload_identified(self, payload): + if payload.get("cape_yara") or payload.get("detections") or payload.get("detection"): + return True + cape_type = (payload.get("cape_type") or "").lower() + return bool(cape_type and "unknown" not in cape_type and "extraction" not in cape_type) + + def _analysis_has_family(self): + return bool(self.results.get("malfamily") or self.results.get("malfamily_tag") or self.results.get("detections")) + + @staticmethod + def _dump_base(dump): + space = dump.get("address_space") + if isinstance(space, list) and space: + start = space[0].get("start") + try: + return int(start, 16) if isinstance(start, str) else int(start) + except (TypeError, ValueError): + pass + base = dump.get("imagebase") or dump.get("base") + try: + return int(base, 16) if isinstance(base, str) else int(base) + except (TypeError, ValueError): + return None + + +def _as_bool(value): + if isinstance(value, bool): + return value + if value is None: + return False + return str(value).strip().lower() in ("1", "yes", "true", "on", "y") diff --git a/utils/similarity_diag.py b/utils/similarity_diag.py new file mode 100644 index 00000000000..7a760758e41 --- /dev/null +++ b/utils/similarity_diag.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +# Copyright (C) 2010-2015 Cuckoo Foundation, 2016-2026 CAPE developers. +# This file is part of CAPE Sandbox - http://www.capesandbox.com +# See the file 'docs/LICENSE' for copying permission. + +"""Diagnostic for the code-similarity integration. + +Run from the CAPE root inside CAPE's venv: + + python3 utils/similarity_diag.py + python3 utils/similarity_diag.py /path/to/a/payload.bin + +No argument: checks config + engine availability. +With a file: detects its format, checks which engines accept it, submits +it, and prints the matches — confirming the full path end-to-end. +""" + +import hashlib +import os +import sys + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) +sys.path.insert(0, os.getcwd()) + + +def main(): + print("=" * 60) + print("CAPE code-similarity diagnostic") + print("=" * 60) + + try: + from lib.cuckoo.common.config import Config + options = dict(Config("integrations").get("similarity") or {}) + except Exception as err: + print(f"[FAIL] Could not read [similarity] from integrations.conf: {err}") + return + if not options: + print("[FAIL] [similarity] section empty or missing.") + return + print("[ OK ] [similarity] config read. Effective values:") + for key in sorted(options): + print(f" {key} = {options[key]!r}") + + try: + proc = dict(Config("processing").get("similarity") or {}) + enabled = str(proc.get("enabled")).lower() in ("1", "true", "yes", "on") + print(f"[{'OK' if enabled else 'WARN'}] processing.conf [similarity] enabled = {proc.get('enabled')}") + except Exception as err: + print(f"[WARN] processing.conf has no [similarity] section: {err}") + + from lib.cuckoo.common.integrations.similarity.registry import get_enabled_engines + engines = get_enabled_engines(options) + if not engines: + print("[FAIL] No engines available. Either none enabled, or the server") + print(f" did not answer GET {options.get('mcrit_host')}/version .") + print(" Test: curl /version") + return + for e in engines: + print(f"[ OK ] Engine '{e.name}' available. Supported formats: {sorted(e.supported_formats)}") + + if len(sys.argv) <= 1: + print("\nPass a file path to submit it and see matches:") + print(" python3 utils/similarity_diag.py /path/to/payload.bin") + return + + path = sys.argv[1] + if not os.path.exists(path): + print(f"[FAIL] File not found: {path}") + return + + from lib.cuckoo.common.integrations.similarity.base import detect_format + with open(path, "rb") as fh: + data = fh.read() + sha256 = hashlib.sha256(data).hexdigest() + fmt = detect_format(None, magic=data[:8]) + print(f"\nSubmitting {path}") + print(f" size={len(data)} sha256={sha256[:16]} detected_format={fmt}") + + for engine in engines: + print(f"\n--- engine: {engine.name} ---") + if not engine.accepts_format(fmt): + print(f" SKIP: {engine.name} does not support format '{fmt}' " + f"(supports {sorted(engine.supported_formats)})") + continue + result = engine.analyze(file_path=path, sha256=sha256, + filename=os.path.basename(path), source="manual") + print(f" status: {result.status}" + (f" error: {result.error}" if result.error else "")) + for m in result.matches: + print(f" MATCH family={m.family or '(none)':20} " + f"sim={m.similarity:5.1f}% sha256={(m.sample_sha256 or '')[:16]}") + if not result.matches and result.status == "no_match": + print(" (no matches — empty corpus, below threshold, or genuinely dissimilar)") + + print("\nDone.") + + +if __name__ == "__main__": + main() diff --git a/web/templates/analysis/generic/_file_info.html b/web/templates/analysis/generic/_file_info.html index a4c4ff47566..5ec5dce4385 100644 --- a/web/templates/analysis/generic/_file_info.html +++ b/web/templates/analysis/generic/_file_info.html @@ -232,6 +232,32 @@
File {% endif %} +{# similarity-inline-rows BEGIN #} +{% if file.similarity_matches %} + + Code Similarity + +
    + {% for m in file.similarity_matches %} +
  • + {% if m.family %} + {{ m.family }} + {% if m.version %}{{ m.version }}{% endif %} + {% else %} + unlabelled + {% endif %} + {{ m.similarity }}% + {% if m.low_confidence %}low{% endif %} + {% if m.sample_sha256 %}{{ m.sample_sha256 }}{% endif %} + {{ m.engine }} +
    • + {% endfor %} +
+ + +{% endif %} +{# similarity-inline-rows END #} + {% if file.trid %} TriD @@ -461,4 +487,3 @@
File {% endif %} -