[pull] master from kevoreilly:master

conf/default/integrations.conf.default

@@ -148,3 +148,93 @@ tight_strings = yes
 min_length = 5
 # Download FLOSS signatures from https://github.com/mandiant/flare-floss/tree/master/sigs
 sigs_path = data/flare-signatures
+
+# Code-similarity engines
+# Enable too in processing.conf. 
+# We recommend ensuring local similairty engines such as mcrit are populated with unpacked samples ideally aquired from CAPE payloads/procdumps or are not heavily packed/obfuscated to ensure best matching.
+# Also for accurate matching populate with libraries (i.e. import from https://github.com/danielplohmann/mcrit-data dragging and dropping all the files for each into import, 
+# or importing binaries and then editing them and marking them as libraries once imported).
+[similarity]
+
+# Hard per-job deadline (seconds) so a stuck engine can't stall processing.
+timeout = 120
+
+# ---- Engines ------------------------------------------------
+# Only listed engines are attempted. Each engine only receives files in the
+# formats it supports, so enabling more engines is safe. Currently only MCRIT is supported (https://github.com/danielplohmann/mcrit) 
+# although the integration allows adding other performative similairty engines later.
+mcrit = yes
+# Formats sent to MCRIT. MCRIT's SMDA disassembler supports PE, ELF and
+# Mach-O (x86/x64/aarch64). Narrow
+# this only if your MCRIT server runs an older SMDA: e.g.  mcrit_formats = pe.
+# As CAPE does not support OSX but only Windows and Linux macho is ommited.
+mcrit_formats = pe,elf
+
+# File sources to submit (default is memory extracted files for unpacked files).
+similarity_payloads = yes
+similarity_procdump = yes
+similarity_dropped = no
+similarity_submittedfile = no
+
+# Skip artifacts already resolved to a known family. Set no to
+# submit everything (corpus building / re-checking known files).
+only_unidentified = yes
+
+# ---- Result selection (what is SHOWN) -----------------------
+# Minimum similarity % for a match to appear in the report.
+minimum_similarity = 50
+# Cap the number of matches returned/shown per artifact (top N by
+# score). Tunable limit on samples returned by each engine.
+max_results = 3
+# Include matches that carry a family label.
+match_families = yes
+# Include matches with no family label (shows sha256 + score).
+match_nofamily = yes
+# If nothing clears minimum_similarity, still show the single best
+# hit flagged low-confidence. Off by default.
+best_match = no
+# Comma-separated family substrings to suppress (case-insensitive).
+# exclude_families = msvc,mingw
+
+# Ignore matches that are mostly shared library / runtime code (Go runtime,
+# OpenSSL, the C runtime, etc.) by scoring on MCRIT's non-library-weighted
+# similarity and discarding pure-library matches. This is the robust,
+# list-free way to keep only malware-code matches and is preferred over
+# maintaining a long exclude_families list. Relies on library code in your
+# MCRIT corpus being flagged as library samples (is_library) and/or MCRIT's
+# automatic library-function detection.
+ignore_library_matches = yes
+
+# ---- Family classification (CAPE malfamily detections)
+# When yes, a family match is promoted into CAPE's detections list (main
+# page + report header), acting as a classification engine. Only for
+# artifacts with no existing family; each family added at most once per run.
+update_malfamily = no
+# SEPARATE, higher threshold for promotion. Lets low-confidence matches
+# still appear in the similarity table (minimum_similarity) while only
+# strong matches set the malware family.
+malfamily_minimum_similarity = 80
+# Which matches become the malware family when update_malfamily = yes:
+#   top - only the single highest-similarity family per artifact (default)
+#   all - every family whose match clears malfamily_minimum_similarity
+# Either way each family is added at most once per analysis.
+malfamily_mode = top
+
+# Concurrent submission
+# Submit multiple artifacts simultaneously. max_workers bounds simultaneous jobs.
+concurrent_submissions = no
+max_workers = 4
+
+# ---- MCRIT connection ---------------------------------------
+# REST API endpoint - NOT the web UI (5000), the API (8000).
+mcrit_host = http://127.0.0.1:8000
+mcrit_apitoken =
+mcrit_username = cape
+mcrit_poll_interval = 3
+
+# Persistence:
+#   no  (default) - transient /query: matched on the worker, never
+#       stored. Nothing to delete; labelled samples never touched.
+#   yes - stored via /samples/binary for corpus building. A sha256
+#       pre-check reuses an existing sample instead of re-adding.
+persist_samples = no

conf/default/processing.conf.default

@@ -357,3 +357,7 @@ enabled = no
 # Enable when using the PolarProxy option during analysis. This will merge the tls.pcap containing
 # plain-text TLS streams into the task PCAP.
 enabled = no
+
+# Code-similarity engine(s). See conf/integrations.conf [similarity].
+[similarity]
+enabled = no

lib/cuckoo/common/integrations/similarity/base.py

@@ -0,0 +1,189 @@
+# Copyright (C) 2010-2015 Cuckoo Foundation, 2016-2026 CAPE developers.
+# This file is part of CAPE Sandbox - http://www.capesandbox.com
+# See the file 'docs/LICENSE' for copying permission.
+
+"""Code-similarity engine framework — base abstractions.
+
+Zero external dependencies (stdlib + whatever CAPE already ships). New
+engines subclass SimilarityEngine, declare the file formats they accept,
+implement available()/analyze(), and register in registry.py.
+"""
+
+import logging
+from typing import Dict, List, Optional, Set
+
+log = logging.getLogger(__name__)
+
+# Canonical file-format tokens used across the framework.
+FMT_PE = "pe"
+FMT_ELF = "elf"
+FMT_MACHO = "macho"
+FMT_UNKNOWN = "unknown"
+
+
+def _as_bool(value) -> bool:
+    if isinstance(value, bool):
+        return value
+    if value is None:
+        return False
+    return str(value).strip().lower() in ("1", "yes", "true", "on", "y")
+
+
+def detect_format(type_string: Optional[str], magic: Optional[bytes] = None) -> str:
+    """Map CAPE's libmagic 'type' string (or raw magic) to a format token.
+
+    Prefers CAPE's reported type (already computed during processing);
+    falls back to magic-byte sniffing when the type string is absent.
+    """
+    t = (type_string or "").lower()
+    if "pe32" in t or "ms-dos" in t or "ms windows" in t or "for ms windows" in t:
+        return FMT_PE
+    if "elf" in t:
+        return FMT_ELF
+    if "mach-o" in t or "mach o" in t:
+        return FMT_MACHO
+    if magic:
+        if magic[:2] == b"MZ":
+            return FMT_PE
+        if magic[:4] == b"\x7fELF":
+            return FMT_ELF
+        if magic[:4] in (b"\xfe\xed\xfa\xce", b"\xfe\xed\xfa\xcf",
+                         b"\xce\xfa\xed\xfe", b"\xcf\xfa\xed\xfe", b"\xca\xfe\xba\xbe"):
+            return FMT_MACHO
+    return FMT_UNKNOWN
+
+
+class MatchRecord:
+    """Single normalized similarity hit, engine-agnostic."""
+
+    def __init__(
+        self,
+        family: Optional[str],
+        similarity: float,
+        sample_sha256: Optional[str] = None,
+        sample_id=None,
+        matched_functions: Optional[int] = None,
+        total_functions: Optional[int] = None,
+        version: Optional[str] = None,
+        is_low_confidence: bool = False,
+    ):
+        self.family = family or None
+        self.similarity = round(float(similarity), 2)
+        self.sample_sha256 = sample_sha256
+        self.sample_id = sample_id
+        self.matched_functions = matched_functions
+        self.total_functions = total_functions
+        self.version = version
+        self.is_low_confidence = is_low_confidence
+
+    @property
+    def has_family(self) -> bool:
+        return bool(self.family)
+
+    def to_dict(self) -> Dict:
+        return {
+            "family": self.family,
+            "similarity": self.similarity,
+            "sample_sha256": self.sample_sha256,
+            "sample_id": self.sample_id,
+            "matched_functions": self.matched_functions,
+            "total_functions": self.total_functions,
+            "version": self.version,
+            "low_confidence": self.is_low_confidence,
+        }
+
+
+class EngineResult:
+    """Per-file result from one engine for one artifact."""
+
+    def __init__(self, status: str = "ok", error: Optional[str] = None):
+        # status: ok | no_match | skipped | timeout | error | disabled
+        self.status = status
+        self.error = error
+        self.matches: List[MatchRecord] = []
+
+    def to_dict(self) -> Dict:
+        out = {"status": self.status, "matches": [m.to_dict() for m in self.matches]}
+        if self.error:
+            out["error"] = self.error
+        return out
+
+
+class SimilarityEngine:
+    """Base class for a code-similarity backend.
+
+    Subclasses set ``supported_formats`` to the set of file formats they
+    can analyze, and implement available()/analyze(). External packages
+    must only be imported lazily (inside available()) so a missing optional
+    dependency cannot crash CAPE's plugin loader.
+    """
+
+    name = "base"
+    # Formats this engine accepts. Override per engine.
+    #   MCRIT -> {FMT_PE}            (PE only)
+    #   a multi-format engine -> {FMT_PE, FMT_ELF, FMT_MACHO}
+    supported_formats: Set[str] = {FMT_PE}
+
+    def __init__(self, options: Dict):
+        self.options = options or {}
+        self.minimum_similarity = float(self.options.get("minimum_similarity", 50) or 0)
+        self.match_families = _as_bool(self.options.get("match_families", True))
+        self.match_nofamily = _as_bool(self.options.get("match_nofamily", True))
+        self.best_match = _as_bool(self.options.get("best_match", False))
+        self.max_results = int(self.options.get("max_results", 3) or 3)
+        self.timeout = int(self.options.get("timeout", 120) or 120)
+        excluded = self.options.get("exclude_families", "") or ""
+        self.exclude_families = [e.strip().lower() for e in excluded.split(",") if e.strip()]
+
+    def accepts_format(self, file_format: str) -> bool:
+        return file_format in self.supported_formats
+
+    def available(self) -> bool:
+        return True
+
+    def analyze(
+        self,
+        file_path: str,
+        sha256: str,
+        filename: str,
+        source: str,
+        is_dump: bool = False,
+        base_addr: Optional[int] = None,
+        bitness: Optional[int] = None,
+    ) -> EngineResult:
+        raise NotImplementedError
+
+    def _is_excluded_family(self, family: Optional[str]) -> bool:
+        if not family or not self.exclude_families:
+            return False
+        return any(token in family.lower() for token in self.exclude_families)
+
+    def _select_matches(self, candidates: List[MatchRecord]) -> List[MatchRecord]:
+        """Apply shared family/threshold/best-match selection, then cap to max_results."""
+        candidates = [c for c in candidates if not self._is_excluded_family(c.family)]
+
+        kept: List[MatchRecord] = []
+        for cand in candidates:
+            if cand.has_family and not self.match_families:
+                continue
+            if not cand.has_family and not self.match_nofamily:
+                continue
+            if cand.similarity >= self.minimum_similarity:
+                kept.append(cand)
+
+        if kept:
+            kept.sort(key=lambda m: m.similarity, reverse=True)
+            return kept[: self.max_results] if self.max_results > 0 else kept
+
+        # Nothing cleared the threshold: optionally surface the single best hit.
+        if self.best_match and candidates:
+            eligible = [
+                c for c in candidates
+                if (c.has_family and self.match_families) or (not c.has_family and self.match_nofamily)
+            ]
+            if eligible:
+                best = max(eligible, key=lambda m: m.similarity)
+                best.is_low_confidence = True
+                return [best]
+
+        return []