Merge pull request #1596 from mandiant/sync-master

Sync master
2025-12-12 15:49:46 -08:00 · 2023-07-10 10:30:23 +02:00
parent e140fba5df b6580f99db
commit d29e7140b6
16 changed files with 98 additions and 51 deletions
--- a/.github/workflows/publish.yml
+++ b/.github/workflows/publish.yml
@@ -15,7 +15,7 @@ jobs:
      - name: Set up Python
        uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435 # v4.5.0
        with:
-          python-version: '3.7'
+          python-version: '3.8'
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -69,7 +69,7 @@ jobs:
      matrix:
        os: [ubuntu-20.04, windows-2019, macos-11]
        # across all operating systems
-        python-version: ["3.7", "3.11"]
+        python-version: ["3.8", "3.11"]
        include:
          # on Ubuntu run these as well
          - os: ubuntu-20.04
@@ -104,7 +104,7 @@ jobs:
    strategy:
      fail-fast: false
      matrix:
-        python-version: ["3.7", "3.11"]
+        python-version: ["3.8", "3.11"]
    steps:
    - name: Checkout capa with submodules
      # do only run if BN_SERIAL is available, have to do this in every step, see https://github.com/orgs/community/discussions/26726#discussioncomment-3253118
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -13,9 +13,11 @@

 ### Breaking Changes
 - Update Metadata type in capa main [#1411](https://github.com/mandiant/capa/issues/1411) [@Aayush-Goel-04](https://github.com/aayush-goel-04) @manasghandat
+- Python 3.8 is now the minimum supported Python version #1578 @williballenthin
 - Change the old FeatureExtractor class' name into StaticFeatureExtractor, and make the former an alias for both the StaticFeatureExtractor and DynamicFeatureExtractor classes @yelhamer [#1567](https://github.com/mandiant/capa/issues/1567)
+- use fancy box drawing characters for default output #1586 @williballenthin

-### New Rules (11)
+### New Rules (22)

 - load-code/shellcode/execute-shellcode-via-windows-callback-function ervin.ocampo@mandiant.com jakub.jozwiak@mandiant.com
 - nursery/execute-shellcode-via-indirect-call ronnie.salomonsen@mandiant.com
@@ -28,6 +30,16 @@
 - host-interaction/hardware/enumerate-devices-by-category @mr-tz
 - host-interaction/service/continue-service @mr-tz
 - host-interaction/service/pause-service @mr-tz
+- persistence/exchange/act-as-exchange-transport-agent jakub.jozwiak@mandiant.com
+- host-interaction/file-system/create-virtual-file-system-in-dotnet jakub.jozwiak@mandiant.com
+- compiler/cx_freeze/compiled-with-cx_freeze @mr-tz jakub.jozwiak@mandiant.com
+- communication/socket/create-vmci-socket jakub.jozwiak@mandiant.com
+- persistence/office/act-as-excel-xll-add-in jakub.jozwiak@mandiant.com
+- persistence/office/act-as-office-com-add-in jakub.jozwiak@mandiant.com
+- persistence/office/act-as-word-wll-add-in jakub.jozwiak@mandiant.com
+- anti-analysis/anti-debugging/debugger-evasion/hide-thread-from-debugger michael.hunhoff@mandiant.com jakub.jozwiak@mandiant.com
+- host-interaction/memory/create-new-application-domain-in-dotnet jakub.jozwiak@mandiant.com
+- host-interaction/gui/switch-active-desktop jakub.jozwiak@mandiant.com
 -

 ### Bug Fixes
@@ -43,10 +55,13 @@
 - Add logging and print redirect to tqdm for capa main [#749](https://github.com/mandiant/capa/issues/749) [@Aayush-Goel-04](https://github.com/aayush-goel-04)
 - extractor: fix binja installation path detection does not work with Python 3.11
 - tests: refine the IDA test runner script #1513 @williballenthin
+- output: don't leave behind traces of progress bar @williballenthin
+- import-to-ida: fix bug introduced with JSON report changes in v5 #1584 @williballenthin

 ### capa explorer IDA Pro plugin

 ### Development
+- update ATT&CK/MBC data for linting #1568 @mr-tz

 ### Raw diffs
 - [capa v5.1.0...master](https://github.com/mandiant/capa/compare/v5.1.0...master)
--- a/README.md
+++ b/README.md
@@ -2,7 +2,7 @@

 [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/flare-capa)](https://pypi.org/project/flare-capa)
 [![Last release](https://img.shields.io/github/v/release/mandiant/capa)](https://github.com/mandiant/capa/releases)
-[![Number of rules](https://img.shields.io/badge/rules-802-blue.svg)](https://github.com/mandiant/capa-rules)
+[![Number of rules](https://img.shields.io/badge/rules-810-blue.svg)](https://github.com/mandiant/capa-rules)
 [![CI status](https://github.com/mandiant/capa/workflows/CI/badge.svg)](https://github.com/mandiant/capa/actions?query=workflow%3ACI+event%3Apush+branch%3Amaster)
 [![Downloads](https://img.shields.io/github/downloads/mandiant/capa/total)](https://github.com/mandiant/capa/releases)
 [![License](https://img.shields.io/badge/license-Apache--2.0-green.svg)](LICENSE.txt)
--- a/capa/helpers.py
+++ b/capa/helpers.py
@@ -168,7 +168,7 @@ def log_unsupported_runtime_error():
    logger.error("-" * 80)
    logger.error(" Unsupported runtime or Python interpreter.")
    logger.error(" ")
-    logger.error(" capa supports running under Python 3.7 and higher.")
+    logger.error(" capa supports running under Python 3.8 and higher.")
    logger.error(" ")
    logger.error(
        " If you're seeing this message on the command line, please ensure you're running a supported Python version."
--- a/capa/ida/plugin/README.md
+++ b/capa/ida/plugin/README.md
@@ -95,7 +95,7 @@ can update using the `Settings` button.

 ### Requirements

-capa explorer supports Python versions >= 3.7.x and IDA Pro versions >= 7.4. The following IDA Pro versions have been tested:
+capa explorer supports Python versions >= 3.8.x and IDA Pro versions >= 7.4. The following IDA Pro versions have been tested:

 * IDA 7.4
 * IDA 7.5
@@ -105,7 +105,7 @@ capa explorer supports Python versions >= 3.7.x and IDA Pro versions >= 7.4. The
 * IDA 8.1
 * IDA 8.2

-capa explorer is however limited to the Python versions supported by your IDA installation (which may not include all Python versions >= 3.7.x).
+capa explorer is however limited to the Python versions supported by your IDA installation (which may not include all Python versions >= 3.8.x).

 If you encounter issues with your specific setup, please open a new [Issue](https://github.com/mandiant/capa/issues).

--- a/capa/main.py
+++ b/capa/main.py
@@ -8,6 +8,7 @@ Unless required by applicable law or agreed to in writing, software distributed
 is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and limitations under the License.
 """
+import io
 import os
 import sys
 import json
@@ -274,7 +275,7 @@ def find_static_capabilities(
            functions = list(extractor.get_functions())
            n_funcs = len(functions)

-            pb = pbar(functions, desc="matching", unit=" functions", postfix="skipped 0 library functions")
+            pb = pbar(functions, desc="matching", unit=" functions", postfix="skipped 0 library functions", leave=False)
            for f in pb:
                if extractor.is_library_function(f.address):
                    function_name = extractor.get_function_name(f.address)
@@ -1028,12 +1029,20 @@ def handle_common_args(args):
    # disable vivisect-related logging, it's verbose and not relevant for capa users
    set_vivisect_log_level(logging.CRITICAL)

-    # Since Python 3.8 cp65001 is an alias to utf_8, but not for Python < 3.8
-    # TODO: remove this code when only supporting Python 3.8+
-    # https://stackoverflow.com/a/3259271/87207
-    import codecs
-
-    codecs.register(lambda name: codecs.lookup("utf-8") if name == "cp65001" else None)
+    if isinstance(sys.stdout, io.TextIOWrapper) or hasattr(sys.stdout, "reconfigure"):
+        # from sys.stdout type hint:
+        #
+        # TextIO is used instead of more specific types for the standard streams,
+        # since they are often monkeypatched at runtime. At startup, the objects
+        # are initialized to instances of TextIOWrapper.
+        #
+        # To use methods from TextIOWrapper, use an isinstance check to ensure that
+        # the streams have not been overridden:
+        #
+        # if isinstance(sys.stdout, io.TextIOWrapper):
+        #    sys.stdout.reconfigure(...)
+        sys.stdout.reconfigure(encoding="utf-8")
+    colorama.just_fix_windows_console()

    if args.color == "always":
        colorama.init(strip=False)
@@ -1110,8 +1119,8 @@ def handle_common_args(args):


 def main(argv=None):
-    if sys.version_info < (3, 7):
-        raise UnsupportedRuntimeError("This version of capa can only be used with Python 3.7+")
+    if sys.version_info < (3, 8):
+        raise UnsupportedRuntimeError("This version of capa can only be used with Python 3.8+")

    if argv is None:
        argv = sys.argv[1:]
--- a/capa/render/default.py
+++ b/capa/render/default.py
@@ -40,7 +40,7 @@ def render_meta(doc: rd.ResultDocument, ostream: StringIO):
        ("path", doc.meta.sample.path),
    ]

-    ostream.write(tabulate.tabulate(rows, tablefmt="psql"))
+    ostream.write(tabulate.tabulate(rows, tablefmt="mixed_outline"))
    ostream.write("\n")


@@ -102,7 +102,7 @@ def render_capabilities(doc: rd.ResultDocument, ostream: StringIO):

    if rows:
        ostream.write(
-            tabulate.tabulate(rows, headers=[width("CAPABILITY", 50), width("NAMESPACE", 50)], tablefmt="psql")
+            tabulate.tabulate(rows, headers=[width("Capability", 50), width("Namespace", 50)], tablefmt="mixed_outline")
        )
        ostream.write("\n")
    else:
@@ -148,7 +148,7 @@ def render_attack(doc: rd.ResultDocument, ostream: StringIO):
    if rows:
        ostream.write(
            tabulate.tabulate(
-                rows, headers=[width("ATT&CK Tactic", 20), width("ATT&CK Technique", 80)], tablefmt="psql"
+                rows, headers=[width("ATT&CK Tactic", 20), width("ATT&CK Technique", 80)], tablefmt="mixed_grid"
            )
        )
        ostream.write("\n")
@@ -190,7 +190,9 @@ def render_mbc(doc: rd.ResultDocument, ostream: StringIO):

    if rows:
        ostream.write(
-            tabulate.tabulate(rows, headers=[width("MBC Objective", 25), width("MBC Behavior", 75)], tablefmt="psql")
+            tabulate.tabulate(
+                rows, headers=[width("MBC Objective", 25), width("MBC Behavior", 75)], tablefmt="mixed_grid"
+            )
        )
        ostream.write("\n")

--- a/2
+++ b/2
--- a/scripts/import-to-ida.py
+++ b/scripts/import-to-ida.py
@@ -28,13 +28,17 @@ Unless required by applicable law or agreed to in writing, software distributed
 is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and limitations under the License.
 """
-import json
 import logging
+import binascii

 import ida_nalt
 import ida_funcs
 import ida_kernwin

+import capa.rules
+import capa.features.freeze
+import capa.render.result_document
+
 logger = logging.getLogger("capa")


@@ -64,37 +68,37 @@ def main():
    if not path:
        return 0

-    with open(path, "rb") as f:
-        doc = json.loads(f.read().decode("utf-8"))
-
-    if "meta" not in doc or "rules" not in doc:
-        logger.error("doesn't appear to be a capa report")
-        return -1
+    result_doc = capa.render.result_document.ResultDocument.parse_file(path)
+    meta, capabilities = result_doc.to_capa()

    # in IDA 7.4, the MD5 hash may be truncated, for example:
    # wanted: 84882c9d43e23d63b82004fae74ebb61
    # found: b'84882C9D43E23D63B82004FAE74EBB6\x00'
    #
    # see: https://github.com/idapython/bin/issues/11
-    a = doc["meta"]["sample"]["md5"].lower()
-    b = ida_nalt.retrieve_input_file_md5().lower()
+    a = meta.sample.md5.lower()
+    b = binascii.hexlify(ida_nalt.retrieve_input_file_md5()).decode("ascii").lower()
    if not a.startswith(b):
        logger.error("sample mismatch")
        return -2

    rows = []
-    for rule in doc["rules"].values():
-        if rule["meta"].get("lib"):
+    for name in capabilities.keys():
+        rule = result_doc.rules[name]
+        if rule.meta.lib:
            continue
-        if rule["meta"].get("capa/subscope"):
+        if rule.meta.is_subscope_rule:
            continue
-        if rule["meta"]["scope"] != "function":
+        if rule.meta.scope != capa.rules.Scope.FUNCTION:
            continue

-        name = rule["meta"]["name"]
-        ns = rule["meta"].get("namespace", "")
-        for va in rule["matches"].keys():
-            va = int(va)
+        ns = rule.meta.namespace
+
+        for address, _ in rule.matches:
+            if address.type != capa.features.freeze.AddressType.ABSOLUTE:
+                continue
+
+            va = address.value
            rows.append((ns, name, va))

    # order by (namespace, name) so that like things show up together
--- a/scripts/lint.py
+++ b/scripts/lint.py
@@ -873,7 +873,7 @@ def lint(ctx: Context):
    ret = {}

    source_rules = [rule for rule in ctx.rules.rules.values() if not rule.is_subscope_rule()]
-    with tqdm.contrib.logging.tqdm_logging_redirect(source_rules, unit="rule") as pbar:
+    with tqdm.contrib.logging.tqdm_logging_redirect(source_rules, unit="rule", leave=False) as pbar:
        with capa.helpers.redirecting_print_to_tqdm(False):
            for rule in pbar:
                name = rule.name
--- a/scripts/linter-data.json
+++ b/scripts/linter-data.json
@@ -54,6 +54,7 @@
      "T1583.005": "Acquire Infrastructure::Botnet",
      "T1583.006": "Acquire Infrastructure::Web Services",
      "T1583.007": "Acquire Infrastructure::Serverless",
+      "T1583.008": "Acquire Infrastructure::Malvertising",
      "T1584": "Compromise Infrastructure",
      "T1584.001": "Compromise Infrastructure::Domains",
      "T1584.002": "Compromise Infrastructure::DNS Server",
@@ -88,7 +89,8 @@
      "T1608.003": "Stage Capabilities::Install Digital Certificate",
      "T1608.004": "Stage Capabilities::Drive-by Target",
      "T1608.005": "Stage Capabilities::Link Target",
-      "T1608.006": "Stage Capabilities::SEO Poisoning"
+      "T1608.006": "Stage Capabilities::SEO Poisoning",
+      "T1650": "Acquire Access"
    },
    "Initial Access": {
      "T1078": "Valid Accounts",
@@ -128,6 +130,7 @@
      "T1059.006": "Command and Scripting Interpreter::Python",
      "T1059.007": "Command and Scripting Interpreter::JavaScript",
      "T1059.008": "Command and Scripting Interpreter::Network Device CLI",
+      "T1059.009": "Command and Scripting Interpreter::Cloud API",
      "T1072": "Software Deployment Tools",
      "T1106": "Native API",
      "T1129": "Shared Modules",
@@ -145,7 +148,8 @@
      "T1569.002": "System Services::Service Execution",
      "T1609": "Container Administration Command",
      "T1610": "Deploy Container",
-      "T1648": "Serverless Execution"
+      "T1648": "Serverless Execution",
+      "T1651": "Cloud Administration Command"
    },
    "Persistence": {
      "T1037": "Boot or Logon Initialization Scripts",
@@ -247,6 +251,7 @@
      "T1556.005": "Modify Authentication Process::Reversible Encryption",
      "T1556.006": "Modify Authentication Process::Multi-Factor Authentication",
      "T1556.007": "Modify Authentication Process::Hybrid Identity",
+      "T1556.008": "Modify Authentication Process::Network Provider DLL",
      "T1574": "Hijack Execution Flow",
      "T1574.001": "Hijack Execution Flow::DLL Search Order Hijacking",
      "T1574.002": "Hijack Execution Flow::DLL Side-Loading",
@@ -372,6 +377,8 @@
      "T1027.007": "Obfuscated Files or Information::Dynamic API Resolution",
      "T1027.008": "Obfuscated Files or Information::Stripped Payloads",
      "T1027.009": "Obfuscated Files or Information::Embedded Payloads",
+      "T1027.010": "Obfuscated Files or Information::Command Obfuscation",
+      "T1027.011": "Obfuscated Files or Information::Fileless Storage",
      "T1036": "Masquerading",
      "T1036.001": "Masquerading::Invalid Code Signature",
      "T1036.002": "Masquerading::Right-to-Left Override",
@@ -380,6 +387,7 @@
      "T1036.005": "Masquerading::Match Legitimate Name or Location",
      "T1036.006": "Masquerading::Space after Filename",
      "T1036.007": "Masquerading::Double File Extension",
+      "T1036.008": "Masquerading::Masquerade File Type",
      "T1055": "Process Injection",
      "T1055.001": "Process Injection::Dynamic-link Library Injection",
      "T1055.002": "Process Injection::Portable Executable Injection",
@@ -487,6 +495,7 @@
      "T1556.005": "Modify Authentication Process::Reversible Encryption",
      "T1556.006": "Modify Authentication Process::Multi-Factor Authentication",
      "T1556.007": "Modify Authentication Process::Hybrid Identity",
+      "T1556.008": "Modify Authentication Process::Network Provider DLL",
      "T1562": "Impair Defenses",
      "T1562.001": "Impair Defenses::Disable or Modify Tools",
      "T1562.002": "Impair Defenses::Disable Windows Event Logging",
@@ -497,6 +506,7 @@
      "T1562.008": "Impair Defenses::Disable Cloud Logs",
      "T1562.009": "Impair Defenses::Safe Mode Boot",
      "T1562.010": "Impair Defenses::Downgrade Attack",
+      "T1562.011": "Impair Defenses::Spoof Security Alerting",
      "T1564": "Hide Artifacts",
      "T1564.001": "Hide Artifacts::Hidden Files and Directories",
      "T1564.002": "Hide Artifacts::Hidden Users",
@@ -574,6 +584,7 @@
      "T1552.005": "Unsecured Credentials::Cloud Instance Metadata API",
      "T1552.006": "Unsecured Credentials::Group Policy Preferences",
      "T1552.007": "Unsecured Credentials::Container API",
+      "T1552.008": "Unsecured Credentials::Chat Messages",
      "T1555": "Credentials from Password Stores",
      "T1555.001": "Credentials from Password Stores::Keychain",
      "T1555.002": "Credentials from Password Stores::Securityd Memory",
@@ -588,6 +599,7 @@
      "T1556.005": "Modify Authentication Process::Reversible Encryption",
      "T1556.006": "Modify Authentication Process::Multi-Factor Authentication",
      "T1556.007": "Modify Authentication Process::Hybrid Identity",
+      "T1556.008": "Modify Authentication Process::Network Provider DLL",
      "T1557": "Adversary-in-the-Middle",
      "T1557.001": "Adversary-in-the-Middle::LLMNR/NBT-NS Poisoning and SMB Relay",
      "T1557.002": "Adversary-in-the-Middle::ARP Cache Poisoning",
@@ -630,7 +642,7 @@
      "T1124": "System Time Discovery",
      "T1135": "Network Share Discovery",
      "T1201": "Password Policy Discovery",
-      "T1217": "Browser Bookmark Discovery",
+      "T1217": "Browser Information Discovery",
      "T1482": "Domain Trust Discovery",
      "T1497": "Virtualization/Sandbox Evasion",
      "T1497.001": "Virtualization/Sandbox Evasion::System Checks",
@@ -646,7 +658,8 @@
      "T1614.001": "System Location Discovery::System Language Discovery",
      "T1615": "Group Policy Discovery",
      "T1619": "Cloud Storage Object Discovery",
-      "T1622": "Debugger Evasion"
+      "T1622": "Debugger Evasion",
+      "T1652": "Device Driver Discovery"
    },
    "Lateral Movement": {
      "T1021": "Remote Services",
@@ -656,6 +669,7 @@
      "T1021.004": "Remote Services::SSH",
      "T1021.005": "Remote Services::VNC",
      "T1021.006": "Remote Services::Windows Remote Management",
+      "T1021.007": "Remote Services::Cloud Services",
      "T1072": "Software Deployment Tools",
      "T1080": "Taint Shared Content",
      "T1091": "Replication Through Removable Media",
@@ -768,7 +782,8 @@
      "T1537": "Transfer Data to Cloud Account",
      "T1567": "Exfiltration Over Web Service",
      "T1567.001": "Exfiltration Over Web Service::Exfiltration to Code Repository",
-      "T1567.002": "Exfiltration Over Web Service::Exfiltration to Cloud Storage"
+      "T1567.002": "Exfiltration Over Web Service::Exfiltration to Cloud Storage",
+      "T1567.003": "Exfiltration Over Web Service::Exfiltration to Text Storage Sites"
    },
    "Impact": {
      "T1485": "Data Destruction",
--- a/scripts/profile-time.py
+++ b/scripts/profile-time.py
@@ -112,7 +112,7 @@ def main(argv=None):
        )

    assert isinstance(extractor, StaticFeatureExtractor)
-    with tqdm.tqdm(total=args.number * args.repeat) as pbar:
+    with tqdm.tqdm(total=args.number * args.repeat, leave=False) as pbar:

        def do_iteration():
            capa.perf.reset()
--- a/setup.py
+++ b/setup.py
@@ -14,13 +14,13 @@ requirements = [
    "tqdm==4.65.0",
    "pyyaml==6.0",
    "tabulate==0.9.0",
-    "colorama==0.4.5",
+    "colorama==0.4.6",
    "termcolor==2.3.0",
    "wcwidth==0.2.6",
    "ida-settings==2.1.0",
    "viv-utils[flirt]==0.7.9",
    "halo==0.0.31",
-    "networkx==2.5.1",  # newer versions no longer support py3.7.
+    "networkx==3.1",
    "ruamel.yaml==0.17.32",
    "vivisect==1.1.1",
    "pefile==2023.2.7",
@@ -84,7 +84,7 @@ setuptools.setup(
            "mypy-protobuf==3.4.0",
            # type stubs for mypy
            "types-backports==0.1.3",
-            "types-colorama==0.4.15",
+            "types-colorama==0.4.15.11",
            "types-PyYAML==6.0.8",
            "types-tabulate==0.9.0.1",
            "types-termcolor==1.1.4",
@@ -107,5 +107,5 @@ setuptools.setup(
        "Programming Language :: Python :: 3",
        "Topic :: Security",
    ],
-    python_requires=">=3.7",
+    python_requires=">=3.8",
 )
--- a/tests/data
+++ b/tests/data
--- a/tests/test_binja_features.py
+++ b/tests/test_binja_features.py
@@ -37,6 +37,8 @@ except ImportError:
    indirect=["sample", "scope"],
 )
 def test_binja_features(sample, scope, feature, expected):
+    if feature == capa.features.common.Characteristic("stack string"):
+        pytest.xfail("skip failing Binja stack string detection temporarily, see #1473")
    fixtures.do_test_feature_presence(fixtures.get_binja_extractor, sample, scope, feature, expected)