Merge branch 'master' into sync-master

scripts/import-to-ida.py

@@ -28,13 +28,17 @@ Unless required by applicable law or agreed to in writing, software distributed
  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and limitations under the License.
 """
-import json
 import logging
+import binascii
 
 import ida_nalt
 import ida_funcs
 import ida_kernwin
 
+import capa.rules
+import capa.features.freeze
+import capa.render.result_document
+
 logger = logging.getLogger("capa")
 
 
@@ -64,37 +68,37 @@ def main():
     if not path:
         return 0
 
-    with open(path, "rb") as f:
-        doc = json.loads(f.read().decode("utf-8"))
-
-    if "meta" not in doc or "rules" not in doc:
-        logger.error("doesn't appear to be a capa report")
-        return -1
+    result_doc = capa.render.result_document.ResultDocument.parse_file(path)
+    meta, capabilities = result_doc.to_capa()
 
     # in IDA 7.4, the MD5 hash may be truncated, for example:
     # wanted: 84882c9d43e23d63b82004fae74ebb61
     # found: b'84882C9D43E23D63B82004FAE74EBB6\x00'
     #
     # see: https://github.com/idapython/bin/issues/11
-    a = doc["meta"]["sample"]["md5"].lower()
-    b = ida_nalt.retrieve_input_file_md5().lower()
+    a = meta.sample.md5.lower()
+    b = binascii.hexlify(ida_nalt.retrieve_input_file_md5()).decode("ascii").lower()
     if not a.startswith(b):
         logger.error("sample mismatch")
         return -2
 
     rows = []
-    for rule in doc["rules"].values():
-        if rule["meta"].get("lib"):
+    for name in capabilities.keys():
+        rule = result_doc.rules[name]
+        if rule.meta.lib:
             continue
-        if rule["meta"].get("capa/subscope"):
+        if rule.meta.is_subscope_rule:
             continue
-        if rule["meta"]["scope"] != "function":
+        if rule.meta.scope != capa.rules.Scope.FUNCTION:
             continue
 
-        name = rule["meta"]["name"]
-        ns = rule["meta"].get("namespace", "")
-        for va in rule["matches"].keys():
-            va = int(va)
+        ns = rule.meta.namespace
+
+        for address, _ in rule.matches:
+            if address.type != capa.features.freeze.AddressType.ABSOLUTE:
+                continue
+
+            va = address.value
             rows.append((ns, name, va))
 
     # order by (namespace, name) so that like things show up together

scripts/lint.py

@@ -873,7 +873,7 @@ def lint(ctx: Context):
     ret = {}
 
     source_rules = [rule for rule in ctx.rules.rules.values() if not rule.is_subscope_rule()]
-    with tqdm.contrib.logging.tqdm_logging_redirect(source_rules, unit="rule") as pbar:
+    with tqdm.contrib.logging.tqdm_logging_redirect(source_rules, unit="rule", leave=False) as pbar:
         with capa.helpers.redirecting_print_to_tqdm(False):
             for rule in pbar:
                 name = rule.name

scripts/linter-data.json

@@ -54,6 +54,7 @@
       "T1583.005": "Acquire Infrastructure::Botnet",
       "T1583.006": "Acquire Infrastructure::Web Services",
       "T1583.007": "Acquire Infrastructure::Serverless",
+      "T1583.008": "Acquire Infrastructure::Malvertising",
       "T1584": "Compromise Infrastructure",
       "T1584.001": "Compromise Infrastructure::Domains",
       "T1584.002": "Compromise Infrastructure::DNS Server",
@@ -88,7 +89,8 @@
       "T1608.003": "Stage Capabilities::Install Digital Certificate",
       "T1608.004": "Stage Capabilities::Drive-by Target",
       "T1608.005": "Stage Capabilities::Link Target",
-      "T1608.006": "Stage Capabilities::SEO Poisoning"
+      "T1608.006": "Stage Capabilities::SEO Poisoning",
+      "T1650": "Acquire Access"
     },
     "Initial Access": {
       "T1078": "Valid Accounts",
@@ -128,6 +130,7 @@
       "T1059.006": "Command and Scripting Interpreter::Python",
       "T1059.007": "Command and Scripting Interpreter::JavaScript",
       "T1059.008": "Command and Scripting Interpreter::Network Device CLI",
+      "T1059.009": "Command and Scripting Interpreter::Cloud API",
       "T1072": "Software Deployment Tools",
       "T1106": "Native API",
       "T1129": "Shared Modules",
@@ -145,7 +148,8 @@
       "T1569.002": "System Services::Service Execution",
       "T1609": "Container Administration Command",
       "T1610": "Deploy Container",
-      "T1648": "Serverless Execution"
+      "T1648": "Serverless Execution",
+      "T1651": "Cloud Administration Command"
     },
     "Persistence": {
       "T1037": "Boot or Logon Initialization Scripts",
@@ -247,6 +251,7 @@
       "T1556.005": "Modify Authentication Process::Reversible Encryption",
       "T1556.006": "Modify Authentication Process::Multi-Factor Authentication",
       "T1556.007": "Modify Authentication Process::Hybrid Identity",
+      "T1556.008": "Modify Authentication Process::Network Provider DLL",
       "T1574": "Hijack Execution Flow",
       "T1574.001": "Hijack Execution Flow::DLL Search Order Hijacking",
       "T1574.002": "Hijack Execution Flow::DLL Side-Loading",
@@ -372,6 +377,8 @@
       "T1027.007": "Obfuscated Files or Information::Dynamic API Resolution",
       "T1027.008": "Obfuscated Files or Information::Stripped Payloads",
       "T1027.009": "Obfuscated Files or Information::Embedded Payloads",
+      "T1027.010": "Obfuscated Files or Information::Command Obfuscation",
+      "T1027.011": "Obfuscated Files or Information::Fileless Storage",
       "T1036": "Masquerading",
       "T1036.001": "Masquerading::Invalid Code Signature",
       "T1036.002": "Masquerading::Right-to-Left Override",
@@ -380,6 +387,7 @@
       "T1036.005": "Masquerading::Match Legitimate Name or Location",
       "T1036.006": "Masquerading::Space after Filename",
       "T1036.007": "Masquerading::Double File Extension",
+      "T1036.008": "Masquerading::Masquerade File Type",
       "T1055": "Process Injection",
       "T1055.001": "Process Injection::Dynamic-link Library Injection",
       "T1055.002": "Process Injection::Portable Executable Injection",
@@ -487,6 +495,7 @@
       "T1556.005": "Modify Authentication Process::Reversible Encryption",
       "T1556.006": "Modify Authentication Process::Multi-Factor Authentication",
       "T1556.007": "Modify Authentication Process::Hybrid Identity",
+      "T1556.008": "Modify Authentication Process::Network Provider DLL",
       "T1562": "Impair Defenses",
       "T1562.001": "Impair Defenses::Disable or Modify Tools",
       "T1562.002": "Impair Defenses::Disable Windows Event Logging",
@@ -497,6 +506,7 @@
       "T1562.008": "Impair Defenses::Disable Cloud Logs",
       "T1562.009": "Impair Defenses::Safe Mode Boot",
       "T1562.010": "Impair Defenses::Downgrade Attack",
+      "T1562.011": "Impair Defenses::Spoof Security Alerting",
       "T1564": "Hide Artifacts",
       "T1564.001": "Hide Artifacts::Hidden Files and Directories",
       "T1564.002": "Hide Artifacts::Hidden Users",
@@ -574,6 +584,7 @@
       "T1552.005": "Unsecured Credentials::Cloud Instance Metadata API",
       "T1552.006": "Unsecured Credentials::Group Policy Preferences",
       "T1552.007": "Unsecured Credentials::Container API",
+      "T1552.008": "Unsecured Credentials::Chat Messages",
       "T1555": "Credentials from Password Stores",
       "T1555.001": "Credentials from Password Stores::Keychain",
       "T1555.002": "Credentials from Password Stores::Securityd Memory",
@@ -588,6 +599,7 @@
       "T1556.005": "Modify Authentication Process::Reversible Encryption",
       "T1556.006": "Modify Authentication Process::Multi-Factor Authentication",
       "T1556.007": "Modify Authentication Process::Hybrid Identity",
+      "T1556.008": "Modify Authentication Process::Network Provider DLL",
       "T1557": "Adversary-in-the-Middle",
       "T1557.001": "Adversary-in-the-Middle::LLMNR/NBT-NS Poisoning and SMB Relay",
       "T1557.002": "Adversary-in-the-Middle::ARP Cache Poisoning",
@@ -630,7 +642,7 @@
       "T1124": "System Time Discovery",
       "T1135": "Network Share Discovery",
       "T1201": "Password Policy Discovery",
-      "T1217": "Browser Bookmark Discovery",
+      "T1217": "Browser Information Discovery",
       "T1482": "Domain Trust Discovery",
       "T1497": "Virtualization/Sandbox Evasion",
       "T1497.001": "Virtualization/Sandbox Evasion::System Checks",
@@ -646,7 +658,8 @@
       "T1614.001": "System Location Discovery::System Language Discovery",
       "T1615": "Group Policy Discovery",
       "T1619": "Cloud Storage Object Discovery",
-      "T1622": "Debugger Evasion"
+      "T1622": "Debugger Evasion",
+      "T1652": "Device Driver Discovery"
     },
     "Lateral Movement": {
       "T1021": "Remote Services",
@@ -656,6 +669,7 @@
       "T1021.004": "Remote Services::SSH",
       "T1021.005": "Remote Services::VNC",
       "T1021.006": "Remote Services::Windows Remote Management",
+      "T1021.007": "Remote Services::Cloud Services",
       "T1072": "Software Deployment Tools",
       "T1080": "Taint Shared Content",
       "T1091": "Replication Through Removable Media",
@@ -768,7 +782,8 @@
       "T1537": "Transfer Data to Cloud Account",
       "T1567": "Exfiltration Over Web Service",
       "T1567.001": "Exfiltration Over Web Service::Exfiltration to Code Repository",
-      "T1567.002": "Exfiltration Over Web Service::Exfiltration to Cloud Storage"
+      "T1567.002": "Exfiltration Over Web Service::Exfiltration to Cloud Storage",
+      "T1567.003": "Exfiltration Over Web Service::Exfiltration to Text Storage Sites"
     },
     "Impact": {
       "T1485": "Data Destruction",

scripts/profile-time.py

@@ -112,7 +112,7 @@ def main(argv=None):
         )
 
     assert isinstance(extractor, StaticFeatureExtractor)
-    with tqdm.tqdm(total=args.number * args.repeat) as pbar:
+    with tqdm.tqdm(total=args.number * args.repeat, leave=False) as pbar:
 
         def do_iteration():
             capa.perf.reset()