merging upstream

.github/workflows/tests.yml

@@ -49,11 +49,9 @@ jobs:
       matrix:
         include:
           - python: 2.7
-          - python: 3.6
           - python: 3.7
           - python: 3.8
-          #- python: '3.9.0-rc.1' # Python latest
-          # disabled due to LIEF, see #362
+          - python: 3.9.1
     steps:
     - name: Checkout capa with submodules
       uses: actions/checkout@v2

README.md

@@ -1,7 +1,7 @@
 ![capa](.github/logo.png)
 
 [![CI status](https://github.com/fireeye/capa/workflows/CI/badge.svg)](https://github.com/fireeye/capa/actions?query=workflow%3ACI+event%3Apush+branch%3Amaster)
-[![Number of rules](https://img.shields.io/badge/rules-443-blue.svg)](https://github.com/fireeye/capa-rules)
+[![Number of rules](https://img.shields.io/badge/rules-446-blue.svg)](https://github.com/fireeye/capa-rules)
 [![License](https://img.shields.io/badge/license-Apache--2.0-green.svg)](LICENSE.txt)
 
 capa detects capabilities in executable files.

capa/features/insn.py

@@ -16,7 +16,7 @@ class API(Feature):
             modname, _, impname = name.rpartition(".")
             name = modname.lower() + "." + impname
 
-        super(API, self).__init__(name, description)
+        super(API, self).__init__(name, description=description)
 
 
 class Number(Feature):

capa/rules.py

@@ -866,7 +866,8 @@ class RuleSet(object):
         given a collection of rules, collect the rules that are needed at the given scope.
         these rules are ordered topologically.
 
-        don't include "lib" rules, unless they are dependencies of other rules.
+        don't include auto-generated "subscope" rules.
+        we want to include general "lib" rules here - even if they are not dependencies of other rules, see #398
         """
         scope_rules = set([])
 
@@ -875,7 +876,7 @@ class RuleSet(object):
         #  at lower scope, e.g. function scope.
         # so, we find all dependencies of all rules, and later will filter them down.
         for rule in rules:
-            if rule.meta.get("lib", False):
+            if rule.meta.get("capa/subscope-rule", False):
                 continue
 
             scope_rules.update(get_rules_and_dependencies(rules, rule.name))

scripts/lint.py

@@ -15,6 +15,7 @@ See the License for the specific language governing permissions and limitations
 """
 import os
 import sys
+import time
 import string
 import hashlib
 import logging
@@ -194,7 +195,7 @@ class DoesntMatchExample(Lint):
                 continue
 
             try:
-                extractor = capa.main.get_extractor(path, "auto")
+                extractor = capa.main.get_extractor(path, "auto", disable_progress=True)
                 capabilities, meta = capa.main.find_capabilities(ctx["rules"], extractor, disable_progress=True)
             except Exception as e:
                 logger.error("failed to extract capabilities: %s %s %s", rule.name, path, e)
@@ -232,7 +233,7 @@ class LibRuleNotInLibDirectory(Lint):
         if "lib" not in rule.meta:
             return False
 
-        return "/lib/" not in get_normpath(rule.meta["capa/path"])
+        return "lib/" not in get_normpath(rule.meta["capa/path"])
 
 
 class LibRuleHasNamespace(Lint):
@@ -518,8 +519,10 @@ def main(argv=None):
     capa.main.set_vivisect_log_level(logging.CRITICAL)
     logging.getLogger("capa").setLevel(logging.CRITICAL)
 
+    time0 = time.time()
+
     try:
-        rules = capa.main.get_rules(args.rules)
+        rules = capa.main.get_rules(args.rules, disable_progress=True)
         rules = capa.rules.RuleSet(rules)
         logger.info("successfully loaded %s rules", len(rules))
         if args.tag:
@@ -545,6 +548,10 @@ def main(argv=None):
     }
 
     did_violate = lint(ctx, rules)
+
+    diff = time.time() - time0
+    logger.debug("lint ran for ~ %02d:%02d", (diff // 60), diff)
+
     if not did_violate:
         logger.info("no suggestions, nice!")
         return 0

setup.py

@@ -27,7 +27,7 @@ if sys.version_info >= (3, 0):
     # py3
     requirements.append("halo")
     requirements.append("networkx")
-    requirements.append("smda==1.5.10")
+    requirements.append("smda==1.5.13")
 else:
     # py2
     requirements.append("enum34==1.1.6")  # v1.1.6 is needed by halo 0.0.30 / spinners 0.0.24

tests/test_rules.py

@@ -282,7 +282,8 @@ def test_lib_rules():
             ),
         ]
     )
-    assert len(rules.function_rules) == 1
+    # lib rules are added to the rule set
+    assert len(rules.function_rules) == 2
 
 
 def test_subscope_rules():