changelog: v1.4.1

fix build on windows-latest

.github/pyinstaller/pyinstaller.spec

@@ -44,7 +44,6 @@ a = Analysis(
     hiddenimports=[
         # vivisect does manual/runtime importing of its modules,
         # so declare the things that could be imported here.
-        "pycparser",
         "vivisect",
         "vivisect.analysis",
         "vivisect.analysis.amd64",
@@ -92,11 +91,13 @@ a = Analysis(
         "vivisect.impapi.windows",
         "vivisect.impapi.windows.amd64",
         "vivisect.impapi.windows.i386",
+        "vivisect.impapi.winkern.i386",
+        "vivisect.impapi.winkern.amd64",
         "vivisect.parsers.blob",
         "vivisect.parsers.elf",
         "vivisect.parsers.ihex",
         "vivisect.parsers.macho",
-        "vivisect.parsers.parse_pe",
+        "vivisect.parsers.pe",
         "vivisect.parsers.utils",
         "vivisect.storage",
         "vivisect.storage.basicfile",

.github/workflows/build.yml

@@ -2,7 +2,7 @@ name: build
 
 on:
   release:
-    types: [created, edited]
+    types: [created, edited, published]
 
 jobs:
   build:
@@ -30,6 +30,12 @@ jobs:
         uses: actions/setup-python@v2
         with:
           python-version: 2.7
+      - if: matrix.os == 'ubuntu-latest'
+        run: sudo apt-get install -y libyaml-dev
+      - if: matrix.os == 'windows-latest'
+        run: |
+          choco install vcredist2008
+          choco install --ignore-dependencies vcpython27
       - name: Install PyInstaller
         # pyinstaller 4 doesn't support Python 2.7
         run: pip install 'pyinstaller==3.*'
@@ -65,7 +71,7 @@ jobs:
       - name: Set executable flag
         run: chmod +x ${{ matrix.artifact_name }}
       - name: Set zip name
-        run: echo ::set-env name=zip_name::capa-${GITHUB_REF#refs/tags/}-${{ matrix.asset_name }}.zip
+        run: echo "zip_name=capa-${GITHUB_REF#refs/tags/}-${{ matrix.asset_name }}.zip" >> $GITHUB_ENV
       - name: Zip ${{ matrix.artifact_name }} into ${{ env.zip_name }}
         run: zip ${{ env.zip_name }} ${{ matrix.artifact_name }}
       - name: Upload ${{ env.zip_name }} to GH Release
@@ -74,4 +80,3 @@ jobs:
           repo_token: ${{ secrets.GITHUB_TOKEN}}
           file: ${{ env.zip_name }}
           tag: ${{ github.ref }}
-

.github/workflows/publish.yml

@@ -0,0 +1,29 @@
+# This workflows will upload a Python Package using Twine when a release is created
+# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
+
+name: publish to pypi
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '2.7'
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install setuptools wheel twine
+      - name: Build and publish
+        env:
+          TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
+          TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+        run: |
+          python setup.py sdist bdist_wheel
+          twine upload --skip-existing dist/*

.github/workflows/tests.yml

@@ -51,7 +51,7 @@ jobs:
           - python: 3.6
           - python: 3.7
           - python: 3.8
-          - python: '3.9.0-alpha - 3.9.x' # Python latest
+          - python: '3.9.0-rc.1' # Python latest
     steps:
     - name: Checkout capa with submodules
       uses: actions/checkout@v2
@@ -61,6 +61,8 @@ jobs:
       uses: actions/setup-python@v2
       with:
         python-version: ${{ matrix.python }}
+    - name: Install pyyaml
+      run: sudo apt-get install -y libyaml-dev
     - name: Install capa
       run: pip install -e .[dev]
     - name: Run tests

CHANGELOG.md

@@ -1,5 +1,205 @@
 # Change Log
 
+## v1.4.1 (2020-10-23)
+
+This release fixes an issue building capa on our CI server, which prevented us from building standalone binaries for v1.4.1.
+
+### Bug Fixes
+
+  - install VC dependencies for Python 2.7 during Windows build
+  
+### Raw diffs
+
+  - [capa v1.4.0...v1.4.1](https://github.com/fireeye/capa/compare/v1.4.0...v1.4.1)
+  - [capa-rules v1.4.0...v1.4.1](https://github.com/fireeye/capa-rules/compare/v1.4.0...v1.4.1)  
+
+## v1.4.0 (2020-10-23)
+
+This capa release includes changes to the rule parsing, enhanced feature extraction, various bug fixes, and improved capa scripts. Everyone should benefit from the improved functionality and performance. The community helped to add 69 new rules. We appreciate everyone who opened issues, provided feedback, and contributed code and rules. A special shout out to the following new project contributors:
+
+  - @mwilliams31
+  - @yt0ng
+
+@dzbeck added [Malware Behavior Catalog](https://github.com/MBCProject/mbc-markdown) (MBC) and ATT&CK mappings for 86 rules.
+
+Download a standalone binary below and checkout the readme [here on GitHub](https://github.com/fireeye/capa/). Report issues on our [issue tracker](https://github.com/fireeye/capa/issues) and contribute new rules at [capa-rules](https://github.com/fireeye/capa-rules/).
+
+### New features
+
+  - script that demonstrates bulk processing @williballenthin #307
+  - main: render MBC table @mr-tz #332
+  - ida backend: improve detection of APIs called via two or more chained thunks @mike-hunhoff #340
+  - viv backend: improve detection of APIs called via two or more chained thunks @mr-tz #341
+  - features: extract APIs called via jmp instruction @mr-tz #337
+
+### New rules
+
+  - clear the Windows event log @mike-hunhoff
+  - crash the Windows event logging service @mike-hunhoff
+  - packed with kkrunchy @re-fox
+  - packed with nspack @re-fox
+  - packed with pebundle @re-fox
+  - packed with pelocknt @re-fox
+  - packed with peshield @re-fox
+  - packed with petite @re-fox
+  - packed with rlpack @re-fox
+  - packed with upack @re-fox
+  - packed with y0da crypter @re-fox
+  - compiled with rust @re-fox
+  - compute adler32 checksum @mwilliams31
+  - encrypt-data-using-hc-128 @recvfrom
+  - manipulate console @williballenthin
+  - references logon banner @re-fox
+  - terminate process via fastfail @re-fox
+  - delete volume shadow copies @mr-tz
+  - authenticate HMAC @mr-tz
+  - compiled from EPL @williballenthin
+  - compiled with Go @williballenthin
+  - create Restart Manager session @mike-hunhoff
+  - decode data using Base64 via WinAPI @mike-hunhoff
+  - empty recycle bin quietly @mwilliams31
+  - enumerate network shares @mike-hunhoff
+  - hook routines via microsoft detours @williballenthin
+  - hooked by API Override @williballenthin
+  - impersonate user @mike-hunhoff
+  - the @williballenthin packer detection package, thanks to Hexacorn for the data, see https://www.hexacorn.com/blog/2016/12/15/pe-section-names-re-visited/
+    - packed with CCG
+    - packed with Crunch
+    - packed with Dragon Armor
+    - packed with enigma
+    - packed with Epack
+    - packed with MaskPE
+    - packed with MEW
+    - packed with Mpress
+    - packed with Neolite
+    - packed with PECompact
+    - packed with Pepack
+    - packed with Perplex
+    - packed with ProCrypt
+    - packed with RPCrypt
+    - packed with SeauSFX
+    - packed with Shrinker
+    - packed with Simple Pack
+    - packed with StarForce
+    - packed with SVKP
+    - packed with Themida
+    - packed with TSULoader
+    - packed with VProtect
+    - packed with WWPACK
+    - rebuilt by ImpRec
+    - packaged as a Pintool
+    - packaged as a CreateInstall installer
+    - packaged as a WinZip self-extracting archive
+  - reference 114DNS DNS server @williballenthin
+  - reference AliDNS DNS server @williballenthin
+  - reference Cloudflare DNS server @williballenthin
+  - reference Comodo Secure DNS server @williballenthin
+  - reference Google Public DNS server @williballenthin
+  - reference Hurricane Electric DNS server @williballenthin
+  - reference kornet DNS server @williballenthin
+  - reference L3 DNS server @williballenthin
+  - reference OpenDNS DNS server @williballenthin
+  - reference Quad9 DNS server @williballenthin
+  - reference Verisign DNS server @williballenthin
+  - run as service @mike-hunhoff
+  - schedule task via ITaskService @mike-hunhoff
+  - references DNS over HTTPS endpoints @yt0ng
+
+### Bug fixes
+
+  - ida plugin: fix tree-view exception @mike-hunhoff #315
+  - ida plugin: fix feature count @mike-hunhoff
+  - main: fix reported total rule count @williballenthin #325
+  - features: fix handling of API names with multiple periods @mike-hunhoff #329
+  - ida backend: find all byte sequences instead of only first @mike-hunhoff #335
+  - features: display 0 value @mr-tz #338
+  - ida backend: extract ordinal and name imports @mr-tz #343
+  - show-features: improvements and support within IDA @mr-tz #342
+  - main: sanity check MBC rendering @williballenthin
+  - main: handle sample path that contains non-ASCII characters @mr-tz #328
+
+### Changes
+
+  - rules: use yaml.CLoader for better performance @williballenthin #306
+  - rules: parse descriptions for statements @mr-tz #312
+
+### Raw diffs
+
+  - [capa v1.3.0...v1.4.0](https://github.com/fireeye/capa/compare/v1.3.0...v1.4.0)
+  - [capa-rules v1.3.0...v1.4.0](https://github.com/fireeye/capa-rules/compare/v1.3.0...v1.4.0)
+
+## v1.3.0 (2020-09-14)
+
+This release brings newly updated mappings to the [Malware Behavior Catalog version 2.0](https://github.com/MBCProject/mbc-markdown), many enhancements to the IDA Pro plugin, [flare-capa on PyPI](https://pypi.org/project/flare-capa/), a bunch of bug fixes to improve feature extraction, and four new rules. We received contributions from ten reverse engineers, including seven new ones:
+
+  - @dzbeck
+  - @recvfrom
+  - @toomanybananas
+  - @cclauss 
+  - @adamprescott91 
+  - @weslambert
+  - @stevemk14ebr 
+  
+Download a standalone binary below and checkout the readme [here on GitHub](https://github.com/fireeye/capa/). Report issues on our [issue tracker](https://github.com/fireeye/capa/issues) and contribute new rules at [capa-rules](https://github.com/fireeye/capa-rules/).
+
+### Key changes to IDA Plugin
+
+The IDA Pro integration is now distributed as a real plugin, instead of a script. This enables a few things:
+
+  - keyboard shortcuts and file menu integration
+  - updates distributed PyPI/`pip install --upgrade` without touching your `%IDADIR%`
+  - generally doing thing the "right way"
+
+How to get this new version? Its easy: download [capa_explorer.py](https://raw.githubusercontent.com/fireeye/capa/master/capa/ida/plugin/capa_explorer.py) to your IDA plugins directory and update your capa installation (incidentally, this is a good opportunity to migrate to `pip install flare-capa` instead of git checkouts). Now you should see the plugin listed in the `Edit > Plugins > FLARE capa explorer` menu in IDA. 
+
+Please refer to the plugin [readme](https://github.com/fireeye/capa/blob/master/capa/ida/plugin/README.md) for additional information on installing and using the IDA Pro plugin.
+
+Please open an issue in this repository if you notice anything weird.
+ 
+### New features
+
+  - ida plugin: now a real plugin, not a script @mike-hunhoff 
+  - core: distributed via PyPI as [flare-capa](https://pypi.org/project/flare-capa/) @williballenthin 
+  - features: enable automatic A/W handling for imports @williballenthin @Ana06 #246 
+  - ida plugin: persist rules directory setting via [ida-settings](https://github.com/williballenthin/ida-settings) @williballenthin #268
+  - ida plugin: add search bar to results view @williballenthin #285
+  - ida plugin: add `Analyze` and `Reset` buttons to tree view @mike-hunhoff #304
+  - ida plugin: add status label to tree view @mike-hunhoff
+  - ida plugin: add progress indicator @mike-hunhoff, @mr-tz
+
+### New rules
+
+  - compiled with py2exe @re-fox
+  - resolve path using msvcrt @re-fox 
+  - decompress data using QuickLZ @edeca
+  - encrypt data using sosemanuk @recvfrom 
+
+### Bug fixes
+
+  - rule: reduce FP in DNS resolution @toomanybananas
+  - engine: report correct strings matched via regex @williballenthin #262 
+  - formatter: correctly format descriptions in two-line syntax @williballenthin @recvfrom #263 
+  - viv: better extract offsets from SibOper operands @williballenthin @edeca #276 
+  - import-to-ida: fix import error @cclauss 
+  - viv: don't write settings to ~/.viv/viv.json @williballenthin @rakuy0 @weslambert #244
+  - ida plugin: remove dependency loop that resulted in unnecessary overhead @mike-hunhoff #303
+  - ida plugin: correctly highlight regex matches in IDA Disassembly view @mike-hunhoff #305
+  - ida plugin: better handle rule directory prompt and failure case @stevemk14ebr @mike-hunhoff #309
+
+### Changes
+
+  - rules: update meta mapping to MBC 2.0! @dzbeck
+  - render: don't display rules that are also matched by other rules @williballenthin @Ana06 #224
+  - ida plugin: simplify tabs, removing summary and adding detail to results view @williballenthin #286
+  - ida plugin: analysis is no longer automatically started when plugin is first opened @mike-hunhoff #304
+  - ida plugin: user must manually select a capa rules directory before analysis can be performed @mike-hunhoff
+  - ida plugin: user interface controls are disabled until analysis is performed @mike-hunhoff #304
+
+### Raw diffs
+
+  - [capa v1.2.0...v1.3.0](https://github.com/fireeye/capa/compare/v1.2.0...v1.3.0)
+  - [capa-rules v1.2.0...v1.3.0](https://github.com/fireeye/capa-rules/compare/v1.2.0...v1.3.0)
+
 ## v1.2.0 (2020-08-31)
 
 This release brings UI enhancements, especially for the IDA Pro plugin, 

README.md

@@ -1,7 +1,7 @@
 ![capa](.github/logo.png)
 
 [![CI status](https://github.com/fireeye/capa/workflows/CI/badge.svg)](https://github.com/fireeye/capa/actions?query=workflow%3ACI+event%3Apush+branch%3Amaster)
-[![Number of rules](https://img.shields.io/badge/rules-341-blue.svg)](https://github.com/fireeye/capa-rules)
+[![Number of rules](https://img.shields.io/badge/rules-414-blue.svg)](https://github.com/fireeye/capa-rules)
 [![License](https://img.shields.io/badge/license-Apache--2.0-green.svg)](LICENSE.txt)
 
 capa detects capabilities in executable files.
@@ -71,7 +71,7 @@ Alternatively, you can fetch a nightly build of a standalone binary from one of
 
 To use capa as a library or integrate with another tool, see [doc/installation.md](doc/installation.md) for further setup instructions.
 
-For more information about how to use capa, including running it as an IDA script/plugin see [doc/usage.md](doc/usage.md).
+For more information about how to use capa, see [doc/usage.md](doc/usage.md).
 
 # example
 
@@ -146,12 +146,11 @@ rule:
 The [github.com/fireeye/capa-rules](https://github.com/fireeye/capa-rules) repository contains hundreds of standard library rules that are distributed with capa.
 Please learn to write rules and contribute new entries as you find interesting techniques in malware.
 
-If you use IDA Pro, then you use can use the [IDA Pro plugin for capa](./capa/ida/ida_capa_explorer.py).
-This script adds new user interface elements to IDA, including an interactive tree view of rule matches and their locations within the current database.
-As you select the checkboxes, the plugin will highlight the addresses associated with the features.
-We use this plugin all the time to quickly jump to interesting parts of a program.
+If you use IDA Pro, then you use can use the [capa explorer IDA plugin](capa/ida/plugin/).
+capa explorer lets you quickly identify and navigate to interesting areas of a program and dissect capa rule matches at
+the assembly level.
 
-![capa + IDA Pro integration](.github/capa-ida.jpg)
+![capa + IDA Pro integration](doc/img/ida_plugin_intro.gif)
 
 # further information
 ## capa

capa/engine.py

@@ -6,7 +6,6 @@
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
 
-import sys
 import copy
 import collections
 

capa/features/__init__.py

@@ -16,6 +16,9 @@ import capa.engine
 logger = logging.getLogger(__name__)
 MAX_BYTES_FEATURE_SIZE = 0x100
 
+# thunks may be chained so we specify a delta to control the depth to which these chains are explored
+THUNK_CHAIN_DEPTH_DELTA = 5
+
 # identifiers for supported architectures names that tweak a feature
 # for example, offset/x32
 ARCH_X32 = "x32"
@@ -74,7 +77,7 @@ class Feature(object):
         return self.value
 
     def __str__(self):
-        if self.value:
+        if self.value is not None:
             if self.description:
                 return "%s(%s = %s)" % (self.name, self.get_value_str(), self.description)
             else:
@@ -139,7 +142,6 @@ class Regex(String):
             raise ValueError(
                 "invalid regular expression: %s it should use Python syntax, try it at https://pythex.org" % value
             )
-        self.match = None
 
     def evaluate(self, ctx):
         for feature, locations in ctx.items():
@@ -151,10 +153,38 @@ class Regex(String):
             # using this mode cleans is more convenient for rule authors,
             # so that they don't have to prefix/suffix their terms like: /.*foo.*/.
             if self.re.search(feature.value):
-                self.match = feature.value
-                return capa.engine.Result(True, self, [], locations=locations)
+                # unlike other features, we cannot return put a reference to `self` directly in a `Result`.
+                # this is because `self` may match on many strings, so we can't stuff the matched value into it.
+                # instead, return a new instance that has a reference to both the regex and the matched value.
+                # see #262.
+                return capa.engine.Result(True, _MatchedRegex(self, feature.value), [], locations=locations)
 
-        return capa.engine.Result(False, self, [])
+        return capa.engine.Result(False, _MatchedRegex(self, None), [])
+
+    def __str__(self):
+        return "regex(string =~ %s)" % self.value
+
+
+class _MatchedRegex(Regex):
+    """
+    this represents a specific instance of a regular expression feature match.
+    treat it the same as a `Regex` except it has the `match` field that contains the complete string that matched.
+
+    note: this type should only ever be constructed by `Regex.evaluate()`. it is not part of the public API.
+    """
+
+    def __init__(self, regex, match):
+        """
+        args:
+          regex (Regex): the regex feature that matches
+          match (string|None): the matching string or None if it doesn't match
+        """
+        super(_MatchedRegex, self).__init__(regex.value, description=regex.description)
+        # we want this to collide with the name of `Regex` above,
+        # so that it works nicely with the renderers.
+        self.name = "regex"
+        # this may be None if the regex doesn't match
+        self.match = match
 
     def __str__(self):
         return 'regex(string =~ %s, matched = "%s")' % (self.value, self.match)

capa/features/extractors/helpers.py

@@ -9,6 +9,7 @@
 import sys
 import builtins
 
+from capa.features.file import Import
 from capa.features.insn import API
 
 MIN_STACKSTRING_LEN = 8
@@ -21,25 +22,32 @@ def xor_static(data, i):
         return "".join(chr(ord(c) ^ i) for c in data)
 
 
-def is_aw_function(function_name):
+def is_aw_function(symbol):
     """
     is the given function name an A/W function?
     these are variants of functions that, on Windows, accept either a narrow or wide string.
     """
-    if len(function_name) < 2:
+    if len(symbol) < 2:
         return False
 
     # last character should be 'A' or 'W'
-    if function_name[-1] not in ("A", "W"):
+    if symbol[-1] not in ("A", "W"):
         return False
 
     # second to last character should be lowercase letter
-    return "a" <= function_name[-2] <= "z" or "0" <= function_name[-2] <= "9"
+    return "a" <= symbol[-2] <= "z" or "0" <= symbol[-2] <= "9"
 
 
-def generate_api_features(apiname, va):
+def is_ordinal(symbol):
     """
-    for a given function name and address, generate API names.
+    is the given symbol an ordinal that is prefixed by "#"?
+    """
+    return symbol[0] == "#"
+
+
+def generate_symbols(dll, symbol):
+    """
+    for a given dll and symbol name, generate variants.
     we over-generate features to make matching easier.
     these include:
       - kernel32.CreateFileA
@@ -47,22 +55,20 @@ def generate_api_features(apiname, va):
       - CreateFileA
       - CreateFile
     """
-    # (kernel32.CreateFileA, 0x401000)
-    yield API(apiname), va
+    # kernel32.CreateFileA
+    yield "%s.%s" % (dll, symbol)
 
-    if is_aw_function(apiname):
-        # (kernel32.CreateFile, 0x401000)
-        yield API(apiname[:-1]), va
+    if not is_ordinal(symbol):
+        # CreateFileA
+        yield symbol
 
-    if "." in apiname:
-        modname, impname = apiname.split(".")
-        # strip modname to support importname-only matching
-        # (CreateFileA, 0x401000)
-        yield API(impname), va
+    if is_aw_function(symbol):
+        # kernel32.CreateFile
+        yield "%s.%s" % (dll, symbol[:-1])
 
-        if is_aw_function(impname):
-            # (CreateFile, 0x401000)
-            yield API(impname[:-1]), va
+        if not is_ordinal(symbol):
+            # CreateFile
+            yield symbol[:-1]
 
 
 def all_zeros(bytez):

capa/features/extractors/ida/file.py

@@ -37,11 +37,11 @@ def check_segment_for_pe(seg):
         )
         for i in range(256)
     ]
-    todo = [
-        (capa.features.extractors.ida.helpers.find_byte_sequence(seg.start_ea, seg.end_ea, mzx), mzx, pex, i)
-        for mzx, pex, i in mz_xor
-    ]
-    todo = [(off, mzx, pex, i) for (off, mzx, pex, i) in todo if off != idaapi.BADADDR]
+
+    todo = []
+    for (mzx, pex, i) in mz_xor:
+        for off in capa.features.extractors.ida.helpers.find_byte_sequence(seg.start_ea, seg.end_ea, mzx):
+            todo.append((off, mzx, pex, i))
 
     while len(todo):
         off, mzx, pex, i = todo.pop()
@@ -61,8 +61,7 @@ def check_segment_for_pe(seg):
         if idc.get_bytes(peoff, 2) == pex:
             yield (off, i)
 
-        nextres = capa.features.extractors.ida.helpers.find_byte_sequence(off + 1, seg.end_ea, mzx)
-        if nextres != -1:
+        for nextres in capa.features.extractors.ida.helpers.find_byte_sequence(off + 1, seg.end_ea, mzx):
             todo.append((nextres, mzx, pex, i))
 
 
@@ -96,11 +95,24 @@ def extract_file_import_names():
      - importname
     """
     for (ea, info) in capa.features.extractors.ida.helpers.get_file_imports().items():
-        if info[1]:
-            yield Import("%s.%s" % (info[0], info[1])), ea
-            yield Import(info[1]), ea
-        if info[2]:
-            yield Import("%s.#%s" % (info[0], str(info[2]))), ea
+        if info[1] and info[2]:
+            # e.g. in mimikatz: ('cabinet', 'FCIAddFile', 11L)
+            # extract by name here and by ordinal below
+            for name in capa.features.extractors.helpers.generate_symbols(info[0], info[1]):
+                yield Import(name), ea
+            dll = info[0]
+            symbol = "#%d" % (info[2])
+        elif info[1]:
+            dll = info[0]
+            symbol = info[1]
+        elif info[2]:
+            dll = info[0]
+            symbol = "#%d" % (info[2])
+        else:
+            continue
+
+        for name in capa.features.extractors.helpers.generate_symbols(dll, symbol):
+            yield Import(name), ea
 
 
 def extract_file_section_names():

capa/features/extractors/ida/helpers.py

@@ -12,20 +12,28 @@ import string
 import idc
 import idaapi
 import idautils
+import ida_bytes
 
 
 def find_byte_sequence(start, end, seq):
-    """find byte sequence
+    """yield all ea of a given byte sequence
 
     args:
         start: min virtual address
         end: max virtual address
-        seq: bytes to search e.g. b'\x01\x03'
+        seq: bytes to search e.g. b"\x01\x03"
     """
     if sys.version_info[0] >= 3:
-        return idaapi.find_binary(start, end, " ".join(["%02x" % b for b in seq]), 0, idaapi.SEARCH_DOWN)
+        seq = " ".join(["%02x" % b for b in seq])
     else:
-        return idaapi.find_binary(start, end, " ".join(["%02x" % ord(b) for b in seq]), 0, idaapi.SEARCH_DOWN)
+        seq = " ".join(["%02x" % ord(b) for b in seq])
+
+    while True:
+        ea = idaapi.find_binary(start, end, seq, 0, idaapi.SEARCH_DOWN)
+        if ea == idaapi.BADADDR:
+            break
+        start = ea + 1
+        yield ea
 
 
 def get_functions(start=None, end=None, skip_thunks=False, skip_libs=False):
@@ -229,6 +237,12 @@ def is_op_read(insn, op):
     return idaapi.has_cf_use(insn.get_canon_feature(), op.n)
 
 
+def is_op_offset(insn, op):
+    """ Check is an operand has been marked as an offset (by auto-analysis or manually) """
+    flags = idaapi.get_flags(insn.ea)
+    return ida_bytes.is_off(flags, op.n)
+
+
 def is_sp_modified(insn):
     """ determine if instruction modifies SP, ESP, RSP """
     for op in get_insn_ops(insn, target_ops=(idaapi.o_reg,)):

capa/features/extractors/ida/insn.py

@@ -12,8 +12,16 @@ import idautils
 
 import capa.features.extractors.helpers
 import capa.features.extractors.ida.helpers
-from capa.features import ARCH_X32, ARCH_X64, MAX_BYTES_FEATURE_SIZE, Bytes, String, Characteristic
-from capa.features.insn import Number, Offset, Mnemonic
+from capa.features import (
+    ARCH_X32,
+    ARCH_X64,
+    MAX_BYTES_FEATURE_SIZE,
+    THUNK_CHAIN_DEPTH_DELTA,
+    Bytes,
+    String,
+    Characteristic,
+)
+from capa.features.insn import API, Number, Offset, Mnemonic
 
 # security cookie checks may perform non-zeroing XORs, these are expected within a certain
 # byte range within the first and returning basic blocks, this helps to reduce FP features
@@ -46,23 +54,34 @@ def get_imports(ctx):
 
 def check_for_api_call(ctx, insn):
     """ check instruction for API call """
-    if not idaapi.is_call_insn(insn):
+    if not insn.get_canon_mnem() in ("call", "jmp"):
         return
 
-    for ref in idautils.CodeRefsFrom(insn.ea, False):
+    info = ()
+    ref = insn.ea
+
+    # attempt to resolve API calls by following chained thunks to a reasonable depth
+    for _ in range(THUNK_CHAIN_DEPTH_DELTA):
+        # assume only one code/data ref when resolving "call" or "jmp"
+        try:
+            ref = tuple(idautils.CodeRefsFrom(ref, False))[0]
+        except IndexError:
+            try:
+                # thunks may be marked as data refs
+                ref = tuple(idautils.DataRefsFrom(ref))[0]
+            except IndexError:
+                break
+
         info = get_imports(ctx).get(ref, ())
         if info:
-            yield "%s.%s" % (info[0], info[1])
-        else:
-            f = idaapi.get_func(ref)
-            # check if call to thunk
-            # TODO: first instruction might not always be the thunk
-            if f and (f.flags & idaapi.FUNC_THUNK):
-                for thunk_ref in idautils.DataRefsFrom(ref):
-                    # TODO: always data ref for thunk??
-                    info = get_imports(ctx).get(thunk_ref, ())
-                    if info:
-                        yield "%s.%s" % (info[0], info[1])
+            break
+
+        f = idaapi.get_func(ref)
+        if not f or not (f.flags & idaapi.FUNC_THUNK):
+            break
+
+    if info:
+        yield "%s.%s" % (info[0], info[1])
 
 
 def extract_insn_api_features(f, bb, insn):
@@ -77,8 +96,9 @@ def extract_insn_api_features(f, bb, insn):
         call dword [0x00473038]
     """
     for api in check_for_api_call(f.ctx, insn):
-        for (feature, ea) in capa.features.extractors.helpers.generate_api_features(api, insn.ea):
-            yield feature, ea
+        dll, _, symbol = api.rpartition(".")
+        for name in capa.features.extractors.helpers.generate_symbols(dll, symbol):
+            yield API(name), insn.ea
 
 
 def extract_insn_number_features(f, bb, insn):
@@ -103,13 +123,18 @@ def extract_insn_number_features(f, bb, insn):
         return
 
     for op in capa.features.extractors.ida.helpers.get_insn_ops(insn, target_ops=(idaapi.o_imm, idaapi.o_mem)):
+        # skip things like:
+        #   .text:00401100 shr eax, offset loc_C
+        if capa.features.extractors.ida.helpers.is_op_offset(insn, op):
+            continue
+
         if op.type == idaapi.o_imm:
             const = capa.features.extractors.ida.helpers.mask_op_val(op)
         else:
             const = op.addr
-        if not idaapi.is_mapped(const):
-            yield Number(const), insn.ea
-            yield Number(const, arch=get_arch(f.ctx)), insn.ea
+
+        yield Number(const), insn.ea
+        yield Number(const, arch=get_arch(f.ctx)), insn.ea
 
 
 def extract_insn_bytes_features(f, bb, insn):

capa/features/extractors/viv/file.py

@@ -8,6 +8,7 @@
 
 import PE.carve as pe_carve  # vivisect PE
 
+import capa.features.extractors.helpers
 import capa.features.extractors.strings
 from capa.features import String, Characteristic
 from capa.features.file import Export, Import, Section
@@ -41,11 +42,9 @@ def extract_file_import_names(vw, file_path):
         if is_viv_ord_impname(impname):
             # replace ord prefix with #
             impname = "#%s" % impname[len("ord") :]
-            tinfo = "%s.%s" % (modname, impname)
-            yield Import(tinfo), va
-        else:
-            yield Import(tinfo), va
-            yield Import(impname), va
+
+        for name in capa.features.extractors.helpers.generate_symbols(modname, impname):
+            yield Import(name), va
 
 
 def is_viv_ord_impname(impname):

capa/features/extractors/viv/helpers.py

@@ -0,0 +1,20 @@
+# Copyright (C) 2020 FireEye, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+from vivisect.const import XR_TO, REF_CODE
+
+
+def get_coderef_from(vw, va):
+    """
+    return first code `tova` whose origin is the specified va
+    return None if no code reference is found
+    """
+    xrefs = vw.getXrefsFrom(va, REF_CODE)
+    if len(xrefs) > 0:
+        return xrefs[0][XR_TO]
+    else:
+        return None

capa/features/extractors/viv/indirect_calls.py

@@ -132,7 +132,7 @@ def is_indirect_call(vw, va, insn=None):
     if insn is None:
         insn = vw.parseOpcode(va)
 
-    return insn.mnem == "call" and isinstance(insn.opers[0], envi.archs.i386.disasm.i386RegOper)
+    return insn.mnem in ("call", "jmp") and isinstance(insn.opers[0], envi.archs.i386.disasm.i386RegOper)
 
 
 def resolve_indirect_call(vw, va, insn=None):

capa/features/extractors/viv/insn.py

@@ -7,12 +7,20 @@
 # See the License for the specific language governing permissions and limitations under the License.
 
 import envi.memory
-import vivisect.const
 import envi.archs.i386.disasm
 
 import capa.features.extractors.helpers
-from capa.features import ARCH_X32, ARCH_X64, MAX_BYTES_FEATURE_SIZE, Bytes, String, Characteristic
-from capa.features.insn import Number, Offset, Mnemonic
+import capa.features.extractors.viv.helpers
+from capa.features import (
+    ARCH_X32,
+    ARCH_X64,
+    MAX_BYTES_FEATURE_SIZE,
+    THUNK_CHAIN_DEPTH_DELTA,
+    Bytes,
+    String,
+    Characteristic,
+)
+from capa.features.insn import API, Number, Offset, Mnemonic
 from capa.features.extractors.viv.indirect_calls import NotFoundError, resolve_indirect_call
 
 # security cookie checks may perform non-zeroing XORs, these are expected within a certain
@@ -47,11 +55,15 @@ def get_imports(vw):
     """
     caching accessor to vivisect workspace imports
     avoids performance issues in vivisect when collecting locations
+
+    returns: Dict[int, Tuple[str, str]]
     """
     if "imports" in vw.metadata:
         return vw.metadata["imports"]
     else:
-        imports = {p[0]: p[3] for p in vw.getImports()}
+        imports = {
+            p[0]: (p[3].rpartition(".")[0], p[3].replace(".ord", ".#").rpartition(".")[2]) for p in vw.getImports()
+        }
         vw.metadata["imports"] = imports
         return imports
 
@@ -63,35 +75,51 @@ def extract_insn_api_features(f, bb, insn):
     #
     #    call dword [0x00473038]
 
-    if insn.mnem != "call":
+    if insn.mnem not in ("call", "jmp"):
         return
 
+    if insn.mnem == "jmp":
+        if f.vw.getFunctionMeta(f.va, "Thunk"):
+            return
+
     # traditional call via IAT
     if isinstance(insn.opers[0], envi.archs.i386.disasm.i386ImmMemOper):
         oper = insn.opers[0]
         target = oper.getOperAddr(insn)
 
         imports = get_imports(f.vw)
-        if target in imports.keys():
-            for feature, va in capa.features.extractors.helpers.generate_api_features(imports[target], insn.va):
-                yield feature, va
+        if target in imports:
+            dll, symbol = imports[target]
+            for name in capa.features.extractors.helpers.generate_symbols(dll, symbol):
+                yield API(name), insn.va
 
     # call via thunk on x86,
     # see 9324d1a8ae37a36ae560c37448c9705a at 0x407985
     #
     # this is also how calls to internal functions may be decoded on x64.
     # see Lab21-01.exe_:0x140001178
-    elif isinstance(insn.opers[0], envi.archs.i386.disasm.i386PcRelOper):
-        target = insn.opers[0].getOperValue(insn)
+    #
+    # follow chained thunks, e.g. in 82bf6347acf15e5d883715dc289d8a2b at 0x14005E0FF in
+    # 0x140059342 (viv) / 0x14005E0C0 (IDA)
+    # 14005E0FF call    j_ElfClearEventLogFileW (14005AAF8)
+    #   14005AAF8 jmp     ElfClearEventLogFileW (14005E196)
+    #     14005E196 jmp     cs:__imp_ElfClearEventLogFileW
 
-        try:
-            thunk = f.vw.getFunctionMeta(target, "Thunk")
-        except vivisect.exc.InvalidFunction:
+    elif isinstance(insn.opers[0], envi.archs.i386.disasm.i386PcRelOper):
+        imports = get_imports(f.vw)
+        target = capa.features.extractors.viv.helpers.get_coderef_from(f.vw, insn.va)
+        if not target:
             return
-        else:
-            if thunk:
-                for feature, va in capa.features.extractors.helpers.generate_api_features(thunk, insn.va):
-                    yield feature, va
+
+        for _ in range(THUNK_CHAIN_DEPTH_DELTA):
+            if target in imports:
+                dll, symbol = imports[target]
+                for name in capa.features.extractors.helpers.generate_symbols(dll, symbol):
+                    yield API(name), insn.va
+
+            target = capa.features.extractors.viv.helpers.get_coderef_from(f.vw, target)
+            if not target:
+                return
 
     # call via import on x64
     # see Lab21-01.exe_:0x14000118C
@@ -100,9 +128,10 @@ def extract_insn_api_features(f, bb, insn):
         target = op.getOperAddr(insn)
 
         imports = get_imports(f.vw)
-        if target in imports.keys():
-            for feature, va in capa.features.extractors.helpers.generate_api_features(imports[target], insn.va):
-                yield feature, va
+        if target in imports:
+            dll, symbol = imports[target]
+            for name in capa.features.extractors.helpers.generate_symbols(dll, symbol):
+                yield API(name), insn.va
 
     elif isinstance(insn.opers[0], envi.archs.i386.disasm.i386RegOper):
         try:
@@ -116,9 +145,10 @@ def extract_insn_api_features(f, bb, insn):
             return
 
         imports = get_imports(f.vw)
-        if target in imports.keys():
-            for feature, va in capa.features.extractors.helpers.generate_api_features(imports[target], insn.va):
-                yield feature, va
+        if target in imports:
+            dll, symbol = imports[target]
+            for name in capa.features.extractors.helpers.generate_symbols(dll, symbol):
+                yield API(name), insn.va
 
 
 def extract_insn_number_features(f, bb, insn):
@@ -318,25 +348,38 @@ def extract_insn_offset_features(f, bb, insn):
     #
     #     .text:0040112F    cmp     [esi+4], ebx
     for oper in insn.opers:
+
         # this is for both x32 and x64
-        if not isinstance(oper, envi.archs.i386.disasm.i386RegMemOper):
-            continue
+        # like [esi + 4]
+        #       reg   ^
+        #             disp
+        if isinstance(oper, envi.archs.i386.disasm.i386RegMemOper):
+            if oper.reg == envi.archs.i386.disasm.REG_ESP:
+                continue
 
-        if oper.reg == envi.archs.i386.disasm.REG_ESP:
-            continue
+            if oper.reg == envi.archs.i386.disasm.REG_EBP:
+                continue
 
-        if oper.reg == envi.archs.i386.disasm.REG_EBP:
-            continue
+            # TODO: do x64 support for real.
+            if oper.reg == envi.archs.amd64.disasm.REG_RBP:
+                continue
 
-        # TODO: do x64 support for real.
-        if oper.reg == envi.archs.amd64.disasm.REG_RBP:
-            continue
+            # viv already decodes offsets as signed
+            v = oper.disp
 
-        # viv already decodes offsets as signed
-        v = oper.disp
+            yield Offset(v), insn.va
+            yield Offset(v, arch=get_arch(f.vw)), insn.va
 
-        yield Offset(v), insn.va
-        yield Offset(v, arch=get_arch(f.vw)), insn.va
+        # like: [esi + ecx + 16384]
+        #        reg   ^     ^
+        #              index ^
+        #                    disp
+        elif isinstance(oper, envi.archs.i386.disasm.i386SibOper):
+            # viv already decodes offsets as signed
+            v = oper.disp
+
+            yield Offset(v), insn.va
+            yield Offset(v, arch=get_arch(f.vw)), insn.va
 
 
 def is_security_cookie(f, bb, insn):

capa/features/insn.py

@@ -13,7 +13,7 @@ class API(Feature):
     def __init__(self, name, description=None):
         # Downcase library name if given
         if "." in name:
-            modname, impname = name.split(".")
+            modname, _, impname = name.rpartition(".")
             name = modname.lower() + "." + impname
 
         super(API, self).__init__(name, description)

capa/ida/explorer/proxy.py

@@ -1,112 +0,0 @@
-# Copyright (C) 2020 FireEye, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-
-from PyQt5 import QtCore
-
-from capa.ida.explorer.model import CapaExplorerDataModel
-
-
-class CapaExplorerSortFilterProxyModel(QtCore.QSortFilterProxyModel):
-    def __init__(self, parent=None):
-        """ """
-        super(CapaExplorerSortFilterProxyModel, self).__init__(parent)
-
-        self.min_ea = None
-        self.max_ea = None
-
-    def lessThan(self, left, right):
-        """true if the value of the left item is less than value of right item
-
-        @param left: QModelIndex*
-        @param right: QModelIndex*
-
-        @retval True/False
-        """
-        ldata = left.internalPointer().data(left.column())
-        rdata = right.internalPointer().data(right.column())
-
-        if (
-            ldata
-            and rdata
-            and left.column() == CapaExplorerDataModel.COLUMN_INDEX_VIRTUAL_ADDRESS
-            and left.column() == right.column()
-        ):
-            # convert virtual address before compare
-            return int(ldata, 16) < int(rdata, 16)
-        else:
-            # compare as lowercase
-            return ldata.lower() < rdata.lower()
-
-    def filterAcceptsRow(self, row, parent):
-        """true if the item in the row indicated by the given row and parent
-        should be included in the model; otherwise returns false
-
-        @param row: int
-        @param parent: QModelIndex*
-
-        @retval True/False
-        """
-        if self.filter_accepts_row_self(row, parent):
-            return True
-
-        alpha = parent
-        while alpha.isValid():
-            if self.filter_accepts_row_self(alpha.row(), alpha.parent()):
-                return True
-            alpha = alpha.parent()
-
-        if self.index_has_accepted_children(row, parent):
-            return True
-
-        return False
-
-    def index_has_accepted_children(self, row, parent):
-        """ """
-        model_index = self.sourceModel().index(row, 0, parent)
-
-        if model_index.isValid():
-            for idx in range(self.sourceModel().rowCount(model_index)):
-                if self.filter_accepts_row_self(idx, model_index):
-                    return True
-                if self.index_has_accepted_children(idx, model_index):
-                    return True
-
-        return False
-
-    def filter_accepts_row_self(self, row, parent):
-        """ """
-        # filter not set
-        if self.min_ea is None and self.max_ea is None:
-            return True
-
-        index = self.sourceModel().index(row, 0, parent)
-        data = index.internalPointer().data(CapaExplorerDataModel.COLUMN_INDEX_VIRTUAL_ADDRESS)
-
-        if not data:
-            return False
-
-        ea = int(data, 16)
-
-        if self.min_ea <= ea and ea < self.max_ea:
-            return True
-
-        return False
-
-    def add_address_range_filter(self, min_ea, max_ea):
-        """ """
-        self.min_ea = min_ea
-        self.max_ea = max_ea
-
-        self.setFilterKeyColumn(CapaExplorerDataModel.COLUMN_INDEX_VIRTUAL_ADDRESS)
-        self.invalidateFilter()
-
-    def reset_address_range_filter(self):
-        """ """
-        self.min_ea = None
-        self.max_ea = None
-        self.invalidateFilter()

capa/ida/helpers/__init__.py

@@ -46,7 +46,6 @@ def is_supported_ida_version():
         logger.warning(
             "Your IDA Pro version is: %s. Supported versions are: %s." % (version, ", ".join(SUPPORTED_IDA_VERSIONS))
         )
-        capa.ida.helpers.inform_user_ida_ui(warning_msg)
         return False
     return True
 
@@ -62,7 +61,6 @@ def is_supported_file_type():
         )
         logger.error(" If you don't know the input file type, you can try using the `file` utility to guess it.")
         logger.error("-" * 80)
-        inform_user_ida_ui("capa does not support the format of this file")
         return False
     return True
 

capa/ida/ida_capa_explorer.py

@@ -1,571 +0,0 @@
-# Copyright (C) 2020 FireEye, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-
-import os
-import json
-import logging
-import collections
-
-import idaapi
-from PyQt5 import QtGui, QtCore, QtWidgets
-
-import capa.main
-import capa.rules
-import capa.ida.helpers
-import capa.render.utils as rutils
-import capa.features.extractors.ida
-from capa.ida.explorer.view import CapaExplorerQtreeView
-from capa.ida.explorer.model import CapaExplorerDataModel
-from capa.ida.explorer.proxy import CapaExplorerSortFilterProxyModel
-
-PLUGIN_NAME = "capa explorer"
-
-logger = logging.getLogger("capa")
-
-
-class CapaExplorerIdaHooks(idaapi.UI_Hooks):
-    def __init__(self, screen_ea_changed_hook, action_hooks):
-        """facilitate IDA UI hooks
-
-        @param screen_ea_changed_hook: function hook for IDA screen ea changed
-        @param action_hooks: dict of IDA action handles
-        """
-        super(CapaExplorerIdaHooks, self).__init__()
-
-        self.screen_ea_changed_hook = screen_ea_changed_hook
-        self.process_action_hooks = action_hooks
-        self.process_action_handle = None
-        self.process_action_meta = {}
-
-    def preprocess_action(self, name):
-        """called prior to action completed
-
-        @param name: name of action defined by idagui.cfg
-
-        @retval must be 0
-        """
-        self.process_action_handle = self.process_action_hooks.get(name, None)
-
-        if self.process_action_handle:
-            self.process_action_handle(self.process_action_meta)
-
-        # must return 0 for IDA
-        return 0
-
-    def postprocess_action(self):
-        """ called after action completed """
-        if not self.process_action_handle:
-            return
-
-        self.process_action_handle(self.process_action_meta, post=True)
-        self.reset()
-
-    def screen_ea_changed(self, curr_ea, prev_ea):
-        """called after screen location is changed
-
-        @param curr_ea: current location
-        @param prev_ea: prev location
-        """
-        self.screen_ea_changed_hook(idaapi.get_current_widget(), curr_ea, prev_ea)
-
-    def reset(self):
-        """ reset internal state """
-        self.process_action_handle = None
-        self.process_action_meta.clear()
-
-
-class CapaExplorerForm(idaapi.PluginForm):
-    def __init__(self):
-        """ """
-        super(CapaExplorerForm, self).__init__()
-
-        self.form_title = PLUGIN_NAME
-        self.file_loc = __file__
-
-        self.parent = None
-        self.ida_hooks = None
-        self.doc = None
-
-        # models
-        self.model_data = None
-        self.model_proxy = None
-
-        # user interface elements
-        self.view_limit_results_by_function = None
-        self.view_tree = None
-        self.view_summary = None
-        self.view_attack = None
-        self.view_tabs = None
-        self.view_menu_bar = None
-
-    def OnCreate(self, form):
-        """ """
-        self.parent = self.FormToPyQtWidget(form)
-        self.load_interface()
-        self.load_capa_results()
-        self.load_ida_hooks()
-
-        self.view_tree.reset()
-
-        logger.info("form created.")
-
-    def Show(self):
-        """ """
-        return idaapi.PluginForm.Show(
-            self, self.form_title, options=(idaapi.PluginForm.WOPN_TAB | idaapi.PluginForm.WCLS_CLOSE_LATER)
-        )
-
-    def OnClose(self, form):
-        """ form is closed """
-        self.unload_ida_hooks()
-        self.ida_reset()
-
-        logger.info("form closed.")
-
-    def load_interface(self):
-        """ load user interface """
-        # load models
-        self.model_data = CapaExplorerDataModel()
-        self.model_proxy = CapaExplorerSortFilterProxyModel()
-        self.model_proxy.setSourceModel(self.model_data)
-
-        # load tree
-        self.view_tree = CapaExplorerQtreeView(self.model_proxy, self.parent)
-
-        # load summary table
-        self.load_view_summary()
-        self.load_view_attack()
-
-        # load parent tab and children tab views
-        self.load_view_tabs()
-        self.load_view_checkbox_limit_by()
-        self.load_view_summary_tab()
-        self.load_view_attack_tab()
-        self.load_view_tree_tab()
-
-        # load menu bar and sub menus
-        self.load_view_menu_bar()
-        self.load_file_menu()
-
-        # load parent view
-        self.load_view_parent()
-
-    def load_view_tabs(self):
-        """ load tabs """
-        tabs = QtWidgets.QTabWidget()
-        self.view_tabs = tabs
-
-    def load_view_menu_bar(self):
-        """ load menu bar """
-        bar = QtWidgets.QMenuBar()
-        self.view_menu_bar = bar
-
-    def load_view_summary(self):
-        """ load capa summary table """
-        table_headers = [
-            "Capability",
-            "Namespace",
-        ]
-
-        table = QtWidgets.QTableWidget()
-
-        table.setColumnCount(len(table_headers))
-        table.verticalHeader().setVisible(False)
-        table.setSortingEnabled(False)
-        table.setEditTriggers(QtWidgets.QAbstractItemView.NoEditTriggers)
-        table.setFocusPolicy(QtCore.Qt.NoFocus)
-        table.setSelectionMode(QtWidgets.QAbstractItemView.NoSelection)
-        table.setHorizontalHeaderLabels(table_headers)
-        table.horizontalHeader().setDefaultAlignment(QtCore.Qt.AlignLeft)
-        table.setShowGrid(False)
-        table.setStyleSheet("QTableWidget::item { padding: 25px; }")
-
-        self.view_summary = table
-
-    def load_view_attack(self):
-        """ load MITRE ATT&CK table """
-        table_headers = [
-            "ATT&CK Tactic",
-            "ATT&CK Technique ",
-        ]
-
-        table = QtWidgets.QTableWidget()
-
-        table.setColumnCount(len(table_headers))
-        table.verticalHeader().setVisible(False)
-        table.setSortingEnabled(False)
-        table.setEditTriggers(QtWidgets.QAbstractItemView.NoEditTriggers)
-        table.setFocusPolicy(QtCore.Qt.NoFocus)
-        table.setSelectionMode(QtWidgets.QAbstractItemView.NoSelection)
-        table.setHorizontalHeaderLabels(table_headers)
-        table.horizontalHeader().setDefaultAlignment(QtCore.Qt.AlignLeft)
-        table.setShowGrid(False)
-        table.setStyleSheet("QTableWidget::item { padding: 25px; }")
-
-        self.view_attack = table
-
-    def load_view_checkbox_limit_by(self):
-        """ load limit results by function checkbox """
-        check = QtWidgets.QCheckBox("Limit results to current function")
-        check.setChecked(False)
-        check.stateChanged.connect(self.slot_checkbox_limit_by_changed)
-
-        self.view_limit_results_by_function = check
-
-    def load_view_parent(self):
-        """ load view parent """
-        layout = QtWidgets.QVBoxLayout()
-
-        layout.addWidget(self.view_tabs)
-        layout.setMenuBar(self.view_menu_bar)
-
-        self.parent.setLayout(layout)
-
-    def load_view_tree_tab(self):
-        """ load capa tree tab view """
-        layout = QtWidgets.QVBoxLayout()
-        layout.addWidget(self.view_limit_results_by_function)
-        layout.addWidget(self.view_tree)
-
-        tab = QtWidgets.QWidget()
-        tab.setLayout(layout)
-
-        self.view_tabs.addTab(tab, "Tree View")
-
-    def load_view_summary_tab(self):
-        """ load capa summary tab view """
-        layout = QtWidgets.QVBoxLayout()
-        layout.addWidget(self.view_summary)
-
-        tab = QtWidgets.QWidget()
-        tab.setLayout(layout)
-
-        self.view_tabs.addTab(tab, "Summary")
-
-    def load_view_attack_tab(self):
-        """ load MITRE ATT&CK tab view """
-        layout = QtWidgets.QVBoxLayout()
-        layout.addWidget(self.view_attack)
-
-        tab = QtWidgets.QWidget()
-        tab.setLayout(layout)
-
-        self.view_tabs.addTab(tab, "MITRE")
-
-    def load_file_menu(self):
-        """ load file menu actions """
-        actions = (
-            ("Reset view", "Reset plugin view", self.reset),
-            ("Run analysis", "Run capa analysis on current database", self.reload),
-            ("Export results...", "Export capa results as JSON file", self.export_json),
-        )
-
-        menu = self.view_menu_bar.addMenu("File")
-        for (name, _, handle) in actions:
-            action = QtWidgets.QAction(name, self.parent)
-            action.triggered.connect(handle)
-            menu.addAction(action)
-
-    def export_json(self):
-        """ export capa results as JSON file """
-        if not self.doc:
-            idaapi.info("No capa results to export.")
-            return
-        path = idaapi.ask_file(True, "*.json", "Choose file")
-        if os.path.exists(path) and 1 != idaapi.ask_yn(1, "File already exists. Overwrite?"):
-            return
-        with open(path, "wb") as export_file:
-            export_file.write(
-                json.dumps(self.doc, sort_keys=True, cls=capa.render.CapaJsonObjectEncoder).encode("utf-8")
-            )
-
-    def load_ida_hooks(self):
-        """ load IDA Pro UI hooks """
-        action_hooks = {
-            "MakeName": self.ida_hook_rename,
-            "EditFunction": self.ida_hook_rename,
-        }
-
-        self.ida_hooks = CapaExplorerIdaHooks(self.ida_hook_screen_ea_changed, action_hooks)
-        self.ida_hooks.hook()
-
-    def unload_ida_hooks(self):
-        """ unload IDA Pro UI hooks """
-        if self.ida_hooks:
-            self.ida_hooks.unhook()
-
-    def ida_hook_rename(self, meta, post=False):
-        """hook for IDA rename action
-
-        called twice, once before action and once after
-        action completes
-
-        @param meta: metadata cache
-        @param post: indicates pre or post action
-        """
-        location = idaapi.get_screen_ea()
-        if not location or not capa.ida.helpers.is_func_start(location):
-            return
-
-        curr_name = idaapi.get_name(location)
-
-        if post:
-            # post action update data model w/ current name
-            self.model_data.update_function_name(meta.get("prev_name", ""), curr_name)
-        else:
-            # pre action so save current name for replacement later
-            meta["prev_name"] = curr_name
-
-    def ida_hook_screen_ea_changed(self, widget, new_ea, old_ea):
-        """hook for IDA screen ea changed
-
-        this hook is currently only relevant for limiting results displayed in the UI
-
-        @param widget: IDA widget type
-        @param new_ea: destination ea
-        @param old_ea: source ea
-        """
-        if not self.view_limit_results_by_function.isChecked():
-            # ignore if limit checkbox not selected
-            return
-
-        if idaapi.get_widget_type(widget) != idaapi.BWN_DISASM:
-            # ignore views not the assembly view
-            return
-
-        if idaapi.get_func(new_ea) == idaapi.get_func(old_ea):
-            # user navigated same function - ignore
-            return
-
-        self.limit_results_to_function(idaapi.get_func(new_ea))
-        self.view_tree.resize_columns_to_content()
-
-    def load_capa_results(self):
-        """ run capa analysis and render results in UI """
-        logger.info("-" * 80)
-        logger.info(" Using default embedded rules.")
-        logger.info(" ")
-        logger.info(" You can see the current default rule set here:")
-        logger.info("     https://github.com/fireeye/capa-rules")
-        logger.info("-" * 80)
-
-        rules_path = os.path.join(os.path.dirname(self.file_loc), "../..", "rules")
-        rules = capa.main.get_rules(rules_path)
-        rules = capa.rules.RuleSet(rules)
-
-        meta = capa.ida.helpers.collect_metadata()
-
-        capabilities, counts = capa.main.find_capabilities(
-            rules, capa.features.extractors.ida.IdaFeatureExtractor(), True
-        )
-        meta["analysis"].update(counts)
-
-        # support binary files specifically for x86/AMD64 shellcode
-        # warn user binary file is loaded but still allow capa to process it
-        # TODO: check specific architecture of binary files based on how user configured IDA processors
-        if idaapi.get_file_type_name() == "Binary file":
-            logger.warning("-" * 80)
-            logger.warning(" Input file appears to be a binary file.")
-            logger.warning(" ")
-            logger.warning(
-                " capa currently only supports analyzing binary files containing x86/AMD64 shellcode with IDA."
-            )
-            logger.warning(
-                " This means the results may be misleading or incomplete if the binary file loaded in IDA is not x86/AMD64."
-            )
-            logger.warning(" If you don't know the input file type, you can try using the `file` utility to guess it.")
-            logger.warning("-" * 80)
-
-            capa.ida.helpers.inform_user_ida_ui("capa encountered warnings during analysis")
-
-        if capa.main.has_file_limitation(rules, capabilities, is_standalone=False):
-            capa.ida.helpers.inform_user_ida_ui("capa encountered warnings during analysis")
-
-        logger.info("analysis completed.")
-
-        self.doc = capa.render.convert_capabilities_to_result_document(meta, rules, capabilities)
-
-        self.model_data.render_capa_doc(self.doc)
-        self.render_capa_doc_summary()
-        self.render_capa_doc_mitre_summary()
-
-        self.set_view_tree_default_sort_order()
-
-        logger.info("render views completed.")
-
-    def set_view_tree_default_sort_order(self):
-        """ set capa tree view default sort order """
-        self.view_tree.sortByColumn(CapaExplorerDataModel.COLUMN_INDEX_RULE_INFORMATION, QtCore.Qt.AscendingOrder)
-
-    def render_capa_doc_summary(self):
-        """ render capa summary results """
-        for (row, rule) in enumerate(rutils.capability_rules(self.doc)):
-            count = len(rule["matches"])
-
-            if count == 1:
-                capability = rule["meta"]["name"]
-            else:
-                capability = "%s (%d matches)" % (rule["meta"]["name"], count)
-
-            self.view_summary.setRowCount(row + 1)
-
-            self.view_summary.setItem(row, 0, self.render_new_table_header_item(capability))
-            self.view_summary.setItem(row, 1, QtWidgets.QTableWidgetItem(rule["meta"]["namespace"]))
-
-        # resize columns to content
-        self.view_summary.resizeColumnsToContents()
-
-    def render_capa_doc_mitre_summary(self):
-        """ render capa MITRE ATT&CK results """
-        tactics = collections.defaultdict(set)
-
-        for rule in rutils.capability_rules(self.doc):
-            if not rule["meta"].get("att&ck"):
-                continue
-
-            for attack in rule["meta"]["att&ck"]:
-                tactic, _, rest = attack.partition("::")
-                if "::" in rest:
-                    technique, _, rest = rest.partition("::")
-                    subtechnique, _, id = rest.rpartition(" ")
-                    tactics[tactic].add((technique, subtechnique, id))
-                else:
-                    technique, _, id = rest.rpartition(" ")
-                    tactics[tactic].add((technique, id))
-
-        column_one = []
-        column_two = []
-
-        for (tactic, techniques) in sorted(tactics.items()):
-            column_one.append(tactic.upper())
-            # add extra space when more than one technique
-            column_one.extend(["" for i in range(len(techniques) - 1)])
-
-            for spec in sorted(techniques):
-                if len(spec) == 2:
-                    technique, id = spec
-                    column_two.append("%s %s" % (technique, id))
-                elif len(spec) == 3:
-                    technique, subtechnique, id = spec
-                    column_two.append("%s::%s %s" % (technique, subtechnique, id))
-                else:
-                    raise RuntimeError("unexpected ATT&CK spec format")
-
-        self.view_attack.setRowCount(max(len(column_one), len(column_two)))
-
-        for row, value in enumerate(column_one):
-            self.view_attack.setItem(row, 0, self.render_new_table_header_item(value))
-
-        for row, value in enumerate(column_two):
-            self.view_attack.setItem(row, 1, QtWidgets.QTableWidgetItem(value))
-
-        # resize columns to content
-        self.view_attack.resizeColumnsToContents()
-
-    def render_new_table_header_item(self, text):
-        """ create new table header item with default style """
-        item = QtWidgets.QTableWidgetItem(text)
-        item.setForeground(QtGui.QColor(88, 139, 174))
-
-        font = QtGui.QFont()
-        font.setBold(True)
-
-        item.setFont(font)
-
-        return item
-
-    def ida_reset(self):
-        """ reset IDA UI """
-        self.model_data.reset()
-        self.view_tree.reset()
-        self.view_limit_results_by_function.setChecked(False)
-        self.set_view_tree_default_sort_order()
-
-    def reload(self):
-        """ reload views and re-run capa analysis """
-        self.ida_reset()
-        self.model_proxy.invalidate()
-        self.model_data.clear()
-        self.view_summary.setRowCount(0)
-        self.load_capa_results()
-
-        logger.info("reload complete.")
-        idaapi.info("%s reload completed." % PLUGIN_NAME)
-
-    def reset(self):
-        """reset UI elements
-
-        e.g. checkboxes and IDA highlighting
-        """
-        self.ida_reset()
-
-        logger.info("reset completed.")
-        idaapi.info("%s reset completed." % PLUGIN_NAME)
-
-    def slot_menu_bar_hovered(self, action):
-        """display menu action tooltip
-
-        @param action: QtWidgets.QAction*
-
-        @reference: https://stackoverflow.com/questions/21725119/why-wont-qtooltips-appear-on-qactions-within-a-qmenu
-        """
-        QtWidgets.QToolTip.showText(
-            QtGui.QCursor.pos(), action.toolTip(), self.view_menu_bar, self.view_menu_bar.actionGeometry(action)
-        )
-
-    def slot_checkbox_limit_by_changed(self):
-        """slot activated if checkbox clicked
-
-        if checked, configure function filter if screen location is located
-        in function, otherwise clear filter
-        """
-        if self.view_limit_results_by_function.isChecked():
-            self.limit_results_to_function(idaapi.get_func(idaapi.get_screen_ea()))
-        else:
-            self.model_proxy.reset_address_range_filter()
-
-        self.view_tree.reset()
-
-    def limit_results_to_function(self, f):
-        """add filter to limit results to current function
-
-        @param f: (IDA func_t)
-        """
-        if f:
-            self.model_proxy.add_address_range_filter(f.start_ea, f.end_ea)
-        else:
-            # if function not exists don't display any results (address should not be -1)
-            self.model_proxy.add_address_range_filter(-1, -1)
-
-
-def main():
-    """ TODO: move to idaapi.plugin_t class """
-    logging.basicConfig(level=logging.INFO)
-
-    if not capa.ida.helpers.is_supported_ida_version():
-        return -1
-
-    if not capa.ida.helpers.is_supported_file_type():
-        return -1
-
-    global CAPA_EXPLORER_FORM
-
-    try:
-        # there is an instance, reload it
-        CAPA_EXPLORER_FORM
-        CAPA_EXPLORER_FORM.Close()
-        CAPA_EXPLORER_FORM = CapaExplorerForm()
-    except Exception:
-        # there is no instance yet
-        CAPA_EXPLORER_FORM = CapaExplorerForm()
-
-    CAPA_EXPLORER_FORM.Show()
-
-
-if __name__ == "__main__":
-    main()

capa/ida/plugin/README.md

@@ -0,0 +1,111 @@
+![capa explorer](../../../.github/capa-explorer-logo.png)
+
+capa explorer is an IDA Pro plugin written in Python that integrates the FLARE team's open-source framework, capa, with IDA. capa is a framework that uses a well-defined collection of rules to 
+identify capabilities in a program. You can run capa against a PE file or shellcode and it tells you what it thinks the program can do. For example, it might suggest that 
+the program is a backdoor, can install services, or relies on HTTP to communicate. You can use capa explorer to run capa directly on an IDA database without requiring access
+to the source binary. Once a database has been analyzed, capa explorer can be used to quickly identify and navigate to interesting areas of a program 
+and dissect capa rule matches at the assembly level.
+
+We love using capa explorer during malware analysis because it teaches us what parts of a program suggest a behavior. As we click on rows, capa explorer jumps directly 
+to important addresses in the IDA Pro database and highlights key features in the Disassembly view so they stand out visually. To illustrate, we use capa explorer to 
+analyze Lab 14-02 from [Practical Malware Analysis](https://nostarch.com/malware) (PMA) available [here](https://practicalmalwareanalysis.com/labs/). Our goal is to understand 
+the program's functionality.
+
+After loading Lab 14-02 into IDA and analyzing the database with capa explorer, we see that capa detected a rule match for `self delete via COMSPEC environment variable`:
+
+![](../../../doc/img/ida_plugin_example_1.png)
+
+We can use capa explorer to navigate the IDA Disassembly view directly to the suspect function and get an assembly-level breakdown of why capa matched `self delete via COMSPEC environment variable` 
+for this particular function.
+
+![](../../../doc/img/ida_plugin_example_2.png)
+
+Using the `Rule Information` and `Details` columns capa explorer shows us that the suspect function matched `self delete via COMSPEC environment variable` because it contains capa rule matches for `create process`, `get COMSPEC environment variable`,
+and `query environment variable`, references to the strings `COMSPEC`, ` > nul`, and `/c del`, and calls to the Windows API functions `GetEnvironmentVariableA` and `ShellExecuteEx`.
+
+For more information on the FLARE team's open-source framework, capa, check out the overview in our first [blog](https://www.fireeye.com/blog/threat-research/2020/07/capa-automatically-identify-malware-capabilities.html).
+
+## Features
+
+![](../../../doc/img/ida_plugin_intro.gif)
+
+* Display capa results in an interactive tree view of rule matches and their locations in the current database
+* Search for keywords or phrases found in the `Rule Information`, `Address`, or `Details` columns
+* Display rule source content when a user hovers their cursor over a rule match
+* Double-click `Address` column to view associated feature in the IDA Disassembly view
+* Limit tree view results to the function currently displayed in the IDA Disassembly view; update results as a user navigates to different functions
+* Export results as formatted JSON by navigating to `File > Export results...`
+* Remember a user's capa rules directory for future runs; change capa rules directory by navigating to `Rules > Change rules directory...`
+* Automatically re-analyze database when user performs a program rebase
+* Automatically update results when IDA is used to rename a function
+* Select one or more checkboxes to highlight the associated addresses in the IDA Disassembly view
+* Right-click a function match to rename it; the new function name is propagated to the current IDA database
+* Right-click to copy a result by column or by row
+* Sort results by column
+* Reset tree view and IDA Disassembly view highlighting by clicking `Reset`
+
+## Getting Started
+
+### Requirements
+
+capa explorer supports the following IDA setups:
+
+* IDA Pro 7.4+ with Python 2.7 or Python 3.
+
+If you encounter issues with your specific setup, please open a new [Issue](https://github.com/fireeye/capa/issues).
+
+### Supported File Types
+
+capa explorer is limited to the file types supported by capa, which includes:
+
+* Windows 32-bit and 64-bit PE files
+* Windows 32-bit and 64-bit shellcode
+
+### Installation
+
+You can install capa explorer using the following steps:
+
+1. Install capa and its dependencies from PyPI for the Python interpreter used by your IDA installation:
+    ```
+    $ pip install flare-capa
+    ```
+3. Download the [standard collection of capa rules](https://github.com/fireeye/capa-rules) (capa explorer needs capa rules to analyze a database)
+4. Copy [capa_explorer.py](https://raw.githubusercontent.com/fireeye/capa/master/capa/ida/plugin/capa_explorer.py) to your IDA plugins directory
+
+### Usage
+
+1. Run IDA and analyze a supported file type (select the `Manual Load` and `Load Resources` options in IDA for best results)
+2. Open capa explorer in IDA by navigating to `Edit > Plugins > FLARE capa explorer` or using the keyboard shortcut `Alt+F5`
+3. Click the `Analyze` button
+
+When running capa explorer for the first time you are prompted to select a file directory containing capa rules. The plugin conveniently
+remembers your selection for future runs; you can change this selection by navigating to `Rules > Change rules directory...`. We recommend 
+downloading and using the [standard collection of capa rules](https://github.com/fireeye/capa-rules) when getting started with the plugin.
+
+#### Tips
+
+* Start analysis by clicking the `Analyze` button
+* Reset the plugin user interface and remove highlighting from IDA disassembly view by clicking the `Reset` button
+* Change your capa rules directory by navigating to `Rules > Change rules directory...` from the plugin menu
+* Hover your cursor over a rule match to view the source content of the rule
+* Double-click the `Address` column to navigate the IDA Disassembly view to the associated feature
+* Double-click a result in the `Rule Information` column to expand its children
+* Select a checkbox in the `Rule Information` column to highlight the address of the associated feature in the IDA Dissasembly view
+
+## Development
+
+Because capa explorer is packaged with capa you will need to install capa locally for development.
+
+You can install capa locally by following the steps outlined in `Method 3: Inspecting the capa source code` of the [capa 
+installation guide](https://github.com/fireeye/capa/blob/master/doc/installation.md#method-3-inspecting-the-capa-source-code). Once installed, copy [capa_explorer.py](https://raw.githubusercontent.com/fireeye/capa/master/capa/ida/plugin/capa_explorer.py) 
+to your IDA plugins directory to run the plugin in IDA.
+
+### Components
+
+capa explorer consists of two main components:
+
+* An IDA [feature extractor](https://github.com/fireeye/capa/tree/master/capa/features/extractors/ida) built on top of IDA's binary analysis engine
+  * This component uses IDAPython to extract [capa features](https://github.com/fireeye/capa-rules/blob/master/doc/format.md#extracted-features) from the IDA database such as strings, 
+disassembly, and control flow; these extracted features are used by capa to find feature combinations that result in a rule match
+* An [interactive user interface](https://github.com/fireeye/capa/tree/master/capa/ida/plugin) for displaying and exploring capa rule matches
+  * This component integrates the IDA feature extractor and capa, providing an interactive user interface to dissect rule matches found by capa using features extracted by the IDA feature extractor

capa/ida/plugin/__init__.py

@@ -0,0 +1,117 @@
+# Copyright (C) 2020 FireEye, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+import logging
+
+import idaapi
+import ida_kernwin
+
+from capa.ida.helpers import is_supported_file_type, is_supported_ida_version
+from capa.ida.plugin.form import CapaExplorerForm
+from capa.ida.plugin.icon import ICON
+
+logger = logging.getLogger(__name__)
+
+
+class CapaExplorerPlugin(idaapi.plugin_t):
+
+    # Mandatory definitions
+    PLUGIN_NAME = "FLARE capa explorer"
+    PLUGIN_VERSION = "1.0.0"
+    PLUGIN_AUTHORS = "michael.hunhoff@mandiant.com, william.ballenthin@mandiant.com, moritz.raabe@mandiant.com"
+
+    wanted_name = PLUGIN_NAME
+    wanted_hotkey = "ALT-F5"
+    comment = "IDA Pro plugin for the FLARE team's capa tool to identify capabilities in executable files."
+    website = "https://github.com/fireeye/capa"
+    help = "See https://github.com/fireeye/capa/blob/master/doc/usage.md"
+    version = ""
+    flags = 0
+
+    def __init__(self):
+        """initialize plugin"""
+        self.form = None
+
+    def init(self):
+        """called when IDA is loading the plugin"""
+        logging.basicConfig(level=logging.INFO)
+
+        # do not load plugin if IDA version/file type not supported
+        if not is_supported_ida_version():
+            return idaapi.PLUGIN_SKIP
+        if not is_supported_file_type():
+            return idaapi.PLUGIN_SKIP
+        return idaapi.PLUGIN_OK
+
+    def term(self):
+        """called when IDA is unloading the plugin"""
+        pass
+
+    def run(self, arg):
+        """called when IDA is running the plugin as a script"""
+        self.form = CapaExplorerForm(self.PLUGIN_NAME)
+        return True
+
+
+# set the capa plugin icon.
+#
+# TL;DR: temporarily install a UI hook set the icon.
+#
+# Long form:
+#
+# in the IDAPython `plugin_t` life cycle,
+#   - `init` decides if a plugin should be registered
+#   - `run` executes the main logic (shows the window)
+#   - `term` cleans this up
+#
+# we want to associate an icon with the plugin action - which is created by IDA.
+# however, this action is created by IDA *after* `init` is called.
+# so, we can't do this in `plugin_t.init`.
+# we also can't spawn a thread and do it after a delay,
+#  since `ida_kernwin.update_action_icon` must be called from the main thread.
+# so we need to register a callback that's invoked from the main thread after the plugin is registered.
+#
+# after a lot of guess-and-check, we can use `UI_Hooks.updated_actions` to
+#  receive notications after IDA has created an action for each plugin.
+# so, create this hook, wait for capa plugin to load, set the icon, and unhook.
+
+
+class OnUpdatedActionsHook(ida_kernwin.UI_Hooks):
+    """register a callback to be invoked each time the UI actions are updated"""
+
+    def __init__(self, cb):
+        super(OnUpdatedActionsHook, self).__init__()
+        self.cb = cb
+
+    def updated_actions(self):
+        if self.cb():
+            # uninstall the callback once its run successfully
+            self.unhook()
+
+
+def install_icon():
+    plugin_name = CapaExplorerPlugin.PLUGIN_NAME
+    action_name = "Edit/Plugins/" + plugin_name
+
+    if action_name not in ida_kernwin.get_registered_actions():
+        # keep the hook registered
+        return False
+
+    # resource leak here. need to call `ida_kernwin.free_custom_icon`?
+    # however, since we're not cycling this icon a lot, its probably ok.
+    # expect to leak exactly one icon per application load.
+    icon = ida_kernwin.load_custom_icon(data=ICON)
+
+    ida_kernwin.update_action_icon(action_name, icon)
+
+    # uninstall the hook
+    return True
+
+
+h = OnUpdatedActionsHook(install_icon)
+h.hook()

capa/ida/plugin/capa_explorer.py

@@ -0,0 +1,17 @@
+# Copyright (C) 2020 FireEye, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+from capa.ida.plugin import CapaExplorerPlugin
+
+
+def PLUGIN_ENTRY():
+    """mandatory entry point for IDAPython plugins
+
+    copy this script to your IDA plugins directory and start the plugin by navigating to Edit > Plugins in IDA Pro
+    """
+    return CapaExplorerPlugin()

capa/ida/plugin/form.py

@@ -0,0 +1,766 @@
+# Copyright (C) 2020 FireEye, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+import os
+import json
+import logging
+import collections
+
+import idaapi
+import ida_kernwin
+import ida_settings
+from PyQt5 import QtGui, QtCore, QtWidgets
+
+import capa.main
+import capa.rules
+import capa.ida.helpers
+import capa.render.utils as rutils
+import capa.features.extractors.ida
+from capa.ida.plugin.icon import QICON
+from capa.ida.plugin.view import CapaExplorerQtreeView
+from capa.ida.plugin.hooks import CapaExplorerIdaHooks
+from capa.ida.plugin.model import CapaExplorerDataModel
+from capa.ida.plugin.proxy import CapaExplorerRangeProxyModel, CapaExplorerSearchProxyModel
+
+logger = logging.getLogger(__name__)
+settings = ida_settings.IDASettings("capa")
+
+
+class UserCancelledError(Exception):
+    """throw exception when user cancels action"""
+
+    pass
+
+
+class CapaExplorerProgressIndicator(QtCore.QObject):
+    """implement progress signal, used during feature extraction"""
+
+    progress = QtCore.pyqtSignal(str)
+
+    def __init__(self):
+        """initialize signal object"""
+        super(CapaExplorerProgressIndicator, self).__init__()
+
+    def update(self, text):
+        """emit progress update
+
+        check if user cancelled action, raise exception for parent function to catch
+        """
+        if ida_kernwin.user_cancelled():
+            raise UserCancelledError("user cancelled")
+        self.progress.emit("extracting features from %s" % text)
+
+
+class CapaExplorerFeatureExtractor(capa.features.extractors.ida.IdaFeatureExtractor):
+    """subclass the IdaFeatureExtractor
+
+    track progress during feature extraction, also allow user to cancel feature extraction
+    """
+
+    def __init__(self):
+        super(CapaExplorerFeatureExtractor, self).__init__()
+        self.indicator = CapaExplorerProgressIndicator()
+
+    def extract_function_features(self, f):
+        self.indicator.update("function at 0x%X" % f.start_ea)
+        return super(CapaExplorerFeatureExtractor, self).extract_function_features(f)
+
+
+class CapaExplorerForm(idaapi.PluginForm):
+    """form element for plugin interface"""
+
+    def __init__(self, name):
+        """initialize form elements"""
+        super(CapaExplorerForm, self).__init__()
+
+        self.form_title = name
+        self.rule_path = ""
+        self.process_total = 0
+        self.process_count = 0
+
+        self.parent = None
+        self.ida_hooks = None
+        self.doc = None
+
+        # models
+        self.model_data = None
+        self.range_model_proxy = None
+        self.search_model_proxy = None
+
+        # UI controls
+        self.view_limit_results_by_function = None
+        self.view_search_bar = None
+        self.view_tree = None
+        self.view_attack = None
+        self.view_tabs = None
+        self.view_menu_bar = None
+        self.view_status_label = None
+        self.view_buttons = None
+        self.view_analyze_button = None
+        self.view_reset_button = None
+
+        self.Show()
+
+    def OnCreate(self, form):
+        """called when plugin form is created
+
+        load interface and install hooks but do not analyze database
+        """
+        self.parent = self.FormToPyQtWidget(form)
+        self.parent.setWindowIcon(QICON)
+        self.load_interface()
+        self.load_ida_hooks()
+
+    def Show(self):
+        """creates form if not already create, else brings plugin to front"""
+        return super(CapaExplorerForm, self).Show(
+            self.form_title,
+            options=(
+                idaapi.PluginForm.WOPN_TAB
+                | idaapi.PluginForm.WOPN_RESTORE
+                | idaapi.PluginForm.WCLS_CLOSE_LATER
+                | idaapi.PluginForm.WCLS_SAVE
+            ),
+        )
+
+    def OnClose(self, form):
+        """called when form is closed
+
+        ensure any plugin modifications (e.g. hooks and UI changes) are reset before the plugin is closed
+        """
+        self.unload_ida_hooks()
+        self.model_data.reset()
+
+    def load_interface(self):
+        """load user interface"""
+        # load models
+        self.model_data = CapaExplorerDataModel()
+
+        # model <- filter range <- filter search <- view
+
+        self.range_model_proxy = CapaExplorerRangeProxyModel()
+        self.range_model_proxy.setSourceModel(self.model_data)
+
+        self.search_model_proxy = CapaExplorerSearchProxyModel()
+        self.search_model_proxy.setSourceModel(self.range_model_proxy)
+
+        self.view_tree = CapaExplorerQtreeView(self.search_model_proxy, self.parent)
+        self.load_view_attack()
+
+        # load parent tab and children tab views
+        self.load_view_tabs()
+        self.load_view_checkbox_limit_by()
+        self.load_view_search_bar()
+        self.load_view_tree_tab()
+        self.load_view_attack_tab()
+        self.load_view_status_label()
+        self.load_view_buttons()
+
+        # load menu bar and sub menus
+        self.load_view_menu_bar()
+        self.load_file_menu()
+        self.load_rules_menu()
+
+        # load parent view
+        self.load_view_parent()
+
+        self.disable_controls()
+
+    def load_view_tabs(self):
+        """load tabs"""
+        tabs = QtWidgets.QTabWidget()
+        self.view_tabs = tabs
+
+    def load_view_menu_bar(self):
+        """load menu bar"""
+        bar = QtWidgets.QMenuBar()
+        self.view_menu_bar = bar
+
+    def load_view_attack(self):
+        """load MITRE ATT&CK table"""
+        table_headers = [
+            "ATT&CK Tactic",
+            "ATT&CK Technique ",
+        ]
+
+        table = QtWidgets.QTableWidget()
+
+        table.setColumnCount(len(table_headers))
+        table.verticalHeader().setVisible(False)
+        table.setSortingEnabled(False)
+        table.setEditTriggers(QtWidgets.QAbstractItemView.NoEditTriggers)
+        table.setFocusPolicy(QtCore.Qt.NoFocus)
+        table.setSelectionMode(QtWidgets.QAbstractItemView.NoSelection)
+        table.setHorizontalHeaderLabels(table_headers)
+        table.horizontalHeader().setDefaultAlignment(QtCore.Qt.AlignLeft)
+        table.setShowGrid(False)
+        table.setStyleSheet("QTableWidget::item { padding: 25px; }")
+
+        self.view_attack = table
+
+    def load_view_checkbox_limit_by(self):
+        """load limit results by function checkbox"""
+        check = QtWidgets.QCheckBox("Limit results to current function")
+        check.setChecked(False)
+        check.stateChanged.connect(self.slot_checkbox_limit_by_changed)
+
+        self.view_limit_results_by_function = check
+
+    def load_view_status_label(self):
+        """load status label"""
+        label = QtWidgets.QLabel()
+        label.setAlignment(QtCore.Qt.AlignLeft)
+        label.setText("Click Analyze to get started...")
+
+        self.view_status_label = label
+
+    def load_view_buttons(self):
+        """load the button controls"""
+        analyze_button = QtWidgets.QPushButton("Analyze")
+        analyze_button.setToolTip("Run capa analysis on IDB")
+        reset_button = QtWidgets.QPushButton("Reset")
+        reset_button.setToolTip("Reset capa explorer and IDA user interfaces")
+
+        analyze_button.clicked.connect(self.slot_analyze)
+        reset_button.clicked.connect(self.slot_reset)
+
+        layout = QtWidgets.QHBoxLayout()
+        layout.addWidget(analyze_button)
+        layout.addWidget(reset_button)
+        layout.addStretch(1)
+
+        self.view_analyze_button = analyze_button
+        self.view_reset_button = reset_button
+        self.view_buttons = layout
+
+    def load_view_search_bar(self):
+        """load the search bar control"""
+        line = QtWidgets.QLineEdit()
+        line.setPlaceholderText("search...")
+        line.textChanged.connect(self.slot_limit_results_to_search)
+
+        self.view_search_bar = line
+
+    def load_view_parent(self):
+        """load view parent"""
+        layout = QtWidgets.QVBoxLayout()
+
+        layout.addWidget(self.view_tabs)
+        layout.addWidget(self.view_status_label)
+        layout.addLayout(self.view_buttons)
+        layout.setMenuBar(self.view_menu_bar)
+
+        self.parent.setLayout(layout)
+
+    def load_view_tree_tab(self):
+        """load tree view tab"""
+        layout = QtWidgets.QVBoxLayout()
+        layout.addWidget(self.view_limit_results_by_function)
+        layout.addWidget(self.view_search_bar)
+        layout.addWidget(self.view_tree)
+
+        tab = QtWidgets.QWidget()
+        tab.setLayout(layout)
+
+        self.view_tabs.addTab(tab, "Tree View")
+
+    def load_view_attack_tab(self):
+        """load MITRE ATT&CK view tab"""
+        layout = QtWidgets.QVBoxLayout()
+        layout.addWidget(self.view_attack)
+
+        tab = QtWidgets.QWidget()
+        tab.setLayout(layout)
+
+        self.view_tabs.addTab(tab, "MITRE")
+
+    def load_file_menu(self):
+        """load file menu controls"""
+        actions = (("Export results...", "Export capa results as JSON file", self.slot_export_json),)
+        self.load_menu("File", actions)
+
+    def load_rules_menu(self):
+        """load rules menu controls"""
+        actions = (("Change rules directory...", "Select new rules directory", self.slot_change_rules_dir),)
+        self.load_menu("Rules", actions)
+
+    def load_menu(self, title, actions):
+        """load menu actions
+
+        @param title: menu name displayed in UI
+        @param actions: tuple of tuples containing action name, tooltip, and slot function
+        """
+        menu = self.view_menu_bar.addMenu(title)
+        for (name, _, slot) in actions:
+            action = QtWidgets.QAction(name, self.parent)
+            action.triggered.connect(slot)
+            menu.addAction(action)
+
+    def slot_export_json(self):
+        """export capa results as JSON file"""
+        if not self.doc:
+            idaapi.info("No capa results to export.")
+            return
+
+        path = idaapi.ask_file(True, "*.json", "Choose file")
+
+        # user cancelled, entered blank input, etc.
+        if not path:
+            return
+
+        # check file exists, ask to override
+        if os.path.exists(path) and 1 != idaapi.ask_yn(1, "The selected file already exists. Overwrite?"):
+            return
+
+        with open(path, "wb") as export_file:
+            export_file.write(
+                json.dumps(self.doc, sort_keys=True, cls=capa.render.CapaJsonObjectEncoder).encode("utf-8")
+            )
+
+    def load_ida_hooks(self):
+        """load IDA UI hooks"""
+        # map named action (defined in idagui.cfg) to Python function
+        action_hooks = {
+            "MakeName": self.ida_hook_rename,
+            "EditFunction": self.ida_hook_rename,
+            "RebaseProgram": self.ida_hook_rebase,
+        }
+
+        self.ida_hooks = CapaExplorerIdaHooks(self.ida_hook_screen_ea_changed, action_hooks)
+        self.ida_hooks.hook()
+
+    def unload_ida_hooks(self):
+        """unload IDA Pro UI hooks
+
+        must be called before plugin is completely destroyed
+        """
+        if self.ida_hooks:
+            self.ida_hooks.unhook()
+
+    def ida_hook_rename(self, meta, post=False):
+        """function hook for IDA "MakeName" and "EditFunction" actions
+
+        called twice, once before action and once after action completes
+
+        @param meta: dict of key/value pairs set when action first called (may be empty)
+        @param post: False if action first call, True if action second call
+        """
+        location = idaapi.get_screen_ea()
+        if not location or not capa.ida.helpers.is_func_start(location):
+            return
+
+        curr_name = idaapi.get_name(location)
+
+        if post:
+            # post action update data model w/ current name
+            self.model_data.update_function_name(meta.get("prev_name", ""), curr_name)
+        else:
+            # pre action so save current name for replacement later
+            meta["prev_name"] = curr_name
+
+    def ida_hook_screen_ea_changed(self, widget, new_ea, old_ea):
+        """function hook for IDA "screen ea changed" action
+
+        called twice, once before action and once after action completes. this hook is currently only relevant
+        for limiting results displayed in the UI
+
+        @param widget: IDA widget type
+        @param new_ea: destination ea
+        @param old_ea: source ea
+        """
+        if not self.view_limit_results_by_function.isChecked():
+            # ignore if limit checkbox not selected
+            return
+
+        if idaapi.get_widget_type(widget) != idaapi.BWN_DISASM:
+            # ignore views not the assembly view
+            return
+
+        if idaapi.get_func(new_ea) == idaapi.get_func(old_ea):
+            # user navigated same function - ignore
+            return
+
+        self.limit_results_to_function(idaapi.get_func(new_ea))
+        self.view_tree.reset_ui()
+
+    def ida_hook_rebase(self, meta, post=False):
+        """function hook for IDA "RebaseProgram" action
+
+        called twice, once before action and once after action completes
+
+        @param meta: dict of key/value pairs set when action first called (may be empty)
+        @param post: False if action first call, True if action second call
+        """
+        if post:
+            if idaapi.get_imagebase() != meta.get("prev_base", -1):
+                capa.ida.helpers.inform_user_ida_ui("Running capa analysis again after program rebase")
+                self.slot_analyze()
+        else:
+            meta["prev_base"] = idaapi.get_imagebase()
+            self.model_data.reset()
+
+    def load_capa_results(self):
+        """run capa analysis and render results in UI
+
+        note: this function must always return, exception or not, in order for plugin to safely close the IDA
+        wait box
+        """
+        # new analysis, new doc
+        self.doc = None
+        self.process_total = 0
+        self.process_count = 1
+
+        def update_wait_box(text):
+            """update the IDA wait box"""
+            ida_kernwin.replace_wait_box("capa explorer...%s" % text)
+
+        def slot_progress_feature_extraction(text):
+            """slot function to handle feature extraction progress updates"""
+            update_wait_box("%s (%d of %d)" % (text, self.process_count, self.process_total))
+            self.process_count += 1
+
+        extractor = CapaExplorerFeatureExtractor()
+        extractor.indicator.progress.connect(slot_progress_feature_extraction)
+
+        update_wait_box("calculating analysis")
+
+        try:
+            self.process_total += len(tuple(extractor.get_functions()))
+        except Exception as e:
+            logger.error("Failed to calculate analysis (error: %s).", e)
+            return False
+
+        if ida_kernwin.user_cancelled():
+            logger.info("User cancelled analysis.")
+            return False
+
+        update_wait_box("loading rules")
+
+        try:
+            # resolve rules directory - check self and settings first, then ask user
+            if not self.rule_path:
+                if "rule_path" in settings and os.path.exists(settings["rule_path"]):
+                    self.rule_path = settings["rule_path"]
+                else:
+                    idaapi.info("Please select a file directory containing capa rules.")
+                    rule_path = self.ask_user_directory()
+                    if not rule_path:
+                        logger.warning(
+                            "You must select a file directory containing capa rules before analysis can be run. The standard collection of capa rules can be downloaded from https://github.com/fireeye/capa-rules."
+                        )
+                        return False
+                    self.rule_path = rule_path
+                    settings.user["rule_path"] = rule_path
+        except Exception as e:
+            logger.error("Failed to load capa rules (error: %s).", e)
+            return False
+
+        if ida_kernwin.user_cancelled():
+            logger.info("User cancelled analysis.")
+            return False
+
+        rule_path = self.rule_path
+
+        try:
+            if not os.path.exists(rule_path):
+                raise IOError("rule path %s does not exist or cannot be accessed" % rule_path)
+
+            rule_paths = []
+            if os.path.isfile(rule_path):
+                rule_paths.append(rule_path)
+            elif os.path.isdir(rule_path):
+                for root, dirs, files in os.walk(rule_path):
+                    if ".github" in root:
+                        # the .github directory contains CI config in capa-rules
+                        # this includes some .yml files
+                        # these are not rules
+                        continue
+                    for file in files:
+                        if not file.endswith(".yml"):
+                            if not (file.endswith(".md") or file.endswith(".git") or file.endswith(".txt")):
+                                # expect to see readme.md, format.md, and maybe a .git directory
+                                # other things maybe are rules, but are mis-named.
+                                logger.warning("skipping non-.yml file: %s", file)
+                            continue
+                        rule_path = os.path.join(root, file)
+                        rule_paths.append(rule_path)
+
+            rules = []
+            total_paths = len(rule_paths)
+            for (i, rule_path) in enumerate(rule_paths):
+                update_wait_box("loading capa rules from %s (%d of %d)" % (self.rule_path, i + 1, total_paths))
+                if ida_kernwin.user_cancelled():
+                    raise UserCancelledError("user cancelled")
+                try:
+                    rule = capa.rules.Rule.from_yaml_file(rule_path)
+                except capa.rules.InvalidRule:
+                    raise
+                else:
+                    rule.meta["capa/path"] = rule_path
+                    if capa.main.is_nursery_rule_path(rule_path):
+                        rule.meta["capa/nursery"] = True
+                    rules.append(rule)
+
+            rule_count = len(rules)
+            rules = capa.rules.RuleSet(rules)
+        except UserCancelledError:
+            logger.info("User cancelled analysis.")
+            return False
+        except Exception as e:
+            capa.ida.helpers.inform_user_ida_ui("Failed to load capa rules from %s" % self.rule_path)
+            logger.error("Failed to load rules from %s (error: %s).", self.rule_path, e)
+            logger.error(
+                "Make sure your file directory contains properly formatted capa rules. You can download the standard collection of capa rules from https://github.com/fireeye/capa-rules."
+            )
+            self.rule_path = ""
+            settings.user.del_value("rule_path")
+            return False
+
+        if ida_kernwin.user_cancelled():
+            logger.info("User cancelled analysis.")
+            return False
+
+        update_wait_box("extracting features")
+
+        try:
+            meta = capa.ida.helpers.collect_metadata()
+            capabilities, counts = capa.main.find_capabilities(rules, extractor, disable_progress=True)
+            meta["analysis"].update(counts)
+        except UserCancelledError:
+            logger.info("User cancelled analysis.")
+            return False
+        except Exception as e:
+            logger.error("Failed to extract capabilities from database (error: %s)", e)
+            return False
+
+        update_wait_box("checking for file limitations")
+
+        try:
+            # support binary files specifically for x86/AMD64 shellcode
+            # warn user binary file is loaded but still allow capa to process it
+            # TODO: check specific architecture of binary files based on how user configured IDA processors
+            if idaapi.get_file_type_name() == "Binary file":
+                logger.warning("-" * 80)
+                logger.warning(" Input file appears to be a binary file.")
+                logger.warning(" ")
+                logger.warning(
+                    " capa currently only supports analyzing binary files containing x86/AMD64 shellcode with IDA."
+                )
+                logger.warning(
+                    " This means the results may be misleading or incomplete if the binary file loaded in IDA is not x86/AMD64."
+                )
+                logger.warning(
+                    " If you don't know the input file type, you can try using the `file` utility to guess it."
+                )
+                logger.warning("-" * 80)
+
+                capa.ida.helpers.inform_user_ida_ui("capa encountered file type warnings during analysis")
+
+            if capa.main.has_file_limitation(rules, capabilities, is_standalone=False):
+                capa.ida.helpers.inform_user_ida_ui("capa encountered file limitation warnings during analysis")
+        except Exception as e:
+            logger.error("Failed to check for file limitations (error: %s)", e)
+            return False
+
+        if ida_kernwin.user_cancelled():
+            logger.info("User cancelled analysis.")
+            return False
+
+        update_wait_box("rendering results")
+
+        try:
+            self.doc = capa.render.convert_capabilities_to_result_document(meta, rules, capabilities)
+            self.model_data.render_capa_doc(self.doc)
+            self.render_capa_doc_mitre_summary()
+            self.enable_controls()
+            self.set_view_status_label("capa rules directory: %s (%d rules)" % (self.rule_path, rule_count))
+        except Exception as e:
+            logger.error("Failed to render results (error: %s)", e)
+            return False
+
+        return True
+
+    def render_capa_doc_mitre_summary(self):
+        """render MITRE ATT&CK results"""
+        tactics = collections.defaultdict(set)
+
+        for rule in rutils.capability_rules(self.doc):
+            if not rule["meta"].get("att&ck"):
+                continue
+
+            for attack in rule["meta"]["att&ck"]:
+                tactic, _, rest = attack.partition("::")
+                if "::" in rest:
+                    technique, _, rest = rest.partition("::")
+                    subtechnique, _, id = rest.rpartition(" ")
+                    tactics[tactic].add((technique, subtechnique, id))
+                else:
+                    technique, _, id = rest.rpartition(" ")
+                    tactics[tactic].add((technique, id))
+
+        column_one = []
+        column_two = []
+
+        for (tactic, techniques) in sorted(tactics.items()):
+            column_one.append(tactic.upper())
+            # add extra space when more than one technique
+            column_one.extend(["" for i in range(len(techniques) - 1)])
+
+            for spec in sorted(techniques):
+                if len(spec) == 2:
+                    technique, id = spec
+                    column_two.append("%s %s" % (technique, id))
+                elif len(spec) == 3:
+                    technique, subtechnique, id = spec
+                    column_two.append("%s::%s %s" % (technique, subtechnique, id))
+                else:
+                    raise RuntimeError("unexpected ATT&CK spec format")
+
+        self.view_attack.setRowCount(max(len(column_one), len(column_two)))
+
+        for (row, value) in enumerate(column_one):
+            self.view_attack.setItem(row, 0, self.render_new_table_header_item(value))
+
+        for (row, value) in enumerate(column_two):
+            self.view_attack.setItem(row, 1, QtWidgets.QTableWidgetItem(value))
+
+        # resize columns to content
+        self.view_attack.resizeColumnsToContents()
+
+    def render_new_table_header_item(self, text):
+        """create new table header item with our style
+
+        @param text: header text to display
+        """
+        item = QtWidgets.QTableWidgetItem(text)
+        item.setForeground(QtGui.QColor(37, 147, 215))
+        font = QtGui.QFont()
+        font.setBold(True)
+        item.setFont(font)
+        return item
+
+    def reset_view_tree(self):
+        """reset tree view UI controls
+
+        called when user selects plugin reset from menu
+        """
+        self.view_limit_results_by_function.setChecked(False)
+        self.view_search_bar.setText("")
+        self.view_tree.reset_ui()
+
+    def slot_analyze(self):
+        """run capa analysis and reload UI controls
+
+        called when user selects plugin reload from menu
+        """
+        self.range_model_proxy.invalidate()
+        self.search_model_proxy.invalidate()
+        self.model_data.reset()
+        self.model_data.clear()
+        self.disable_controls()
+        self.set_view_status_label("Loading...")
+
+        ida_kernwin.show_wait_box("capa explorer")
+        success = self.load_capa_results()
+        ida_kernwin.hide_wait_box()
+
+        self.reset_view_tree()
+
+        if not success:
+            self.set_view_status_label("Click Analyze to get started...")
+            logger.info("Analysis failed.")
+        else:
+            logger.info("Analysis completed.")
+
+    def slot_reset(self, checked):
+        """reset UI elements
+
+        e.g. checkboxes and IDA highlighting
+        """
+        self.model_data.reset()
+        self.reset_view_tree()
+        logger.info("Reset completed.")
+
+    def slot_checkbox_limit_by_changed(self, state):
+        """slot activated if checkbox clicked
+
+        if checked, configure function filter if screen location is located in function, otherwise clear filter
+
+        @param state: checked state
+        """
+        if state == QtCore.Qt.Checked:
+            self.limit_results_to_function(idaapi.get_func(idaapi.get_screen_ea()))
+        else:
+            self.range_model_proxy.reset_address_range_filter()
+
+        self.view_tree.reset_ui()
+
+    def limit_results_to_function(self, f):
+        """add filter to limit results to current function
+
+        adds new address range filter to include function bounds, allowing basic blocks matched within a function
+        to be included in the results
+
+        @param f: (IDA func_t)
+        """
+        if f:
+            self.range_model_proxy.add_address_range_filter(f.start_ea, f.end_ea)
+        else:
+            # if function not exists don't display any results (assume address never -1)
+            self.range_model_proxy.add_address_range_filter(-1, -1)
+
+    def slot_limit_results_to_search(self, text):
+        """limit tree view results to search matches
+
+        reset view after filter to maintain level 1 expansion
+        """
+        self.search_model_proxy.set_query(text)
+        self.view_tree.reset_ui(should_sort=False)
+
+    def ask_user_directory(self):
+        """create Qt dialog to ask user for a directory"""
+        return str(
+            QtWidgets.QFileDialog.getExistingDirectory(
+                self.parent, "Please select a capa rules directory", self.rule_path
+            )
+        )
+
+    def slot_change_rules_dir(self):
+        """allow user to change rules directory
+
+        user selection stored in settings for future runs
+        """
+        rule_path = self.ask_user_directory()
+        if not rule_path:
+            logger.warning("No rule directory selected, nothing to do.")
+            return
+
+        self.rule_path = rule_path
+        settings.user["rule_path"] = rule_path
+
+        if 1 == idaapi.ask_yn(1, "Run analysis now?"):
+            self.slot_analyze()
+
+    def set_view_status_label(self, text):
+        """update status label control
+
+        @param text: updated text
+        """
+        self.view_status_label.setText(text)
+
+    def disable_controls(self):
+        """disable form controls"""
+        self.view_reset_button.setEnabled(False)
+        self.view_tabs.setTabEnabled(0, False)
+        self.view_tabs.setTabEnabled(1, False)
+
+    def enable_controls(self):
+        """enable form controls"""
+        self.view_reset_button.setEnabled(True)
+        self.view_tabs.setTabEnabled(0, True)
+        self.view_tabs.setTabEnabled(1, True)

capa/ida/plugin/hooks.py

@@ -0,0 +1,60 @@
+# Copyright (C) 2020 FireEye, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+import idaapi
+
+
+class CapaExplorerIdaHooks(idaapi.UI_Hooks):
+    def __init__(self, screen_ea_changed_hook, action_hooks):
+        """facilitate IDA UI hooks
+
+        @param screen_ea_changed_hook: function hook for IDA screen ea changed
+        @param action_hooks: dict of IDA action handles
+        """
+        super(CapaExplorerIdaHooks, self).__init__()
+
+        self.screen_ea_changed_hook = screen_ea_changed_hook
+        self.process_action_hooks = action_hooks
+        self.process_action_handle = None
+        self.process_action_meta = {}
+
+    def preprocess_action(self, name):
+        """called prior to action completed
+
+        @param name: name of action defined by idagui.cfg
+
+        @retval must be 0
+        """
+        self.process_action_handle = self.process_action_hooks.get(name, None)
+
+        if self.process_action_handle:
+            self.process_action_handle(self.process_action_meta)
+
+        # must return 0 for IDA
+        return 0
+
+    def postprocess_action(self):
+        """called after action completed"""
+        if not self.process_action_handle:
+            return
+
+        self.process_action_handle(self.process_action_meta, post=True)
+        self.reset()
+
+    def screen_ea_changed(self, curr_ea, prev_ea):
+        """called after screen location is changed
+
+        @param curr_ea: current location
+        @param prev_ea: prev location
+        """
+        self.screen_ea_changed_hook(idaapi.get_current_widget(), curr_ea, prev_ea)
+
+    def reset(self):
+        """reset internal state"""
+        self.process_action_handle = None
+        self.process_action_meta.clear()

capa/ida/plugin/item.py

@@ -28,20 +28,21 @@ def info_to_name(display):
 
 
 def location_to_hex(location):
-    """ convert location to hex for display """
+    """convert location to hex for display"""
     return "%08X" % location
 
 
 class CapaExplorerDataItem(object):
-    """ store data for CapaExplorerDataModel """
+    """store data for CapaExplorerDataModel"""
 
     def __init__(self, parent, data):
-        """ """
+        """initialize item"""
         self.pred = parent
         self._data = data
         self.children = []
         self._checked = False
 
+        # default state for item
         self.flags = (
             QtCore.Qt.ItemIsEnabled
             | QtCore.Qt.ItemIsSelectable
@@ -53,117 +54,146 @@ class CapaExplorerDataItem(object):
             self.pred.appendChild(self)
 
     def setIsEditable(self, isEditable=False):
-        """ modify item flags to be editable or not """
+        """modify item editable flags
+
+        @param isEditable: True, can edit, False cannot edit
+        """
         if isEditable:
             self.flags |= QtCore.Qt.ItemIsEditable
         else:
             self.flags &= ~QtCore.Qt.ItemIsEditable
 
     def setChecked(self, checked):
-        """ set item as checked """
+        """set item as checked
+
+        @param checked: True, item checked, False item not checked
+        """
         self._checked = checked
 
     def isChecked(self):
-        """ get item is checked """
+        """get item is checked"""
         return self._checked
 
     def appendChild(self, item):
-        """add child item
+        """add a new child to specified item
 
-        @param item: CapaExplorerDataItem*
+        @param item: CapaExplorerDataItem
         """
         self.children.append(item)
 
     def child(self, row):
         """get child row
 
-        @param row: TODO
+        @param row: row number
         """
         return self.children[row]
 
     def childCount(self):
-        """ get child count """
+        """get child count"""
         return len(self.children)
 
     def columnCount(self):
-        """ get column count """
+        """get column count"""
         return len(self._data)
 
     def data(self, column):
-        """ get data at column """
+        """get data at column
+
+        @param: column number
+        """
         try:
             return self._data[column]
         except IndexError:
             return None
 
     def parent(self):
-        """ get parent """
+        """get parent"""
         return self.pred
 
     def row(self):
-        """ get row location """
+        """get row location"""
         if self.pred:
             return self.pred.children.index(self)
         return 0
 
     def setData(self, column, value):
-        """ set data in column """
+        """set data in column
+
+        @param column: column number
+        @value: value to set (assume str)
+        """
         self._data[column] = value
 
     def children(self):
-        """ yield children """
+        """yield children"""
         for child in self.children:
             yield child
 
     def removeChildren(self):
-        """ remove children from node """
+        """remove children"""
         del self.children[:]
 
     def __str__(self):
-        """ get string representation of columns """
+        """get string representation of columns
+
+        used for copy-n-paste operations
+        """
         return " ".join([data for data in self._data if data])
 
     @property
     def info(self):
-        """ return data stored in information column """
+        """return data stored in information column"""
         return self._data[0]
 
     @property
     def location(self):
-        """ return data stored in location column """
+        """return data stored in location column"""
         try:
+            # address stored as str, convert to int before return
             return int(self._data[1], 16)
         except ValueError:
             return None
 
     @property
     def details(self):
-        """ return data stored in details column """
+        """return data stored in details column"""
         return self._data[2]
 
 
 class CapaExplorerRuleItem(CapaExplorerDataItem):
-    """ store data relevant to capa function result """
+    """store data for rule result"""
 
     fmt = "%s (%d matches)"
 
-    def __init__(self, parent, display, count, source):
-        """ """
-        display = self.fmt % (display, count) if count > 1 else display
-        super(CapaExplorerRuleItem, self).__init__(parent, [display, "", ""])
+    def __init__(self, parent, name, namespace, count, source):
+        """initialize item
+
+        @param parent: parent node
+        @param name: rule name
+        @param namespace: rule namespace
+        @param count: number of match for this rule
+        @param source: rule source (tooltip)
+        """
+        display = self.fmt % (name, count) if count > 1 else name
+        super(CapaExplorerRuleItem, self).__init__(parent, [display, "", namespace])
         self._source = source
 
     @property
     def source(self):
-        """ return rule contents for display """
+        """return rule source to display (tooltip)"""
         return self._source
 
 
 class CapaExplorerRuleMatchItem(CapaExplorerDataItem):
-    """ store data relevant to capa function match result """
+    """store data for rule match"""
 
     def __init__(self, parent, display, source=""):
-        """ """
+        """initialize item
+
+        @param parent: parent node
+        @param display: text to display in UI
+        @param source: rule match source to display (tooltip)
+        """
         super(CapaExplorerRuleMatchItem, self).__init__(parent, [display, "", ""])
         self._source = source
 
@@ -174,82 +204,125 @@ class CapaExplorerRuleMatchItem(CapaExplorerDataItem):
 
 
 class CapaExplorerFunctionItem(CapaExplorerDataItem):
-    """ store data relevant to capa function result """
+    """store data for function match"""
 
     fmt = "function(%s)"
 
     def __init__(self, parent, location):
-        """ """
+        """initialize item
+
+        @param parent: parent node
+        @param location: virtual address of function as seen by IDA
+        """
         super(CapaExplorerFunctionItem, self).__init__(
             parent, [self.fmt % idaapi.get_name(location), location_to_hex(location), ""]
         )
 
     @property
     def info(self):
-        """ """
+        """return function name"""
         info = super(CapaExplorerFunctionItem, self).info
         display = info_to_name(info)
         return display if display else info
 
     @info.setter
     def info(self, display):
-        """ """
+        """set function name
+
+        called when user changes function name in plugin UI
+
+        @param display: new function name to display
+        """
         self._data[0] = self.fmt % display
 
 
 class CapaExplorerSubscopeItem(CapaExplorerDataItem):
-    """ store data relevant to subscope """
+    """store data for subscope match"""
 
     fmt = "subscope(%s)"
 
     def __init__(self, parent, scope):
-        """ """
+        """initialize item
+
+        @param parent: parent node
+        @param scope: subscope name
+        """
         super(CapaExplorerSubscopeItem, self).__init__(parent, [self.fmt % scope, "", ""])
 
 
 class CapaExplorerBlockItem(CapaExplorerDataItem):
-    """ store data relevant to capa basic block result """
+    """store data for basic block match"""
 
     fmt = "basic block(loc_%08X)"
 
     def __init__(self, parent, location):
-        """ """
+        """initialize item
+
+        @param parent: parent node
+        @param location: virtual address of basic block as seen by IDA
+        """
         super(CapaExplorerBlockItem, self).__init__(parent, [self.fmt % location, location_to_hex(location), ""])
 
 
 class CapaExplorerDefaultItem(CapaExplorerDataItem):
-    """ store data relevant to capa default result """
+    """store data for default match e.g. statement (and, or)"""
 
     def __init__(self, parent, display, details="", location=None):
-        """ """
+        """initialize item
+
+        @param parent: parent node
+        @param display: text to display in UI
+        @param details: text to display in details section of UI
+        @param location: virtual address as seen by IDA
+        """
         location = location_to_hex(location) if location else ""
         super(CapaExplorerDefaultItem, self).__init__(parent, [display, location, details])
 
 
 class CapaExplorerFeatureItem(CapaExplorerDataItem):
-    """ store data relevant to capa feature result """
+    """store data for feature match"""
 
     def __init__(self, parent, display, location="", details=""):
-        """ """
+        """initialize item
+
+        @param parent: parent node
+        @param display: text to display in UI
+        @param details: text to display in details section of UI
+        @param location: virtual address as seen by IDA
+        """
         location = location_to_hex(location) if location else ""
         super(CapaExplorerFeatureItem, self).__init__(parent, [display, location, details])
 
 
 class CapaExplorerInstructionViewItem(CapaExplorerFeatureItem):
-    """ store data relevant to an instruction preview """
+    """store data for instruction match"""
 
     def __init__(self, parent, display, location):
-        """ """
+        """initialize item
+
+        details section shows disassembly view for match
+
+        @param parent: parent node
+        @param display: text to display in UI
+        @param location: virtual address as seen by IDA
+        """
         details = capa.ida.helpers.get_disasm_line(location)
         super(CapaExplorerInstructionViewItem, self).__init__(parent, display, location=location, details=details)
         self.ida_highlight = idc.get_color(location, idc.CIC_ITEM)
 
 
 class CapaExplorerByteViewItem(CapaExplorerFeatureItem):
-    """ store data relevant to byte preview """
+    """store data for byte match"""
 
     def __init__(self, parent, display, location):
-        """ """
+        """initialize item
+
+        details section shows byte preview for match
+
+        @param parent: parent node
+        @param display: text to display in UI
+        @param location: virtual address as seen by IDA
+        """
         byte_snap = idaapi.get_bytes(location, 32)
 
         if byte_snap:
@@ -266,9 +339,14 @@ class CapaExplorerByteViewItem(CapaExplorerFeatureItem):
 
 
 class CapaExplorerStringViewItem(CapaExplorerFeatureItem):
-    """ store data relevant to string preview """
+    """store data for string match"""
 
-    def __init__(self, parent, display, location):
-        """ """
-        super(CapaExplorerStringViewItem, self).__init__(parent, display, location=location)
+    def __init__(self, parent, display, location, value):
+        """initialize item
+
+        @param parent: parent node
+        @param display: text to display in UI
+        @param location: virtual address as seen by IDA
+        """
+        super(CapaExplorerStringViewItem, self).__init__(parent, display, location=location, details=value)
         self.ida_highlight = idc.get_color(location, idc.CIC_ITEM)

capa/ida/plugin/model.py

@@ -9,13 +9,12 @@
 from collections import deque
 
 import idc
-import six
 import idaapi
-from PyQt5 import Qt, QtGui, QtCore
+from PyQt5 import QtGui, QtCore
 
 import capa.ida.helpers
 import capa.render.utils as rutils
-from capa.ida.explorer.item import (
+from capa.ida.plugin.item import (
     CapaExplorerDataItem,
     CapaExplorerRuleItem,
     CapaExplorerBlockItem,
@@ -30,11 +29,11 @@ from capa.ida.explorer.item import (
 )
 
 # default highlight color used in IDA window
-DEFAULT_HIGHLIGHT = 0xD096FF
+DEFAULT_HIGHLIGHT = 0xE6C700
 
 
 class CapaExplorerDataModel(QtCore.QAbstractItemModel):
-    """ """
+    """model for displaying hierarchical results return by capa"""
 
     COLUMN_INDEX_RULE_INFORMATION = 0
     COLUMN_INDEX_VIRTUAL_ADDRESS = 1
@@ -43,14 +42,16 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel):
     COLUMN_COUNT = 3
 
     def __init__(self, parent=None):
-        """ """
+        """initialize model"""
         super(CapaExplorerDataModel, self).__init__(parent)
+        # root node does not have parent, contains header columns
         self.root_node = CapaExplorerDataItem(None, ["Rule Information", "Address", "Details"])
 
     def reset(self):
-        """ """
-        # reset checkboxes and color highlights
-        # TODO: make less hacky
+        """reset UI elements (e.g. checkboxes, IDA color highlights)
+
+        called when view wants to reset UI display
+        """
         for idx in range(self.root_node.childCount()):
             root_index = self.index(idx, 0, QtCore.QModelIndex())
             for model_index in self.iterateChildrenIndexFromRootIndex(root_index, ignore_root=False):
@@ -59,15 +60,18 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel):
                 self.dataChanged.emit(model_index, model_index)
 
     def clear(self):
-        """ """
+        """clear model data
+
+        called when view wants to clear UI display
+        """
         self.beginResetModel()
         self.root_node.removeChildren()
         self.endResetModel()
 
     def columnCount(self, model_index):
-        """get the number of columns for the children of the given parent
+        """return number of columns for the children of the given parent
 
-        @param model_index: QModelIndex*
+        @param model_index: QModelIndex
 
         @retval column count
         """
@@ -77,9 +81,11 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel):
             return self.root_node.columnCount()
 
     def data(self, model_index, role):
-        """get data stored under the given role for the item referred to by the index
+        """return data stored at given index by display role
 
-        @param model_index: QModelIndex*
+        this function is used to control UI elements (e.g. text font, color, etc.) based on column, item type, etc.
+
+        @param model_index: QModelIndex
         @param role: QtCore.Qt.*
 
         @retval data to be displayed
@@ -131,14 +137,14 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel):
             )
             and column == CapaExplorerDataModel.COLUMN_INDEX_RULE_INFORMATION
         ):
-            # set bold font for top-level rules
+            # set bold font for important items
             font = QtGui.QFont()
             font.setBold(True)
             return font
 
         if role == QtCore.Qt.ForegroundRole and column == CapaExplorerDataModel.COLUMN_INDEX_VIRTUAL_ADDRESS:
             # set color for virtual address column
-            return QtGui.QColor(88, 139, 174)
+            return QtGui.QColor(37, 147, 215)
 
         if (
             role == QtCore.Qt.ForegroundRole
@@ -151,9 +157,9 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel):
         return None
 
     def flags(self, model_index):
-        """get item flags for given index
+        """return item flags for given index
 
-        @param model_index: QModelIndex*
+        @param model_index: QModelIndex
 
         @retval QtCore.Qt.ItemFlags
         """
@@ -163,13 +169,13 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel):
         return model_index.internalPointer().flags
 
     def headerData(self, section, orientation, role):
-        """get data for the given role and section in the header with the specified orientation
+        """return data for the given role and section in the header with the specified orientation
 
         @param section: int
         @param orientation: QtCore.Qt.Orientation
         @param role: QtCore.Qt.DisplayRole
 
-        @retval header data list()
+        @retval header data
         """
         if orientation == QtCore.Qt.Horizontal and role == QtCore.Qt.DisplayRole:
             return self.root_node.data(section)
@@ -177,13 +183,13 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel):
         return None
 
     def index(self, row, column, parent):
-        """get index of the item in the model specified by the given row, column and parent index
+        """return index of the item by row, column, and parent index
 
-        @param row: int
-        @param column: int
-        @param parent: QModelIndex*
+        @param row: item row
+        @param column: item column
+        @param parent: QModelIndex of parent
 
-        @retval QModelIndex*
+        @retval QModelIndex of item
         """
         if not self.hasIndex(row, column, parent):
             return QtCore.QModelIndex()
@@ -201,13 +207,13 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel):
             return QtCore.QModelIndex()
 
     def parent(self, model_index):
-        """get parent of the model item with the given index
+        """return parent index by child index
 
-        if the item has no parent, an invalid QModelIndex* is returned
+        if the item has no parent, an invalid QModelIndex is returned
 
-        @param model_index: QModelIndex*
+        @param model_index: QModelIndex of child
 
-        @retval QModelIndex*
+        @retval QModelIndex of parent
         """
         if not model_index.isValid():
             return QtCore.QModelIndex()
@@ -223,10 +229,10 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel):
     def iterateChildrenIndexFromRootIndex(self, model_index, ignore_root=True):
         """depth-first traversal of child nodes
 
-        @param model_index: QModelIndex*
-        @param ignore_root: if set, do not return root index
+        @param model_index: QModelIndex of starting item
+        @param ignore_root: True, do not yield root index, False yield root index
 
-        @retval yield QModelIndex*
+        @retval yield QModelIndex
         """
         visited = set()
         stack = deque((model_index,))
@@ -248,10 +254,10 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel):
                     stack.append(child_index.child(idx, 0))
 
     def reset_ida_highlighting(self, item, checked):
-        """reset IDA highlight for an item
+        """reset IDA highlight for item
 
-        @param item: capa explorer item
-        @param checked: indicates item is or not checked
+        @param item: CapaExplorerDataItem
+        @param checked: True, item checked, False item not checked
         """
         if not isinstance(
             item, (CapaExplorerStringViewItem, CapaExplorerInstructionViewItem, CapaExplorerByteViewItem)
@@ -275,13 +281,11 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel):
                 idc.set_color(item.location, idc.CIC_ITEM, item.ida_highlight)
 
     def setData(self, model_index, value, role):
-        """set the role data for the item at index to value
+        """set data at index by role
 
-        @param model_index: QModelIndex*
-        @param value: QVariant*
+        @param model_index: QModelIndex of item
+        @param value: value to set
         @param role: QtCore.Qt.EditRole
-
-        @retval True/False
         """
         if not model_index.isValid():
             return False
@@ -316,12 +320,11 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel):
         return False
 
     def rowCount(self, model_index):
-        """get the number of rows under the given parent
+        """return number of rows under item by index
 
-        when the parent is valid it means that is returning the number of
-        children of parent
+        when the parent is valid it means that is returning the number of children of parent
 
-        @param model_index: QModelIndex*
+        @param model_index: QModelIndex
 
         @retval row count
         """
@@ -341,11 +344,7 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel):
         @param parent: parent to which new child is assigned
         @param statement: statement read from doc
         @param locations: locations of children (applies to range only?)
-        @param doc: capa result doc
-
-        "statement": {
-            "type": "or"
-        },
+        @param doc: result doc
         """
         if statement["type"] in ("and", "or", "optional"):
             display = statement["type"]
@@ -399,24 +398,7 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel):
 
         @param parent: parent node to which new child is assigned
         @param match: match read from doc
-        @param doc: capa result doc
-
-        "matches": {
-            "0": {
-                "children": [],
-                "locations": [
-                    4317184
-                ],
-                "node": {
-                    "feature": {
-                        "section": ".rsrc",
-                        "type": "section"
-                    },
-                    "type": "feature"
-                },
-                "success": true
-            }
-        },
+        @param doc: result doc
         """
         if not match["success"]:
             # TODO: display failed branches at some point? Help with debugging rules?
@@ -451,7 +433,11 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel):
         self.beginResetModel()
 
         for rule in rutils.capability_rules(doc):
-            parent = CapaExplorerRuleItem(self.root_node, rule["meta"]["name"], len(rule["matches"]), rule["source"])
+            rule_name = rule["meta"]["name"]
+            rule_namespace = rule["meta"].get("namespace")
+            parent = CapaExplorerRuleItem(
+                self.root_node, rule_name, rule_namespace, len(rule["matches"]), rule["source"]
+            )
 
             for (location, match) in doc["rules"][rule["meta"]["name"]]["matches"].items():
                 if rule["meta"]["scope"] == capa.rules.FILE_SCOPE:
@@ -472,15 +458,6 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel):
         """convert capa doc feature type string to display string for ui
 
         @param feature: capa feature read from doc
-
-        Example:
-            "feature": {
-                "bytes": "01 14 02 00 00 00 00 00 C0 00 00 00 00 00 00 46",
-                "description": "CLSID_ShellLink",
-                "type": "bytes"
-            }
-
-            bytes(01 14 02 00 00 00 00 00 C0 00 00 00 00 00 00 46 = CLSID_ShellLink)
         """
         if feature[feature["type"]]:
             if feature.get("description", ""):
@@ -497,13 +474,6 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel):
         @param feature: capa doc feature node
         @param locations: locations identified for feature
         @param doc: capa doc
-
-        Example:
-          "feature": {
-            "description": "FILE_WRITE_DATA",
-            "number": "0x2",
-            "type": "number"
-          }
         """
         display = self.capa_doc_feature_to_display(feature)
 
@@ -532,14 +502,7 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel):
         @param feature: feature read from doc
         @param doc: capa feature doc
         @param location: address of feature
-        @param display: text to display in plugin ui
-
-        Example:
-          "feature": {
-            "description": "FILE_WRITE_DATA",
-            "number": "0x2",
-            "type": "number"
-          }
+        @param display: text to display in plugin UI
         """
         # special handling for characteristic pending type
         if feature["type"] == "characteristic":
@@ -559,7 +522,7 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel):
             )
 
         if feature["type"] == "regex":
-            return CapaExplorerFeatureItem(parent, display, location, details=feature["match"])
+            return CapaExplorerStringViewItem(parent, display, location, feature["match"])
 
         if feature["type"] == "basicblock":
             return CapaExplorerBlockItem(parent, location)
@@ -584,7 +547,7 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel):
 
         if feature["type"] in ("string",):
             # display string preview
-            return CapaExplorerStringViewItem(parent, display, location)
+            return CapaExplorerStringViewItem(parent, display, location, feature[feature["type"]])
 
         if feature["type"] in ("import", "export"):
             # display no preview
@@ -595,7 +558,9 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel):
     def update_function_name(self, old_name, new_name):
         """update all instances of old function name with new function name
 
-        @param old_name: previous function name
+        called when user updates function name using plugin UI
+
+        @param old_name: old function name
         @param new_name: new function name
         """
         # create empty root index for search

capa/ida/plugin/proxy.py

@@ -0,0 +1,226 @@
+# Copyright (C) 2020 FireEye, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+import six
+from PyQt5 import QtCore
+from PyQt5.QtCore import Qt
+
+from capa.ida.plugin.model import CapaExplorerDataModel
+
+
+class CapaExplorerRangeProxyModel(QtCore.QSortFilterProxyModel):
+    """filter results based on virtual address range as seen by IDA
+
+    implements filtering for "limit results by current function" checkbox in plugin UI
+
+    minimum and maximum virtual addresses are used to filter results to a specific address range. this allows
+    basic blocks to be included when limiting results to a specific function
+    """
+
+    def __init__(self, parent=None):
+        """initialize proxy filter"""
+        super(CapaExplorerRangeProxyModel, self).__init__(parent)
+        self.min_ea = None
+        self.max_ea = None
+
+    def lessThan(self, left, right):
+        """return True if left item is less than right item, else False
+
+        @param left: QModelIndex of left
+        @param right: QModelIndex of right
+        """
+        ldata = left.internalPointer().data(left.column())
+        rdata = right.internalPointer().data(right.column())
+
+        if (
+            ldata
+            and rdata
+            and left.column() == CapaExplorerDataModel.COLUMN_INDEX_VIRTUAL_ADDRESS
+            and left.column() == right.column()
+        ):
+            # convert virtual address before compare
+            return int(ldata, 16) < int(rdata, 16)
+        else:
+            # compare as lowercase
+            return ldata.lower() < rdata.lower()
+
+    def filterAcceptsRow(self, row, parent):
+        """return true if the item in the row indicated by the given row and parent should be included in the model;
+        otherwise return false
+
+        @param row: row number
+        @param parent: QModelIndex of parent
+        """
+        if self.filter_accepts_row_self(row, parent):
+            return True
+
+        alpha = parent
+        while alpha.isValid():
+            if self.filter_accepts_row_self(alpha.row(), alpha.parent()):
+                return True
+            alpha = alpha.parent()
+
+        if self.index_has_accepted_children(row, parent):
+            return True
+
+        return False
+
+    def index_has_accepted_children(self, row, parent):
+        """return True if parent has one or more children that match filter, else False
+
+        @param row: row number
+        @param parent: QModelIndex of parent
+        """
+        model_index = self.sourceModel().index(row, 0, parent)
+
+        if model_index.isValid():
+            for idx in range(self.sourceModel().rowCount(model_index)):
+                if self.filter_accepts_row_self(idx, model_index):
+                    return True
+                if self.index_has_accepted_children(idx, model_index):
+                    return True
+
+        return False
+
+    def filter_accepts_row_self(self, row, parent):
+        """return True if filter accepts row, else False
+
+        @param row: row number
+        @param parent: QModelIndex of parent
+        """
+        # filter not set
+        if self.min_ea is None and self.max_ea is None:
+            return True
+
+        index = self.sourceModel().index(row, 0, parent)
+        data = index.internalPointer().data(CapaExplorerDataModel.COLUMN_INDEX_VIRTUAL_ADDRESS)
+
+        # virtual address may be empty
+        if not data:
+            return False
+
+        # convert virtual address str to int
+        ea = int(data, 16)
+
+        if self.min_ea <= ea and ea < self.max_ea:
+            return True
+
+        return False
+
+    def add_address_range_filter(self, min_ea, max_ea):
+        """add new address range filter
+
+        called when user checks "limit results by current function" in plugin UI
+
+        @param min_ea: minimum virtual address as seen by IDA
+        @param max_ea: maximum virtual address as seen by IDA
+        """
+        self.min_ea = min_ea
+        self.max_ea = max_ea
+
+        self.setFilterKeyColumn(CapaExplorerDataModel.COLUMN_INDEX_VIRTUAL_ADDRESS)
+        self.invalidateFilter()
+
+    def reset_address_range_filter(self):
+        """remove address range filter (accept all results)
+
+        called when user un-checks "limit results by current function" in plugin UI
+        """
+        self.min_ea = None
+        self.max_ea = None
+        self.invalidateFilter()
+
+
+class CapaExplorerSearchProxyModel(QtCore.QSortFilterProxyModel):
+    """A SortFilterProxyModel that accepts rows with a substring match for a configurable query.
+
+    Looks for matches in the text of all rows.
+    Displays the entire tree row if any of the tree branches,
+     that is, you can filter by rule name, or also
+     filter by "characteristic(nzxor)" to filter matches with some feature.
+    """
+
+    def __init__(self, parent=None):
+        """ """
+        super(CapaExplorerSearchProxyModel, self).__init__(parent)
+        self.query = ""
+        self.setFilterKeyColumn(-1)  # all columns
+
+    def filterAcceptsRow(self, row, parent):
+        """true if the item in the row indicated by the given row and parent
+        should be included in the model; otherwise returns false
+
+        @param row: int
+        @param parent: QModelIndex*
+
+        @retval True/False
+        """
+        # this row matches, accept it
+        if self.filter_accepts_row_self(row, parent):
+            return True
+
+        # the parent of this row matches, accept it
+        alpha = parent
+        while alpha.isValid():
+            if self.filter_accepts_row_self(alpha.row(), alpha.parent()):
+                return True
+            alpha = alpha.parent()
+
+        # this row is a parent, and a child matches, accept it
+        if self.index_has_accepted_children(row, parent):
+            return True
+
+        return False
+
+    def index_has_accepted_children(self, row, parent):
+        """returns True if the given row or its children should be accepted"""
+        source_model = self.sourceModel()
+        model_index = source_model.index(row, 0, parent)
+
+        if model_index.isValid():
+            for idx in range(source_model.rowCount(model_index)):
+                if self.filter_accepts_row_self(idx, model_index):
+                    return True
+                if self.index_has_accepted_children(idx, model_index):
+                    return True
+
+        return False
+
+    def filter_accepts_row_self(self, row, parent):
+        """returns True if the given row should be accepted"""
+        if self.query == "":
+            return True
+
+        source_model = self.sourceModel()
+
+        for column in (
+            CapaExplorerDataModel.COLUMN_INDEX_RULE_INFORMATION,
+            CapaExplorerDataModel.COLUMN_INDEX_VIRTUAL_ADDRESS,
+            CapaExplorerDataModel.COLUMN_INDEX_DETAILS,
+        ):
+            index = source_model.index(row, column, parent)
+            data = source_model.data(index, Qt.DisplayRole)
+
+            if not data:
+                continue
+
+            if not isinstance(data, six.string_types):
+                # sanity check: should already be a string, but double check
+                continue
+
+            # case in-sensitive matching
+            if self.query.lower() in data.lower():
+                return True
+
+        return False
+
+    def set_query(self, query):
+        self.query = query
+        self.invalidateFilter()
+
+    def reset_query(self):
+        self.set_query("")

capa/ida/plugin/view.py

@@ -7,25 +7,25 @@
 # See the License for the specific language governing permissions and limitations under the License.
 
 import idc
-import idaapi
-from PyQt5 import QtGui, QtCore, QtWidgets
+from PyQt5 import QtCore, QtWidgets
 
-from capa.ida.explorer.item import CapaExplorerRuleItem, CapaExplorerFunctionItem
-from capa.ida.explorer.model import CapaExplorerDataModel
+from capa.ida.plugin.item import CapaExplorerFunctionItem
+from capa.ida.plugin.model import CapaExplorerDataModel
+
+MAX_SECTION_SIZE = 750
 
 
 class CapaExplorerQtreeView(QtWidgets.QTreeView):
-    """capa explorer QTreeView implementation
+    """tree view used to display hierarchical capa results
 
-    view controls UI action responses and displays data from
-    CapaExplorerDataModel
+    view controls UI action responses and displays data from CapaExplorerDataModel
 
-    view does not modify CapaExplorerDataModel directly - data
-    modifications should be implemented in CapaExplorerDataModel
+    view does not modify CapaExplorerDataModel directly - data modifications should be implemented
+    in CapaExplorerDataModel
     """
 
     def __init__(self, model, parent=None):
-        """ initialize CapaExplorerQTreeView """
+        """initialize view"""
         super(CapaExplorerQtreeView, self).__init__(parent)
 
         self.setModel(model)
@@ -33,6 +33,9 @@ class CapaExplorerQtreeView(QtWidgets.QTreeView):
         self.model = model
         self.parent = parent
 
+        # control when we resize columns
+        self.should_resize_columns = True
+
         # configure custom UI controls
         self.setContextMenuPolicy(QtCore.Qt.CustomContextMenu)
         self.setExpandsOnDoubleClick(False)
@@ -43,9 +46,12 @@ class CapaExplorerQtreeView(QtWidgets.QTreeView):
         for idx in range(CapaExplorerDataModel.COLUMN_COUNT):
             self.header().setSectionResizeMode(idx, QtWidgets.QHeaderView.Interactive)
 
+        # disable stretch to enable horizontal scroll for last column, when needed
+        self.header().setStretchLastSection(False)
+
         # connect slots to resize columns when expanded or collapsed
-        self.expanded.connect(self.resize_columns_to_content)
-        self.collapsed.connect(self.resize_columns_to_content)
+        self.expanded.connect(self.slot_resize_columns_to_content)
+        self.collapsed.connect(self.slot_resize_columns_to_content)
 
         # connect slots
         self.customContextMenuRequested.connect(self.slot_custom_context_menu_requested)
@@ -53,27 +59,57 @@ class CapaExplorerQtreeView(QtWidgets.QTreeView):
 
         self.setStyleSheet("QTreeView::item {padding-right: 15 px;padding-bottom: 2 px;}")
 
-    def reset(self):
+    def reset_ui(self, should_sort=True):
         """reset user interface changes
 
-        called when view should reset any user interface changes
-        made since the last reset e.g. IDA window highlighting
-        """
-        self.expandToDepth(0)
-        self.resize_columns_to_content()
+        called when view should reset UI display e.g. expand items, resize columns
 
-    def resize_columns_to_content(self):
-        """ reset view columns to contents """
-        self.header().resizeSections(QtWidgets.QHeaderView.ResizeToContents)
+        @param should_sort: True, sort results after reset, False don't sort results after reset
+        """
+        if should_sort:
+            self.sortByColumn(CapaExplorerDataModel.COLUMN_INDEX_RULE_INFORMATION, QtCore.Qt.AscendingOrder)
+
+        self.should_resize_columns = False
+        self.expandToDepth(0)
+        self.should_resize_columns = True
+
+        self.slot_resize_columns_to_content()
+
+    def slot_resize_columns_to_content(self):
+        """reset view columns to contents"""
+        if self.should_resize_columns:
+            self.header().resizeSections(QtWidgets.QHeaderView.ResizeToContents)
+
+            # limit size of first section
+            if self.header().sectionSize(0) > MAX_SECTION_SIZE:
+                self.header().resizeSection(0, MAX_SECTION_SIZE)
 
     def map_index_to_source_item(self, model_index):
         """map proxy model index to source model item
 
-        @param model_index: QModelIndex*
+        @param model_index: QModelIndex
 
-        @retval QObject*
+        @retval QObject
         """
-        return self.model.mapToSource(model_index).internalPointer()
+        # assume that self.model here is either:
+        #  - CapaExplorerDataModel, or
+        #  - QSortFilterProxyModel subclass
+        #
+        # The ProxyModels may be chained,
+        #  so keep resolving the index the CapaExplorerDataModel.
+
+        model = self.model
+        while not isinstance(model, CapaExplorerDataModel):
+            if not model_index.isValid():
+                raise ValueError("invalid index")
+
+            model_index = model.mapToSource(model_index)
+            model = model.sourceModel()
+
+        if not model_index.isValid():
+            raise ValueError("invalid index")
+
+        return model_index.internalPointer()
 
     def send_data_to_clipboard(self, data):
         """copy data to the clipboard
@@ -91,7 +127,7 @@ class CapaExplorerQtreeView(QtWidgets.QTreeView):
         @param data: data passed to slot
         @param slot: slot to connect
 
-        @retval QAction*
+        @retval QAction
         """
         action = QtWidgets.QAction(display, self.parent)
         action.setData(data)
@@ -104,7 +140,7 @@ class CapaExplorerQtreeView(QtWidgets.QTreeView):
 
         @param data: tuple
 
-        @yield QAction*
+        @yield QAction
         """
         default_actions = (
             ("Copy column", data, self.slot_copy_column),
@@ -120,7 +156,7 @@ class CapaExplorerQtreeView(QtWidgets.QTreeView):
 
         @param data: tuple
 
-        @yield QAction*
+        @yield QAction
         """
         function_actions = (("Rename function", data, self.slot_rename_function),)
 
@@ -137,11 +173,11 @@ class CapaExplorerQtreeView(QtWidgets.QTreeView):
 
         creates custom context menu containing default actions
 
-        @param pos: TODO
-        @param item: TODO
-        @param model_index: TODO
+        @param pos: cursor position
+        @param item: CapaExplorerDataItem
+        @param model_index: QModelIndex
 
-        @retval QMenu*
+        @retval QMenu
         """
         menu = QtWidgets.QMenu()
 
@@ -153,14 +189,13 @@ class CapaExplorerQtreeView(QtWidgets.QTreeView):
     def load_function_item_context_menu(self, pos, item, model_index):
         """create function custom context menu
 
-        creates custom context menu containing actions specific to functions
-        and the default actions
+        creates custom context menu with both default actions and function actions
 
-        @param pos: TODO
-        @param item: TODO
-        @param model_index: TODO
+        @param pos: cursor position
+        @param item: CapaExplorerDataItem
+        @param model_index: QModelIndex
 
-        @retval QMenu*
+        @retval QMenu
         """
         menu = QtWidgets.QMenu()
 
@@ -172,8 +207,8 @@ class CapaExplorerQtreeView(QtWidgets.QTreeView):
     def show_custom_context_menu(self, menu, pos):
         """display custom context menu in view
 
-        @param menu: TODO
-        @param pos: TODO
+        @param menu: QMenu to display
+        @param pos: cursor position
         """
         if menu:
             menu.exec_(self.viewport().mapToGlobal(pos))
@@ -181,10 +216,9 @@ class CapaExplorerQtreeView(QtWidgets.QTreeView):
     def slot_copy_column(self, action):
         """slot connected to custom context menu
 
-        allows user to select a column and copy the data
-        to clipboard
+        allows user to select a column and copy the data to clipboard
 
-        @param action: QAction*
+        @param action: QAction
         """
         _, item, model_index = action.data()
         self.send_data_to_clipboard(item.data(model_index.column()))
@@ -192,10 +226,9 @@ class CapaExplorerQtreeView(QtWidgets.QTreeView):
     def slot_copy_row(self, action):
         """slot connected to custom context menu
 
-        allows user to select a row and copy the space-delimited
-        data to clipboard
+        allows user to select a row and copy the space-delimited data to clipboard
 
-        @param action: QAction*
+        @param action: QAction
         """
         _, item, _ = action.data()
         self.send_data_to_clipboard(str(item))
@@ -203,10 +236,9 @@ class CapaExplorerQtreeView(QtWidgets.QTreeView):
     def slot_rename_function(self, action):
         """slot connected to custom context menu
 
-        allows user to select a edit a function name and push
-        changes to IDA
+        allows user to select a edit a function name and push changes to IDA
 
-        @param action: QAction*
+        @param action: QAction
         """
         _, item, model_index = action.data()
 
@@ -218,10 +250,9 @@ class CapaExplorerQtreeView(QtWidgets.QTreeView):
     def slot_custom_context_menu_requested(self, pos):
         """slot connected to custom context menu request
 
-        displays custom context menu to user containing action
-        relevant to the data item selected
+        displays custom context menu to user containing action relevant to the item selected
 
-        @param pos: TODO
+        @param pos: cursor position
         """
         model_index = self.indexAt(pos)
 
@@ -229,6 +260,7 @@ class CapaExplorerQtreeView(QtWidgets.QTreeView):
             return
 
         item = self.map_index_to_source_item(model_index)
+
         column = model_index.column()
         menu = None
 
@@ -243,9 +275,11 @@ class CapaExplorerQtreeView(QtWidgets.QTreeView):
         self.show_custom_context_menu(menu, pos)
 
     def slot_double_click(self, model_index):
-        """slot connected to double click event
+        """slot connected to double-click event
 
-        @param model_index: QModelIndex*
+        if address column clicked, navigate IDA to address, else un/expand item clicked
+
+        @param model_index: QModelIndex
         """
         if not model_index.isValid():
             return

capa/ida/plugin_helpers.py

@@ -1,99 +0,0 @@
-# Copyright (C) 2020 FireEye, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-
-import os
-import logging
-
-import idc
-import idaapi
-from PyQt5.QtCore import Qt
-from PyQt5.QtWidgets import QTreeWidgetItem, QTreeWidgetItemIterator
-
-CAPA_EXTENSION = ".capas"
-
-
-logger = logging.getLogger("capa_ida")
-
-
-def get_input_file(freeze=True):
-    """
-    get input file path
-
-        freeze (bool): if True, get freeze file if it exists
-    """
-    # try original file in same directory as idb/i64 without idb/i64 file extension
-    input_file = idc.get_idb_path()[:-4]
-
-    if freeze:
-        # use frozen file if it exists
-        freeze_file_cand = "%s%s" % (input_file, CAPA_EXTENSION)
-        if os.path.isfile(freeze_file_cand):
-            return freeze_file_cand
-
-    if not os.path.isfile(input_file):
-        # TM naming
-        input_file = "%s.mal_" % idc.get_idb_path()[:-4]
-        if not os.path.isfile(input_file):
-            input_file = idaapi.ask_file(0, "*.*", "Please specify input file.")
-    if not input_file:
-        raise ValueError("could not find input file")
-    return input_file
-
-
-def get_orig_color_feature_vas(vas):
-    orig_colors = {}
-    for va in vas:
-        orig_colors[va] = idc.get_color(va, idc.CIC_ITEM)
-    return orig_colors
-
-
-def reset_colors(orig_colors):
-    if orig_colors:
-        for va, color in orig_colors.iteritems():
-            idc.set_color(va, idc.CIC_ITEM, orig_colors[va])
-
-
-def reset_selection(tree):
-    iterator = QTreeWidgetItemIterator(tree, QTreeWidgetItemIterator.Checked)
-    while iterator.value():
-        item = iterator.value()
-        item.setCheckState(0, Qt.Unchecked)  # column, state
-        iterator += 1
-
-
-def get_disasm_line(va):
-    return idc.generate_disasm_line(va, idc.GENDSM_FORCE_CODE)
-
-
-def get_selected_items(tree, skip_level_1=False):
-    selected = []
-    iterator = QTreeWidgetItemIterator(tree, QTreeWidgetItemIterator.Checked)
-    while iterator.value():
-        item = iterator.value()
-        if skip_level_1:
-            # hacky way to check if item is at level 1, if so, skip
-            # alternative, check if text in disasm column
-            if item.parent() and item.parent().parent() is None:
-                iterator += 1
-                continue
-        if item.text(1):
-            # logger.debug('selected %s, %s', item.text(0), item.text(1))
-            selected.append(int(item.text(1), 0x10))
-        iterator += 1
-    return selected
-
-
-def add_child_item(parent, values, feature=None):
-    child = QTreeWidgetItem(parent)
-    child.setFlags(child.flags() | Qt.ItemIsTristate | Qt.ItemIsUserCheckable)
-    for i, v in enumerate(values):
-        child.setText(i, v)
-        if feature:
-            child.setData(0, 0x100, feature)
-        child.setCheckState(0, Qt.Unchecked)
-    return child

capa/main.py

@@ -446,7 +446,14 @@ def main(argv=None):
     parser = argparse.ArgumentParser(
         description=desc, epilog=epilog, formatter_class=argparse.RawDescriptionHelpFormatter
     )
-    parser.add_argument("sample", type=str, help="path to sample to analyze")
+    parser.add_argument(
+        # in #328 we noticed that the sample path is not handled correctly if it contains non-ASCII characters
+        # https://stackoverflow.com/a/22947334/ offers a solution and decoding using getfilesystemencoding works
+        # in our testing, however other sources suggest `sys.stdin.encoding` (https://stackoverflow.com/q/4012571/)
+        "sample",
+        type=lambda s: s.decode(sys.getfilesystemencoding()),
+        help="path to sample to analyze",
+    )
     parser.add_argument("--version", action="version", version="%(prog)s {:s}".format(capa.version.__version__))
     parser.add_argument(
         "-r",
@@ -493,7 +500,9 @@ def main(argv=None):
     try:
         taste = get_file_taste(args.sample)
     except IOError as e:
-        logger.error("%s", str(e))
+        # per our research there's not a programmatic way to render the IOError with non-ASCII filename unless we
+        # handle the IOError separately and reach into the args
+        logger.error("%s", e.args[0])
         return -1
 
     # py2 doesn't know about cp65001, which is a variant of utf-8 on windows
@@ -536,7 +545,13 @@ def main(argv=None):
     try:
         rules = get_rules(rules_path, disable_progress=args.quiet)
         rules = capa.rules.RuleSet(rules)
-        logger.debug("successfully loaded %s rules", len(rules))
+        logger.debug(
+            "successfully loaded %s rules",
+            # during the load of the RuleSet, we extract subscope statements into their own rules
+            # that are subsequently `match`ed upon. this inflates the total rule count.
+            # so, filter out the subscope rules when reporting total number of loaded rules.
+            len(filter(lambda r: "capa/subscope-rule" not in r.meta, rules.rules.values())),
+        )
         if args.tag:
             rules = rules.filter_rules_by_meta(args.tag)
             logger.debug("selected %s rules", len(rules))

capa/render/default.py

@@ -36,6 +36,34 @@ def render_meta(doc, ostream):
     ostream.write("\n")
 
 
+def find_subrule_matches(doc):
+    """
+    collect the rule names that have been matched as a subrule match.
+    this way we can avoid displaying entries for things that are too specific.
+    """
+    matches = set([])
+
+    def rec(node):
+        if not node["success"]:
+            # there's probably a bug here for rules that do `not: match: ...`
+            # but we don't have any examples of this yet
+            return
+
+        elif node["node"]["type"] == "statement":
+            for child in node["children"]:
+                rec(child)
+
+        elif node["node"]["type"] == "feature":
+            if node["node"]["feature"]["type"] == "match":
+                matches.add(node["node"]["feature"]["match"])
+
+    for rule in rutils.capability_rules(doc):
+        for node in rule["matches"].values():
+            rec(node)
+
+    return matches
+
+
 def render_capabilities(doc, ostream):
     """
     example::
@@ -48,8 +76,16 @@ def render_capabilities(doc, ostream):
         | ...                                                   | ...                                             |
         +-------------------------------------------------------+-------------------------------------------------+
     """
+    subrule_matches = find_subrule_matches(doc)
+
     rows = []
     for rule in rutils.capability_rules(doc):
+        if rule["meta"]["name"] in subrule_matches:
+            # rules that are also matched by other rules should not get rendered by default.
+            # this cuts down on the amount of output while giving approx the same detail.
+            # see #224
+            continue
+
         count = len(rule["matches"])
         if count == 1:
             capability = rutils.bold(rule["meta"]["name"])
@@ -125,6 +161,65 @@ def render_attack(doc, ostream):
         ostream.write("\n")
 
 
+def render_mbc(doc, ostream):
+    """
+    example::
+
+        +--------------------------+------------------------------------------------------------+
+        | MBC Objective            | MBC Behavior                                               |
+        |--------------------------+------------------------------------------------------------|
+        | ANTI-BEHAVIORAL ANALYSIS | Virtual Machine Detection::Instruction Testing [B0009.029] |
+        | COLLECTION               | Keylogging::Polling [F0002.002]                            |
+        | COMMUNICATION            | Interprocess Communication::Create Pipe [C0003.001]        |
+        |                          | Interprocess Communication::Write Pipe [C0003.004]         |
+        | IMPACT                   | Remote Access::Reverse Shell [B0022.001]                   |
+        +--------------------------+------------------------------------------------------------+
+    """
+    objectives = collections.defaultdict(set)
+    for rule in rutils.capability_rules(doc):
+        if not rule["meta"].get("mbc"):
+            continue
+
+        mbcs = rule["meta"]["mbc"]
+        if not isinstance(mbcs, list):
+            raise ValueError("invalid rule: MBC mapping is not a list")
+
+        for mbc in mbcs:
+            objective, _, rest = mbc.partition("::")
+            if "::" in rest:
+                behavior, _, rest = rest.partition("::")
+                method, _, id = rest.rpartition(" ")
+                objectives[objective].add((behavior, method, id))
+            else:
+                behavior, _, id = rest.rpartition(" ")
+                objectives[objective].add((behavior, id))
+
+    rows = []
+    for objective, behaviors in sorted(objectives.items()):
+        inner_rows = []
+        for spec in sorted(behaviors):
+            if len(spec) == 2:
+                behavior, id = spec
+                inner_rows.append("%s %s" % (rutils.bold(behavior), id))
+            elif len(spec) == 3:
+                behavior, method, id = spec
+                inner_rows.append("%s::%s %s" % (rutils.bold(behavior), method, id))
+            else:
+                raise RuntimeError("unexpected MBC spec format")
+        rows.append(
+            (
+                rutils.bold(objective.upper()),
+                "\n".join(inner_rows),
+            )
+        )
+
+    if rows:
+        ostream.write(
+            tabulate.tabulate(rows, headers=[width("MBC Objective", 25), width("MBC Behavior", 75)], tablefmt="psql")
+        )
+        ostream.write("\n")
+
+
 def render_default(doc):
     ostream = rutils.StringIO()
 
@@ -132,6 +227,8 @@ def render_default(doc):
     ostream.write("\n")
     render_attack(doc, ostream)
     ostream.write("\n")
+    render_mbc(doc, ostream)
+    ostream.write("\n")
     render_capabilities(doc, ostream)
 
     return ostream.getvalue()

capa/rules.py

@@ -12,7 +12,13 @@ import logging
 import binascii
 import functools
 
+try:
+    from functools import lru_cache
+except ImportError:
+    from backports.functools_lru_cache import lru_cache
+
 import six
+import yaml
 import ruamel.yaml
 
 import capa.engine
@@ -25,7 +31,6 @@ from capa.features import MAX_BYTES_FEATURE_SIZE
 
 logger = logging.getLogger(__name__)
 
-
 # these are the standard metadata fields, in the preferred order.
 # when reformatted, any custom keys will come after these.
 META_KEYS = (
@@ -271,27 +276,63 @@ def parse_description(s, value_type, description=None):
     return value, description
 
 
+def pop_statement_description_entry(d):
+    """
+    extracts the description for statements and removes the description entry from the document
+    a statement can only have one description
+
+    example:
+    the features definition
+      - or:
+        - description: statement description
+        - number: 1
+          description: feature description
+
+    becomes
+      <statement>: [
+        { "description": "statement description" },  <-- extracted here
+        { "number": 1, "description": "feature description" }
+      ]
+    """
+    if not isinstance(d, list):
+        return None
+
+    # identify child of form '{ "description": <description> }'
+    descriptions = list(filter(lambda c: isinstance(c, dict) and len(c) == 1 and "description" in c, d))
+    if len(descriptions) > 1:
+        raise InvalidRule("statements can only have one description")
+
+    if not descriptions:
+        return None
+
+    description = descriptions[0]
+    d.remove(description)
+
+    return description["description"]
+
+
 def build_statements(d, scope):
     if len(d.keys()) > 2:
         raise InvalidRule("too many statements")
 
     key = list(d.keys())[0]
+    description = pop_statement_description_entry(d[key])
     if key == "and":
-        return And([build_statements(dd, scope) for dd in d[key]], description=d.get("description"))
+        return And([build_statements(dd, scope) for dd in d[key]], description=description)
     elif key == "or":
-        return Or([build_statements(dd, scope) for dd in d[key]], description=d.get("description"))
+        return Or([build_statements(dd, scope) for dd in d[key]], description=description)
     elif key == "not":
         if len(d[key]) != 1:
             raise InvalidRule("not statement must have exactly one child statement")
-        return Not(build_statements(d[key][0], scope), description=d.get("description"))
+        return Not(build_statements(d[key][0], scope), description=description)
     elif key.endswith(" or more"):
         count = int(key[: -len("or more")])
-        return Some(count, [build_statements(dd, scope) for dd in d[key]], description=d.get("description"))
+        return Some(count, [build_statements(dd, scope) for dd in d[key]], description=description)
     elif key == "optional":
         # `optional` is an alias for `0 or more`
         # which is useful for documenting behaviors,
         # like with `write file`, we might say that `WriteFile` is optionally found alongside `CreateFileA`.
-        return Some(0, [build_statements(dd, scope) for dd in d[key]], description=d.get("description"))
+        return Some(0, [build_statements(dd, scope) for dd in d[key]], description=description)
 
     elif key == "function":
         if scope != FILE_SCOPE:
@@ -350,18 +391,18 @@ def build_statements(d, scope):
 
         count = d[key]
         if isinstance(count, int):
-            return Range(feature, min=count, max=count, description=d.get("description"))
+            return Range(feature, min=count, max=count, description=description)
         elif count.endswith(" or more"):
             min = parse_int(count[: -len(" or more")])
             max = None
-            return Range(feature, min=min, max=max, description=d.get("description"))
+            return Range(feature, min=min, max=max, description=description)
         elif count.endswith(" or fewer"):
             min = None
             max = parse_int(count[: -len(" or fewer")])
-            return Range(feature, min=min, max=max, description=d.get("description"))
+            return Range(feature, min=min, max=max, description=description)
         elif count.startswith("("):
             min, max = parse_range(count)
-            return Range(feature, min=min, max=max, description=d.get("description"))
+            return Range(feature, min=min, max=max, description=description)
         else:
             raise InvalidRule("unexpected range: %s" % (count))
     elif key == "string" and not isinstance(d[key], six.string_types):
@@ -385,26 +426,6 @@ def second(s):
     return s[1]
 
 
-# we use the ruamel.yaml parser because it supports roundtripping of documents with comments.
-yaml = ruamel.yaml.YAML(typ="rt")
-
-
-# use block mode, not inline json-like mode
-yaml.default_flow_style = False
-
-
-# indent lists by two spaces below their parent
-#
-#     features:
-#       - or:
-#         - mnemonic: aesdec
-#         - mnemonic: vaesdec
-yaml.indent(sequence=2, offset=2)
-
-# avoid word wrapping
-yaml.width = 4096
-
-
 class Rule(object):
     def __init__(self, name, scope, statement, meta, definition=""):
         super(Rule, self).__init__()
@@ -533,7 +554,7 @@ class Rule(object):
         return self.statement.evaluate(features)
 
     @classmethod
-    def from_dict(cls, d, s):
+    def from_dict(cls, d, definition):
         name = d["rule"]["meta"]["name"]
         # if scope is not specified, default to function scope.
         # this is probably the mode that rule authors will start with.
@@ -551,11 +572,52 @@ class Rule(object):
         if scope not in SUPPORTED_FEATURES.keys():
             raise InvalidRule("{:s} is not a supported scope".format(scope))
 
-        return cls(name, scope, build_statements(statements[0], scope), d["rule"]["meta"], s)
+        return cls(name, scope, build_statements(statements[0], scope), d["rule"]["meta"], definition)
+
+    @staticmethod
+    @lru_cache()
+    def _get_yaml_loader():
+        try:
+            # prefer to use CLoader to be fast, see #306
+            # on Linux, make sure you install libyaml-dev or similar
+            # on Windows, get WHLs from pyyaml.org/pypi
+            loader = yaml.CLoader
+            logger.debug("using libyaml CLoader.")
+        except:
+            loader = yaml.Loader
+            logger.debug("unable to import libyaml CLoader, falling back to Python yaml parser.")
+            logger.debug("this will be slower to load rules.")
+
+        return loader
+
+    @staticmethod
+    def _get_ruamel_yaml_parser():
+        # use ruamel to enable nice formatting
+
+        # we use the ruamel.yaml parser because it supports roundtripping of documents with comments.
+        y = ruamel.yaml.YAML(typ="rt")
+
+        # use block mode, not inline json-like mode
+        y.default_flow_style = False
+
+        # indent lists by two spaces below their parent
+        #
+        #     features:
+        #       - or:
+        #         - mnemonic: aesdec
+        #         - mnemonic: vaesdec
+        y.indent(sequence=2, offset=2)
+
+        # avoid word wrapping
+        y.width = 4096
+
+        return y
 
     @classmethod
     def from_yaml(cls, s):
-        return cls.from_dict(yaml.load(s), s)
+        # use pyyaml because it can be much faster than ruamel (pure python)
+        doc = yaml.load(s, Loader=cls._get_yaml_loader())
+        return cls.from_dict(doc, s)
 
     @classmethod
     def from_yaml_file(cls, path):
@@ -575,12 +637,25 @@ class Rule(object):
         # but not for rule logic.
         # programmatic generation of rules is not yet supported.
 
-        definition = yaml.load(self.definition)
-        # definition retains a reference to `meta`,
-        # so we're updating that in place.
-        definition["rule"]["meta"] = self.meta
-        meta = self.meta
+        # use ruamel because it supports round tripping.
+        # pyyaml will lose the existing ordering of rule statements.
+        definition = self._get_ruamel_yaml_parser().load(self.definition)
 
+        # we want to apply any updates that have been made to `meta`.
+        # so we would like to assigned it like this:
+        #
+        #     definition["rule"]["meta"] = self.meta
+        #
+        # however, `self.meta` is not ordered, its just a dict, so subsequent formatting doesn't work.
+        # so, we'll manually copy the keys over, re-using the existing ordereddict/CommentedMap
+        meta = definition["rule"]["meta"]
+        for k in meta.keys():
+            if k not in self.meta:
+                del meta[k]
+        for k, v in self.meta.items():
+            meta[k] = v
+
+        # the name and scope of the rule instance overrides anything in meta.
         meta["name"] = self.name
         meta["scope"] = self.scope
 
@@ -617,14 +692,32 @@ class Rule(object):
             del meta[key]
 
         ostream = six.BytesIO()
-        yaml.dump(definition, ostream)
+        self._get_ruamel_yaml_parser().dump(definition, ostream)
 
         for key, value in hidden_meta.items():
             if value is None:
                 continue
             meta[key] = value
 
-        return ostream.getvalue().decode("utf-8").rstrip("\n") + "\n"
+        doc = ostream.getvalue().decode("utf-8").rstrip("\n") + "\n"
+        # when we have something like:
+        #
+        #     and:
+        #       - string: foo
+        #         description: bar
+        #
+        # we want the `description` horizontally aligned with the start of the `string` (like above).
+        # however, ruamel will give us (which I don't think is even valid yaml):
+        #
+        #     and:
+        #       - string: foo
+        #      description: bar
+        #
+        # tweaking `ruamel.indent()` doesn't quite give us the control we want.
+        # so, add the two extra spaces that we've determined we need through experimentation.
+        # see #263
+        doc = doc.replace("  description:", "    description:")
+        return doc
 
 
 def get_rules_with_scope(rules, scope):

capa/version.py

@@ -1 +1 @@
-__version__ = "1.2.0"
+__version__ = "1.4.0"

doc/installation.md

@@ -22,12 +22,15 @@ By default, on MacOS Catalina or greater, Gatekeeper will block execution of the
 ![approve dialog](img/approve.png)
 
 ## Method 2: Using capa as a Python library
-To install capa as a Python library, you'll need to install a few dependencies, and then use `pip` to fetch the capa module.
-Note: this technique doesn't pull the default rule set, so you should check it out separately from [capa-rules](https://github.com/fireeye/capa-rules/) and pass the directory to the entrypoint using `-r`.
+To install capa as a Python library use `pip` to fetch the `flare-capa` module.
+
+#### *Note*:
+This method is appropriate for integrating capa in an existing project.
+This technique doesn't pull the default rule set, so you should check it out separately from [capa-rules](https://github.com/fireeye/capa-rules/) and pass the directory to the entrypoint using `-r` or set the rules path in the IDA Pro plugin.
+Alternatively, see Method 3 below.
 
 ### 1. Install capa module
-Second, use `pip` to install the capa module to your local Python environment. This fetches the library code to your computer but does not keep editable source files around for you to hack on. If you'd like to edit the source files, see below.
-`$ pip install https://github.com/fireeye/capa/archive/master.zip`
+Use `pip` to install the capa module to your local Python environment. This fetches the library code to your computer but does not keep editable source files around for you to hack on. If you'd like to edit the source files, see below. `$ pip install flare-capa`
 
 ### 2. Use capa
 You can now import the `capa` module from a Python script or use the IDA Pro plugins from the `capa/ida` directory. For more information please see the [usage](usage.md) documentation.

doc/usage.md

@@ -4,37 +4,10 @@ See `capa -h` for all supported arguments and usage examples.
 
 ## tips and tricks
 
-  - [match only rules by given author or namespace](#only-run-selected-rules)
-  - [IDA Pro capa explorer](#capa-explorer)
-  - [IDA Pro rule generator](#rule-generator)
-
 ### only run selected rules
 Use the `-t` option to run rules with the given metadata value (see the rule fields `rule.meta.*`).
 For example, `capa -t william.ballenthin@mandiant.com` runs rules that reference Willi's email address (probably as the author), or
 `capa -t communication` runs rules with the namespace `communication`.
 
-### IDA Pro integrations
-You can run capa from within IDA Pro. Run `capa/main.py` via `File - Script file...` (or ALT + F7). 
-When running in IDA, capa uses IDA's disassembly and file analysis as its backend. 
-These results may vary from the standalone version that uses vivisect.
-IDA's analysis is generally a bit faster and more thorough than vivisect's, so you might prefer this mode.
-
-When run under IDA, capa supports both Python 2 and Python 3 interpreters.
-If you encounter issues with your specific setup, please open a new [Issue](https://github.com/fireeye/capa/issues).
-
-Additionally, capa comes with an IDA Pro plugin located in the `capa/ida` directory: the explorer.
-
-#### capa explorer
-The capa explorer allows you to interactively display and browse capabilities capa identified in a binary.
-As you select rules or logic, capa will highlight the addresses that support its analysis conclusions.
-We like to use capa to help find the most interesting parts of a program, such as where the C2 mechanism might be.
-
-![capa explorer](img/capa_explorer.png)
-
-To install the plugin, you'll need to be running IDA Pro 7.4 or 7.5 with either Python 2 or Python 3.
-Next make sure pip commands are run using the Python install that is configured for your IDA install:
-
-  1. Only if running Python 2.7, run command `$ pip install https://github.com/williballenthin/vivisect/zipball/master`
-  2. Run `$ pip install .` from capa root directory
-  3. Open IDA and navigate to `File > Script file…` or `Alt+F7`
-  4. Navigate to `<capa_install_dir>\capa\ida\` and choose `ida_capa_explorer.py`
+### IDA Pro plugin: capa explorer
+Please check out the [capa explorer documentation](/capa/ida/plugin/README.md).

scripts/bulk-process.py

@@ -0,0 +1,247 @@
+#!/usr/bin/env python
+"""
+bulk-process
+
+Invoke capa recursively against a directory of samples
+and emit a JSON document mapping the file paths to their results.
+
+By default, this will use subprocesses for parallelism.
+Use `-n/--parallelism` to change the subprocess count from
+ the default of current CPU count.
+Use `--no-mp` to use threads instead of processes,
+ which is probably not useful unless you set `--parallelism=1`.
+
+example:
+
+    $ python scripts/bulk-process /tmp/suspicious
+    {
+      "/tmp/suspicious/suspicious.dll_": {
+        "rules": {
+          "encode data using XOR": {
+            "matches": {
+              "268440358": {
+              [...]
+      "/tmp/suspicious/1.dll_": { ... }
+      "/tmp/suspicious/2.dll_": { ... }
+    }
+
+
+usage:
+
+    usage: bulk-process.py [-h] [-r RULES] [-d] [-q] [-n PARALLELISM] [--no-mp]
+                           input
+
+    detect capabilities in programs.
+
+    positional arguments:
+      input                 Path to directory of files to recursively analyze
+
+    optional arguments:
+      -h, --help            show this help message and exit
+      -r RULES, --rules RULES
+                            Path to rule file or directory, use embedded rules by
+                            default
+      -d, --debug           Enable debugging output on STDERR
+      -q, --quiet           Disable all output but errors
+      -n PARALLELISM, --parallelism PARALLELISM
+                            parallelism factor
+      --no-mp               disable subprocesses
+
+Copyright (C) 2020 FireEye, Inc. All Rights Reserved.
+Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+You may obtain a copy of the License at: [package root]/LICENSE.txt
+Unless required by applicable law or agreed to in writing, software distributed under the License
+ is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and limitations under the License.
+"""
+import sys
+import json
+import logging
+import os.path
+import argparse
+import multiprocessing
+import multiprocessing.pool
+
+import capa
+import capa.main
+import capa.render
+
+logger = logging.getLogger("capa")
+
+
+def get_capa_results(args):
+    """
+    run capa against the file at the given path, using the given rules.
+
+    args is a tuple, containing:
+      rules (capa.rules.RuleSet): the rules to match
+      format (str): the name of the sample file format
+      path (str): the file system path to the sample to process
+
+    args is a tuple because i'm not quite sure how to unpack multiple arguments using `map`.
+
+    returns an dict with two required keys:
+      path (str): the file system path of the sample to process
+      status (str): either "error" or "ok"
+
+    when status == "error", then a human readable message is found in property "error".
+    when status == "ok", then the capa results are found in the property "ok".
+
+    the capa results are a dictionary with the following keys:
+      meta (dict): the meta analysis results
+      capabilities (dict): the matched capabilities and their result objects
+    """
+    rules, format, path = args
+    logger.info("computing capa results for: %s", path)
+    try:
+        extractor = capa.main.get_extractor(path, format, disable_progress=True)
+    except capa.main.UnsupportedFormatError:
+        # i'm 100% sure if multiprocessing will reliably raise exceptions across process boundaries.
+        # so instead, return an object with explicit success/failure status.
+        #
+        # if success, then status=ok, and results found in property "ok"
+        # if error, then status=error, and human readable message in property "error"
+        return {
+            "path": path,
+            "status": "error",
+            "error": "input file does not appear to be a PE file: %s" % path,
+        }
+    except capa.main.UnsupportedRuntimeError:
+        return {
+            "path": path,
+            "status": "error",
+            "error": "unsupported runtime or Python interpreter",
+        }
+    except Exception as e:
+        return {
+            "path": path,
+            "status": "error",
+            "error": "unexpected error: %s" % (e),
+        }
+
+    meta = capa.main.collect_metadata("", path, "", format, extractor)
+    capabilities, counts = capa.main.find_capabilities(rules, extractor, disable_progress=True)
+    meta["analysis"].update(counts)
+
+    return {
+        "path": path,
+        "status": "ok",
+        "ok": {
+            "meta": meta,
+            "capabilities": capabilities,
+        },
+    }
+
+
+def main(argv=None):
+    if argv is None:
+        argv = sys.argv[1:]
+
+        parser = argparse.ArgumentParser(description="detect capabilities in programs.")
+        parser.add_argument("input", type=str, help="Path to directory of files to recursively analyze")
+        parser.add_argument(
+            "-r",
+            "--rules",
+            type=str,
+            default="(embedded rules)",
+            help="Path to rule file or directory, use embedded rules by default",
+        )
+        parser.add_argument("-d", "--debug", action="store_true", help="Enable debugging output on STDERR")
+        parser.add_argument("-q", "--quiet", action="store_true", help="Disable all output but errors")
+        parser.add_argument(
+            "-n", "--parallelism", type=int, default=multiprocessing.cpu_count(), help="parallelism factor"
+        )
+        parser.add_argument("--no-mp", action="store_true", help="disable subprocesses")
+        args = parser.parse_args(args=argv)
+
+        if args.quiet:
+            logging.basicConfig(level=logging.ERROR)
+            logging.getLogger().setLevel(logging.ERROR)
+        elif args.debug:
+            logging.basicConfig(level=logging.DEBUG)
+            logging.getLogger().setLevel(logging.DEBUG)
+        else:
+            logging.basicConfig(level=logging.INFO)
+            logging.getLogger().setLevel(logging.INFO)
+
+        # disable vivisect-related logging, it's verbose and not relevant for capa users
+        capa.main.set_vivisect_log_level(logging.CRITICAL)
+
+        # py2 doesn't know about cp65001, which is a variant of utf-8 on windows
+        # tqdm bails when trying to render the progress bar in this setup.
+        # because cp65001 is utf-8, we just map that codepage to the utf-8 codec.
+        # see #380 and: https://stackoverflow.com/a/3259271/87207
+        import codecs
+
+        codecs.register(lambda name: codecs.lookup("utf-8") if name == "cp65001" else None)
+
+        if args.rules == "(embedded rules)":
+            logger.info("using default embedded rules")
+            logger.debug("detected running from source")
+            args.rules = os.path.join(os.path.dirname(__file__), "..", "rules")
+            logger.debug("default rule path (source method): %s", args.rules)
+        else:
+            logger.info("using rules path: %s", args.rules)
+
+        try:
+            rules = capa.main.get_rules(args.rules)
+            rules = capa.rules.RuleSet(rules)
+            logger.info("successfully loaded %s rules", len(rules))
+        except (IOError, capa.rules.InvalidRule, capa.rules.InvalidRuleSet) as e:
+            logger.error("%s", str(e))
+            return -1
+
+        samples = []
+        for (base, directories, files) in os.walk(args.input):
+            for file in files:
+                samples.append(os.path.join(base, file))
+
+        def pmap(f, args, parallelism=multiprocessing.cpu_count()):
+            """apply the given function f to the given args using subprocesses"""
+            return multiprocessing.Pool(parallelism).imap(f, args)
+
+        def tmap(f, args, parallelism=multiprocessing.cpu_count()):
+            """apply the given function f to the given args using threads"""
+            return multiprocessing.pool.ThreadPool(parallelism).imap(f, args)
+
+        def map(f, args, parallelism=None):
+            """apply the given function f to the given args in the current thread"""
+            for arg in args:
+                yield f(arg)
+
+        if args.no_mp:
+            if args.parallelism == 1:
+                logger.debug("using current thread mapper")
+                mapper = map
+            else:
+                logger.debug("using threading mapper")
+                mapper = tmap
+        else:
+            logger.debug("using process mapper")
+            mapper = pmap
+
+        results = {}
+        for result in mapper(
+            get_capa_results, [(rules, "pe", sample) for sample in samples], parallelism=args.parallelism
+        ):
+            if result["status"] == "error":
+                logger.warning(result["error"])
+            elif result["status"] == "ok":
+                meta = result["ok"]["meta"]
+                capabilities = result["ok"]["capabilities"]
+                # our renderer expects to emit a json document for a single sample
+                # so we deserialize the json document, store it in a larger dict, and we'll subsequently re-encode.
+                results[result["path"]] = json.loads(capa.render.render_json(meta, rules, capabilities))
+            else:
+                raise ValueError("unexpected status: %s" % (result["status"]))
+
+        print(json.dumps(results))
+
+        logger.info("done.")
+
+        return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/import-to-ida.py

@@ -33,6 +33,7 @@ import logging
 
 import idc
 import idautils
+import ida_funcs
 import ida_idaapi
 import ida_kernwin
 

scripts/show-features.py

@@ -1,5 +1,13 @@
 #!/usr/bin/env python2
 """
+Copyright (C) 2020 FireEye, Inc. All Rights Reserved.
+Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+You may obtain a copy of the License at: [package root]/LICENSE.txt
+Unless required by applicable law or agreed to in writing, software distributed under the License
+ is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and limitations under the License.
+
 show-features
 
 Show the features that capa extracts from the given sample,
@@ -55,14 +63,6 @@ Example::
     insn: 0x10001027: number(0x1)
     insn: 0x10001027: mnemonic(shl)
     ...
-
-Copyright (C) 2020 FireEye, Inc. All Rights Reserved.
-Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
-You may obtain a copy of the License at: [package root]/LICENSE.txt
-Unless required by applicable law or agreed to in writing, software distributed under the License
- is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and limitations under the License.
 """
 import sys
 import logging
@@ -89,12 +89,12 @@ def main(argv=None):
     ]
     format_help = ", ".join(["%s: %s" % (f[0], f[1]) for f in formats])
 
-    parser = argparse.ArgumentParser(description="detect capabilities in programs.")
+    parser = argparse.ArgumentParser(description="Show the features that capa extracts from the given sample")
     parser.add_argument("sample", type=str, help="Path to sample to analyze")
     parser.add_argument(
         "-f", "--format", choices=[f[0] for f in formats], default="auto", help="Select sample format, %s" % format_help
     )
-    parser.add_argument("-F", "--function", type=lambda x: int(x, 0), help="Show features for specific function")
+    parser.add_argument("-F", "--function", type=lambda x: int(x, 0x10), help="Show features for specific function")
     args = parser.parse_args(args=argv)
 
     logging.basicConfig(level=logging.INFO)
@@ -122,6 +122,50 @@ def main(argv=None):
         else:
             functions = filter(lambda f: f.va == args.function, functions)
 
+            if args.function not in [f.va for f in functions]:
+                print("0x%X not a function, creating it" % args.function)
+                vw.makeFunction(args.function)
+                functions = extractor.get_functions()
+                functions = filter(lambda f: f.va == args.function, functions)
+
+        if len(functions) == 0:
+            print("0x%X not a function")
+            return -1
+
+    print_features(functions, extractor)
+
+    return 0
+
+
+def ida_main():
+    function = idc.get_func_attr(idc.here(), idc.FUNCATTR_START)
+    print("getting features for current function 0x%X" % function)
+
+    extractor = capa.features.extractors.ida.IdaFeatureExtractor()
+
+    if not function:
+        for feature, va in extractor.extract_file_features():
+            if va:
+                print("file: 0x%08x: %s" % (va, feature))
+            else:
+                print("file: 0x00000000: %s" % (feature))
+        return
+
+    functions = extractor.get_functions()
+
+    if function:
+        functions = filter(lambda f: f.start_ea == function, functions)
+
+        if len(functions) == 0:
+            print("0x%X not a function" % function)
+            return -1
+
+    print_features(functions, extractor)
+
+    return 0
+
+
+def print_features(functions, extractor):
     for f in functions:
         for feature, va in extractor.extract_function_features(f):
             print("func: 0x%08x: %s" % (va, feature))
@@ -138,8 +182,9 @@ def main(argv=None):
                         # may be an issue while piping to less and encountering non-ascii characters
                         continue
 
-    return 0
-
 
 if __name__ == "__main__":
-    sys.exit(main())
+    if capa.main.is_runtime_ida():
+        ida_main()
+    else:
+        sys.exit(main())

setup.py

@@ -12,7 +12,18 @@ import sys
 import setuptools
 
 # halo==0.0.30 is the last version to support py2.7
-requirements = ["six", "tqdm", "pyyaml", "tabulate", "colorama", "termcolor", "ruamel.yaml", "wcwidth", "halo==0.0.30"]
+requirements = [
+    "six",
+    "tqdm",
+    "pyyaml",
+    "tabulate",
+    "colorama",
+    "termcolor",
+    "ruamel.yaml",
+    "wcwidth",
+    "halo==0.0.30",
+    "ida-settings==2.1.0",
+]
 
 if sys.version_info >= (3, 0):
     # py3
@@ -20,7 +31,7 @@ if sys.version_info >= (3, 0):
 else:
     # py2
     requirements.append("enum34==1.1.6")  # v1.1.6 is needed by halo 0.0.30 / spinners 0.0.24
-    requirements.append("vivisect @ https://github.com/williballenthin/vivisect/tarball/v0.0.20200804#egg=vivisect")
+    requirements.append("vivisect==0.1.0")
     requirements.append("viv-utils")
     requirements.append("networkx==2.2")  # v2.2 is last version supported by Python 2.7
     requirements.append("backports.functools-lru-cache")
@@ -28,18 +39,30 @@ else:
 # this sets __version__
 # via: http://stackoverflow.com/a/7071358/87207
 # and: http://stackoverflow.com/a/2073599/87207
-with open(os.path.join("capa", "version.py"), "rb") as f:
+with open(os.path.join("capa", "version.py"), "r") as f:
     exec(f.read())
 
 
+# via: https://packaging.python.org/guides/making-a-pypi-friendly-readme/
+this_directory = os.path.abspath(os.path.dirname(__file__))
+with open(os.path.join(this_directory, "README.md"), "r") as f:
+    long_description = f.read()
+
+
 setuptools.setup(
     name="flare-capa",
     version=__version__,
     description="The FLARE team's open-source tool to identify capabilities in executable files.",
-    long_description="",
+    long_description=long_description,
+    long_description_content_type="text/markdown",
     author="Willi Ballenthin, Moritz Raabe",
     author_email="william.ballenthin@mandiant.com, moritz.raabe@mandiant.com",
     url="https://www.github.com/fireeye/capa",
+    project_urls={
+        "Documentation": "https://github.com/fireeye/capa/tree/master/doc",
+        "Rules": "https://github.com/fireeye/capa-rules",
+        "Rules Documentation": "https://github.com/fireeye/capa-rules/tree/master/doc",
+    },
     packages=setuptools.find_packages(exclude=["tests"]),
     package_dir={"capa": "capa"},
     entry_points={
@@ -61,12 +84,15 @@ setuptools.setup(
         ]
     },
     zip_safe=False,
-    keywords="capa",
+    keywords="capa malware analysis capability detection FLARE",
     classifiers=[
-        "Development Status :: 3 - Alpha",
+        "Development Status :: 5 - Production/Stable",
         "Intended Audience :: Developers",
+        "Intended Audience :: Information Technology",
+        "License :: OSI Approved :: Apache Software License",
         "Natural Language :: English",
-        "Programming Language :: Python :: 2",
+        "Programming Language :: Python :: 2.7",
         "Programming Language :: Python :: 3",
+        "Topic :: Security",
     ],
 )

tests/fixtures.py

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 # Copyright (C) 2020 FireEye, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
@@ -122,6 +123,8 @@ def get_data_path_by_name(name):
         return os.path.join(CD, "data", "kernel32-64.dll_")
     elif name == "pma12-04":
         return os.path.join(CD, "data", "Practical Malware Analysis Lab 12-04.exe_")
+    elif name == "pma16-01":
+        return os.path.join(CD, "data", "Practical Malware Analysis Lab 16-01.exe_")
     elif name == "pma21-01":
         return os.path.join(CD, "data", "Practical Malware Analysis Lab 21-01.exe_")
     elif name == "al-khaser x86":
@@ -140,6 +143,12 @@ def get_data_path_by_name(name):
         return os.path.join(CD, "data", "bfb9b5391a13d0afd787e87ab90f14f5.dll_")
     elif name.startswith("c9188"):
         return os.path.join(CD, "data", "c91887d861d9bd4a5872249b641bc9f9.exe_")
+    elif name.startswith("64d9f"):
+        return os.path.join(CD, "data", "64d9f7d96b99467f36e22fada623c3bb.dll_")
+    elif name.startswith("82bf6"):
+        return os.path.join(CD, "data", "82BF6347ACF15E5D883715DC289D8A2B.exe_")
+    elif name.startswith("pingtaest"):
+        return os.path.join(CD, "data", "ping_täst.exe_")
     else:
         raise ValueError("unexpected sample fixture")
 
@@ -154,6 +163,8 @@ def get_sample_md5_by_name(name):
         return "a8565440629ac87f6fef7d588fe3ff0f"
     elif name == "pma12-04":
         return "56bed8249e7c2982a90e54e1e55391a2"
+    elif name == "pma16-01":
+        return "7faafc7e4a5c736ebfee6abbbc812d80"
     elif name == "pma21-01":
         return "c8403fb05244e23a7931c766409b5e22"
     elif name == "al-khaser x86":
@@ -172,6 +183,10 @@ def get_sample_md5_by_name(name):
         return "bfb9b5391a13d0afd787e87ab90f14f5"
     elif name.startswith("c9188"):
         return "c91887d861d9bd4a5872249b641bc9f9"
+    elif name.startswith("64d9f"):
+        return "64d9f7d96b99467f36e22fada623c3bb"
+    elif name.startswith("82bf6"):
+        return "82bf6347acf15e5d883715dc289d8a2b"
     else:
         raise ValueError("unexpected sample fixture")
 
@@ -281,6 +296,10 @@ FEATURE_PRESENCE_TESTS = [
     ("mimikatz", "file", capa.features.file.Import("#11"), False),
     ("mimikatz", "file", capa.features.file.Import("#nope"), False),
     ("mimikatz", "file", capa.features.file.Import("nope"), False),
+    ("mimikatz", "file", capa.features.file.Import("advapi32.CryptAcquireContextW"), True),
+    ("mimikatz", "file", capa.features.file.Import("advapi32.CryptAcquireContext"), True),
+    ("mimikatz", "file", capa.features.file.Import("CryptAcquireContextW"), True),
+    ("mimikatz", "file", capa.features.file.Import("CryptAcquireContext"), True),
     # function/characteristic(loop)
     ("mimikatz", "function=0x401517", capa.features.Characteristic("loop"), True),
     ("mimikatz", "function=0x401000", capa.features.Characteristic("loop"), False),
@@ -313,6 +332,8 @@ FEATURE_PRESENCE_TESTS = [
     ("mimikatz", "function=0x40105D", capa.features.insn.Offset(0x0), True),
     ("mimikatz", "function=0x40105D", capa.features.insn.Offset(0x4), True),
     ("mimikatz", "function=0x40105D", capa.features.insn.Offset(0xC), True),
+    # insn/offset, issue #276
+    ("64d9f", "function=0x10001510,bb=0x100015B0", capa.features.insn.Offset(0x4000), True),
     # insn/offset: stack references
     ("mimikatz", "function=0x40105D", capa.features.insn.Offset(0x8), False),
     ("mimikatz", "function=0x40105D", capa.features.insn.Offset(0x10), False),
@@ -355,6 +376,12 @@ FEATURE_PRESENCE_TESTS = [
         True,
     ),
     ("kernel32-64", "function=0x1800202B0", capa.features.insn.API("RtlCaptureContext"), True),
+    # insn/api: x64 nested thunk
+    ("82bf6", "function=0x140059342", capa.features.insn.API("ElfClearEventLogFile"), True),
+    # insn/api: call via jmp
+    ("mimikatz", "function=0x40B3C6", capa.features.insn.API("LocalFree"), True),
+    ("c91887...", "function=0x40156F", capa.features.insn.API("CloseClipboard"), True),
+    # TODO ignore thunk functions that call via jmp?
     # insn/api: resolve indirect calls
     ("c91887...", "function=0x401A77", capa.features.insn.API("kernel32.CreatePipe"), True),
     ("c91887...", "function=0x401A77", capa.features.insn.API("kernel32.SetHandleInformation"), True),
@@ -365,6 +392,9 @@ FEATURE_PRESENCE_TESTS = [
     ("mimikatz", "function=0x40105D", capa.features.String("SCardTransmit"), True),
     ("mimikatz", "function=0x40105D", capa.features.String("ACR  > "), True),
     ("mimikatz", "function=0x40105D", capa.features.String("nope"), False),
+    # insn/regex, issue #262
+    ("pma16-01", "function=0x4021B0", capa.features.Regex("HTTP/1.0"), True),
+    ("pma16-01", "function=0x4021B0", capa.features.Regex("www.practicalmalwareanalysis.com"), False),
     # insn/string, pointer to string
     ("mimikatz", "function=0x44EDEF", capa.features.String("INPUTEVENT"), True),
     # insn/bytes
@@ -404,6 +434,12 @@ FEATURE_PRESENCE_TESTS = [
     ("mimikatz", "function=0x4556E5", capa.features.Characteristic("calls to"), False),
 ]
 
+FEATURE_PRESENCE_TESTS_IDA = [
+    # file/imports
+    # IDA can recover more names of APIs imported by ordinal
+    ("mimikatz", "file", capa.features.file.Import("cabinet.FCIAddFile"), True),
+]
+
 FEATURE_COUNT_TESTS = [
     ("mimikatz", "function=0x40E5C2", capa.features.basicblock.BasicBlock(), 7),
     ("mimikatz", "function=0x4702FD", capa.features.Characteristic("calls from"), 0),
@@ -476,6 +512,11 @@ def pma12_04_extractor():
     return get_extractor(get_data_path_by_name("pma12-04"))
 
 
+@pytest.fixture
+def pma16_01_extractor():
+    return get_extractor(get_data_path_by_name("pma16-01"))
+
+
 @pytest.fixture
 def bfb9b_extractor():
     return get_extractor(get_data_path_by_name("bfb9b..."))
@@ -504,3 +545,8 @@ def z499c2_extractor():
 @pytest.fixture
 def al_khaser_x86_extractor():
     return get_extractor(get_data_path_by_name("al-khaser x86"))
+
+
+@pytest.fixture
+def pingtaest_extractor():
+    return get_extractor(get_data_path_by_name("pingtaest"))

tests/test_fmt.py

@@ -92,6 +92,8 @@ def test_rule_reformat_order():
 
 
 def test_rule_reformat_meta_update():
+    # test updating the rule content after parsing
+
     rule = textwrap.dedent(
         """
         rule:
@@ -112,3 +114,24 @@ def test_rule_reformat_meta_update():
     rule = capa.rules.Rule.from_yaml(rule)
     rule.name = "test rule"
     assert rule.to_yaml() == EXPECTED
+
+
+def test_rule_reformat_string_description():
+    # the `description` should be aligned with the preceding feature name.
+    # see #263
+    src = textwrap.dedent(
+        """
+        rule:
+          meta:
+            name: test rule
+            author: user@domain.com
+            scope: function
+          features:
+            - and:
+              - string: foo
+                description: bar
+        """
+    ).lstrip()
+
+    rule = capa.rules.Rule.from_yaml(src)
+    assert rule.to_yaml() == src

tests/test_ida_features.py

@@ -44,7 +44,7 @@ def get_ida_extractor(_path):
 
 @pytest.mark.skip(reason="IDA Pro tests must be run within IDA")
 def test_ida_features():
-    for (sample, scope, feature, expected) in FEATURE_PRESENCE_TESTS:
+    for (sample, scope, feature, expected) in FEATURE_PRESENCE_TESTS + FEATURE_PRESENCE_TESTS_IDA:
         id = make_test_id((sample, scope, feature, expected))
 
         try:

tests/test_main.py

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 # Copyright (C) 2020 FireEye, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
@@ -57,6 +58,30 @@ def test_main_single_rule(z9324d_extractor, tmpdir):
     )
 
 
+@pytest.mark.xfail(sys.version_info >= (3, 0), reason="vivsect only works on py2")
+def test_main_non_ascii_filename(pingtaest_extractor, tmpdir, capsys):
+    # on py2.7, need to be careful about str (which can hold bytes)
+    #  vs unicode (which is only unicode characters).
+    # on py3, this should not be needed.
+    #
+    # here we print a string with unicode characters in it
+    # (specifically, a byte string with utf-8 bytes in it, see file encoding)
+    assert capa.main.main(["-q", pingtaest_extractor.path]) == 0
+
+    std = capsys.readouterr()
+    # but here, we have to use a unicode instance,
+    # because capsys has decoded the output for us.
+    assert pingtaest_extractor.path.decode("utf-8") in std.out
+
+
+@pytest.mark.xfail(sys.version_info >= (3, 0), reason="vivsect only works on py2")
+def test_main_non_ascii_filename_nonexistent(tmpdir, caplog):
+    NON_ASCII_FILENAME = "täst_not_there.exe"
+    assert capa.main.main(["-q", NON_ASCII_FILENAME]) == -1
+
+    assert NON_ASCII_FILENAME.decode("utf-8") in caplog.text
+
+
 @pytest.mark.xfail(sys.version_info >= (3, 0), reason="vivsect only works on py2")
 def test_main_shellcode(z499c2_extractor):
     path = z499c2_extractor.path
@@ -309,3 +334,40 @@ def test_count_bb(z9324d_extractor):
     )
     capabilities, meta = capa.main.find_capabilities(rules, z9324d_extractor)
     assert "count bb" in capabilities
+
+
+@pytest.mark.xfail(sys.version_info >= (3, 0), reason="vivsect only works on py2")
+def test_fix262(pma16_01_extractor, capsys):
+    # tests rules can be loaded successfully and all output modes
+    path = pma16_01_extractor.path
+    assert capa.main.main([path, "-vv", "-t", "send HTTP request", "-q"]) == 0
+
+    std = capsys.readouterr()
+    assert "HTTP/1.0" in std.out
+    assert "www.practicalmalwareanalysis.com" not in std.out
+
+
+@pytest.mark.xfail(sys.version_info >= (3, 0), reason="vivsect only works on py2")
+def test_not_render_rules_also_matched(z9324d_extractor, capsys):
+    # rules that are also matched by other rules should not get rendered by default.
+    # this cuts down on the amount of output while giving approx the same detail.
+    # see #224
+    path = z9324d_extractor.path
+
+    # `act as TCP client` matches on
+    # `connect TCP client` matches on
+    # `create TCP socket`
+    #
+    # so only `act as TCP client` should be displayed
+    assert capa.main.main([path]) == 0
+    std = capsys.readouterr()
+    assert "act as TCP client" in std.out
+    assert "connect TCP socket" not in std.out
+    assert "create TCP socket" not in std.out
+
+    # this strategy only applies to the default renderer, not any verbose renderer
+    assert capa.main.main([path, "-v"]) == 0
+    std = capsys.readouterr()
+    assert "act as TCP client" in std.out
+    assert "connect TCP socket" in std.out
+    assert "create TCP socket" in std.out

tests/test_rules.py

@@ -69,46 +69,63 @@ def test_rule_yaml_complex():
     assert r.evaluate({Number(6): {1}, Number(7): {1}, Number(8): {1}}) == False
 
 
-def test_rule_yaml_descriptions():
+def test_rule_descriptions():
     rule = textwrap.dedent(
         """
         rule:
-            meta:
-                name: test rule
-            features:
+          meta:
+            name: test rule
+          features:
+            - and:
+              - description: and description
+              - number: 1 = number description
+              - string: mystring
+                description: string description
+              - string: '/myregex/'
+                description: regex description
+              # TODO - count(number(2 = number description)): 2
+              - or:
+                - description: or description
                 - and:
-                    - number: 1 = This is the number 1
-                    - string: This program cannot be run in DOS mode.
-                      description: MS-DOS stub message
-                    - string: '/SELECT.*FROM.*WHERE/i'
-                      description: SQL WHERE Clause
-                    - count(number(2 = AF_INET/SOCK_DGRAM)): 2
-                    - or:
-                        - and:
-                            - offset: 0x50 = IMAGE_NT_HEADERS.OptionalHeader.SizeOfImage
-                            - offset: 0x34 = IMAGE_NT_HEADERS.OptionalHeader.ImageBase
-                          description: 32-bits
-                        - and:
-                            - offset: 0x50 = IMAGE_NT_HEADERS64.OptionalHeader.SizeOfImage
-                            - offset: 0x30 = IMAGE_NT_HEADERS64.OptionalHeader.ImageBase
-                          description: 64-bits
-                      description: PE headers offsets
+                  - offset: 0x50 = offset description
+                  - offset: 0x34 = offset description
+                  - description: and description
+                - and:
+                  - description: and description
+                  - offset/x64: 0x50 = offset/x64 description
+                  - offset/x64: 0x30 = offset/x64 description
         """
     )
     r = capa.rules.Rule.from_yaml(rule)
-    assert (
-        r.evaluate(
-            {
-                Number(1): {1},
-                Number(2): {2, 3},
-                String("This program cannot be run in DOS mode."): {4},
-                String("SELECT password FROM hidden_table WHERE user == admin"): {5},
-                Offset(0x50): {6},
-                Offset(0x30): {7},
-            }
+
+    def rec(statement):
+        if isinstance(statement, capa.engine.Statement):
+            assert statement.description == statement.name.lower() + " description"
+            for child in statement.get_children():
+                rec(child)
+        else:
+            assert statement.description == statement.name + " description"
+
+    rec(r.statement)
+
+
+def test_invalid_rule_statement_descriptions():
+    # statements can only have one description
+    with pytest.raises(capa.rules.InvalidRule):
+        capa.rules.Rule.from_yaml(
+            textwrap.dedent(
+                """
+                rule:
+                  meta:
+                    name: test rule
+                  features:
+                    - or:
+                      - number: 1 = This is the number 1
+                      - description: description
+                      - description: another description (invalid)
+                """
+            )
         )
-        == True
-    )
 
 
 def test_rule_yaml_not():