Merge branch 'master' into backend-ghidra

.github/flake8.ini

@@ -0,0 +1,29 @@
+[flake8]
+max-line-length = 120
+
+extend-ignore =
+    # E203: whitespace before ':'  (black does this)
+    E203,
+    # F401: `foo` imported but unused  (prefer ruff)
+    F401,
+    # F811 Redefinition of unused `foo`  (prefer ruff)
+    F811,
+    # E501 line too long  (prefer black)
+    E501,
+    # B010 Do not call setattr with a constant attribute value
+    B010,
+    # G200 Logging statement uses exception in arguments
+    G200
+
+
+per-file-ignores =
+    # T201 print found.
+    #
+    # scripts are meant to print output
+    scripts/*: T201
+    # capa.exe is meant to print output
+    capa/main.py: T201
+    # IDA tests emit results to output window so need to print
+    tests/test_ida_features.py: T201
+    # utility used to find the Binary Ninja API via invoking python.exe
+    capa/features/extractors/binja/find_binja_api.py: T201

.github/ruff.toml

@@ -1,10 +1,61 @@
-# Enable pycodestyle (`E`) codes
-select = ["E"]
+# Enable the pycodestyle (`E`) and Pyflakes (`F`) rules by default.
+# Unlike Flake8, Ruff doesn't enable pycodestyle warnings (`W`) or
+# McCabe complexity (`C901`) by default.
+select = ["E", "F"]
+
+# Allow autofix for all enabled rules (when `--fix`) is provided.
+fixable = ["ALL"]
+unfixable = []
 
 # E402 module level import not at top of file
 # E722 do not use bare 'except'
-ignore = ["E402", "E722"]
-exclude = ["*_pb2.py", "*_pb2.pyi"]
+# E501 line too long
+ignore = ["E402", "E722", "E501"]
 
-# Same as pycodestyle.
-line-length = 180
+line-length = 120
+
+exclude = [
+    # Exclude a variety of commonly ignored directories.
+    ".bzr",
+    ".direnv",
+    ".eggs",
+    ".git",
+    ".git-rewrite",
+    ".hg",
+    ".mypy_cache",
+    ".nox",
+    ".pants.d",
+    ".pytype",
+    ".ruff_cache",
+    ".svn",
+    ".tox",
+    ".venv",
+    "__pypackages__",
+    "_build",
+    "buck-out",
+    "build",
+    "dist",
+    "node_modules",
+    "venv",
+    # protobuf generated files
+    "*_pb2.py", 
+    "*_pb2.pyi"
+]
+
+[per-file-ignores]
+# until we address #1592 and move test fixtures into conftest.py
+# then we need to ignore imports done to enable pytest fixtures.
+#
+# F401: `foo` imported but unused
+# F811 Redefinition of unused `foo`
+"tests/test_main.py" = ["F401", "F811"]
+"tests/test_proto.py" = ["F401", "F811"]
+"tests/test_freeze.py" = ["F401", "F811"]
+"tests/test_function_id.py" = ["F401", "F811"]
+"tests/test_viv_features.py" = ["F401", "F811"]
+"tests/test_binja_features.py" = ["F401", "F811"]
+"tests/test_pefile_features.py" = ["F401", "F811"]
+"tests/test_dnfile_features.py" = ["F401", "F811"]
+"tests/test_dotnet_features.py" = ["F401", "F811"]
+"tests/test_result_document.py" = ["F401", "F811"]
+"tests/test_dotnetfile_features.py" = ["F401", "F811"]

.github/tox.ini

@@ -1,10 +0,0 @@
-[pycodestyle]
-; E402: module level import not at top of file
-; W503: line break before binary operator
-; E231 missing whitespace after ',' (emitted by black)
-; E203 whitespace before ':' (emitted by black)
-ignore = E402,W503,E203,E231
-max-line-length = 160
-statistics = True
-count = True
-exclude = .*

.github/workflows/build.yml

@@ -1,5 +1,8 @@
 name: build
 
+permissions:
+  contents: write
+
 on:
   pull_request:
     branches: [ master ]

.github/workflows/publish.yml

@@ -1,15 +1,21 @@
-# This workflows will upload a Python Package using Twine when a release is created
-# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
-
+# use PyPI trusted publishing, as described here: 
+# https://blog.trailofbits.com/2023/05/23/trusted-publishing-a-new-benchmark-for-packaging-security/
 name: publish to pypi
 
+permissions:
+  contents: write
+
 on:
   release:
     types: [published]
 
 jobs:
-  deploy:
-    runs-on: ubuntu-20.04
+  pypi-publish:
+    runs-on: ubuntu-latest
+    environment:
+      name: release
+    permissions:
+      id-token: write
     steps:
       - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
       - name: Set up Python
@@ -19,11 +25,24 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install setuptools wheel twine
-      - name: Build and publish
-        env:
-          TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
-          TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+          pip install -e .[build]
+      - name: build package
         run: |
-          python setup.py sdist bdist_wheel
-          twine upload --skip-existing dist/*
+          python -m build
+      - name: upload package artifacts
+        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
+        with:
+          name: ${{ matrix.asset_name }}
+          path: dist/*
+      - name: upload package to GH Release
+        uses: svenstaro/upload-release-action@2728235f7dc9ff598bd86ce3c274b74f802d2208 # v2
+        with:
+          repo_token: ${{ secrets.GITHUB_TOKEN}}
+          file: dist/*
+          tag: ${{ github.ref }}
+      - name: publish package
+        uses: pypa/gh-action-pypi-publish@f5622bde02b04381239da3573277701ceca8f6a0  # release/v1
+        with:
+          skip-existing: true
+          verbose: true
+          print-hash: true

.github/workflows/tests.yml

@@ -34,15 +34,15 @@ jobs:
     - name: Install dependencies
       run: pip install -e .[dev]
     - name: Lint with ruff
-      run: ruff check --config .github/ruff.toml .
+      run: pre-commit run ruff
     - name: Lint with isort
-      run: isort --profile black --length-sort --line-width 120 --skip-glob "*_pb2.py" -c .
+      run: pre-commit run isort
     - name: Lint with black
-      run: black -l 120 --extend-exclude ".*_pb2.py" --check .
-    - name: Lint with pycodestyle
-      run: pycodestyle --exclude="*_pb2.py" --show-source capa/ scripts/ tests/
+      run: pre-commit run black
+    - name: Lint with flake8
+      run: pre-commit run flake8
     - name: Check types with mypy
-      run: mypy --config-file .github/mypy/mypy.ini --check-untyped-defs capa/ scripts/ tests/
+      run:  pre-commit run mypy
 
   rule_linter:
     runs-on: ubuntu-20.04
@@ -56,7 +56,7 @@ jobs:
       with:
         python-version: "3.8"
     - name: Install capa
-      run: pip install -e .
+      run: pip install -e .[dev]
     - name: Run rule linter
       run: python scripts/lint.py rules/