From 83dbf81d2b3615e2d4e39ea6860fc60da0d7021d Mon Sep 17 00:00:00 2001 From: Michael Hunhoff Date: Wed, 24 Jun 2020 16:29:30 -0600 Subject: [PATCH 1/3] adding new checks for file format limitations in capa explorer plugin --- capa/ida/ida_capa_explorer.py | 23 +++++-- capa/main.py | 125 +++++++++++++++++----------------- 2 files changed, 83 insertions(+), 65 deletions(-) diff --git a/capa/ida/ida_capa_explorer.py b/capa/ida/ida_capa_explorer.py index d40c7ab8..5a17d661 100644 --- a/capa/ida/ida_capa_explorer.py +++ b/capa/ida/ida_capa_explorer.py @@ -32,8 +32,11 @@ from capa.ida.explorer.model import CapaExplorerDataModel from capa.ida.explorer.proxy import CapaExplorerSortFilterProxyModel -PLUGIN_NAME = 'capaex' +PLUGIN_NAME = 'capa explorer' +SUPPORTED_FILE_TYPES = [ + 'Portable executable for 80386 (PE)', +] logger = logging.getLogger(PLUGIN_NAME) @@ -326,12 +329,15 @@ class CapaExplorerForm(idaapi.PluginForm): rules_path = os.path.join(os.path.dirname(self._file_loc), '../..', 'rules') rules = capa.main.get_rules(rules_path) rules = capa.rules.RuleSet(rules) - results = capa.main.find_capabilities(rules, capa.features.extractors.ida.IdaFeatureExtractor(), True) + capabilities = capa.main.find_capabilities(rules, capa.features.extractors.ida.IdaFeatureExtractor(), True) + + if capa.main.is_file_limitation(rules, capabilities): + idaapi.info('capa encountered warnings during analysis. Please refer to the IDA Output window for more information.') logger.info('analysis completed.') - self._model_data.render_capa_results(rules, results) - self._render_capa_summary(rules, results) + self._model_data.render_capa_results(rules, capabilities) + self._render_capa_summary(rules, capabilities) logger.info('render views completed.') @@ -441,6 +447,15 @@ def main(): ''' TODO: move to idaapi.plugin_t class ''' logging.basicConfig(level=logging.INFO) + if idaapi.get_file_type_name() not in SUPPORTED_FILE_TYPES: + logger.error('-' * 80) + logger.error(' Input file does not appear to be a PE file.') + logger.error(' ') + logger.error(' capa explorer currently only supports analyzing PE files.') + logger.error('-' * 80) + idaapi.info('capa does not support the format of this file. Please refer to the IDA output window for more information.') + return -1 + global CAPA_EXPLORER_FORM try: diff --git a/capa/main.py b/capa/main.py index 840425a0..1fa5fdfb 100644 --- a/capa/main.py +++ b/capa/main.py @@ -404,6 +404,65 @@ def appears_rule_cat(rules, capabilities, rule_cat): return False +def is_file_limitation(rules, capabilities): + if appears_rule_cat(rules, capabilities, 'other-features/installer/'): + logger.warning('-' * 80) + logger.warning(' This sample appears to be an installer.') + logger.warning(' ') + logger.warning(' capa cannot handle installers well. This means the results may be misleading or incomplete.') + logger.warning(' You should try to understand the install mechanism and analyze created files with capa.') + logger.warning(' ') + logger.warning(' Use -v or -vv if you really want to see the capabilities identified by capa.') + logger.warning('-' * 80) + # capa will likely detect installer specific functionality. + # this is probably not what the user wants. + return True + + if appears_rule_cat(rules, capabilities, 'other-features/compiled-to-dot-net'): + logger.warning('-' * 80) + logger.warning(' This sample appears to be a .NET module.') + logger.warning(' ') + logger.warning(' .NET is a cross-platform framework for running managed applications.') + logger.warning( + ' capa cannot handle non-native files. This means that the results may be misleading or incomplete.') + logger.warning(' You may have to analyze the file manually, using a tool like the .NET decompiler dnSpy.') + logger.warning(' ') + logger.warning(' Use -v or -vv if you really want to see the capabilities identified by capa.') + logger.warning('-' * 80) + # capa won't detect much in .NET samples. + # it might match some file-level things. + # for consistency, bail on things that we don't support. + return True + + if appears_rule_cat(rules, capabilities, 'other-features/compiled-with-autoit'): + logger.warning('-' * 80) + logger.warning(' This sample appears to be compiled with AutoIt.') + logger.warning(' ') + logger.warning(' AutoIt is a freeware BASIC-like scripting language designed for automating the Windows GUI.') + logger.warning( + ' capa cannot handle AutoIt scripts. This means that the results will be misleading or incomplete.') + logger.warning(' You may have to analyze the file manually, using a tool like the AutoIt decompiler MyAut2Exe.') + logger.warning(' ') + logger.warning(' Use -v or -vv if you really want to see the capabilities identified by capa.') + logger.warning('-' * 80) + # capa will detect dozens of capabilities for AutoIt samples, + # but these are due to the AutoIt runtime, not the payload script. + # so, don't confuse the user with FP matches - bail instead + return True + + if appears_rule_cat(rules, capabilities, 'anti-analysis/packing/'): + logger.warning('-' * 80) + logger.warning(' This sample appears packed.') + logger.warning(' ') + logger.warning(' Packed samples have often been obfuscated to hide their logic.') + logger.warning(' capa cannot handle obfuscation well. This means the results may be misleading or incomplete.') + logger.warning(' If possible, you should try to unpack this input file before analyzing it with capa.') + logger.warning('-' * 80) + return True + + return False + + def is_supported_file_type(sample): ''' Return if this is a supported file based on magic header values @@ -657,70 +716,12 @@ def main(argv=None): capabilities = find_capabilities(rules, extractor) - if appears_rule_cat(rules, capabilities, 'other-features/installer/'): - logger.warning('-' * 80) - logger.warning(' This sample appears to be an installer.') - logger.warning(' ') - logger.warning(' capa cannot handle installers well. This means the results may be misleading or incomplete.') - logger.warning(' You should try to understand the install mechanism and analyze created files with capa.') - logger.warning(' ') - logger.warning(' Use -v or -vv if you really want to see the capabilities identified by capa.') - logger.warning('-' * 80) - # capa will likely detect installer specific functionality. - # this is probably not what the user wants. - # + if is_file_limitation(rules, capabilities): + # bail if capa encountered file limitation e.g. a packed binary # do show the output in verbose mode, though. if not (args.verbose or args.vverbose): return -1 - if appears_rule_cat(rules, capabilities, 'other-features/compiled-to-dot-net'): - logger.warning('-' * 80) - logger.warning(' This sample appears to be a .NET module.') - logger.warning(' ') - logger.warning(' .NET is a cross-platform framework for running managed applications.') - logger.warning( - ' capa cannot handle non-native files. This means that the results may be misleading or incomplete.') - logger.warning(' You may have to analyze the file manually, using a tool like the .NET decompiler dnSpy.') - logger.warning(' ') - logger.warning(' Use -v or -vv if you really want to see the capabilities identified by capa.') - logger.warning('-' * 80) - - # capa won't detect much in .NET samples. - # it might match some file-level things. - # for consistency, bail on things that we don't support. - # - # do show the output in verbose mode, though. - if not (args.verbose or args.vverbose): - return -1 - - if appears_rule_cat(rules, capabilities, 'other-features/compiled-with-autoit'): - logger.warning('-' * 80) - logger.warning(' This sample appears to be compiled with AutoIt.') - logger.warning(' ') - logger.warning(' AutoIt is a freeware BASIC-like scripting language designed for automating the Windows GUI.') - logger.warning( - ' capa cannot handle AutoIt scripts. This means that the results will be misleading or incomplete.') - logger.warning(' You may have to analyze the file manually, using a tool like the AutoIt decompiler MyAut2Exe.') - logger.warning(' ') - logger.warning(' Use -v or -vv if you really want to see the capabilities identified by capa.') - logger.warning('-' * 80) - # capa will detect dozens of capabilities for AutoIt samples, - # but these are due to the AutoIt runtime, not the payload script. - # so, don't confuse the user with FP matches - bail instead - # - # do show the output in verbose mode, though. - if not (args.verbose or args.vverbose): - return -1 - - if appears_rule_cat(rules, capabilities, 'anti-analysis/packing/'): - logger.warning('-' * 80) - logger.warning(' This sample appears packed.') - logger.warning(' ') - logger.warning(' Packed samples have often been obfuscated to hide their logic.') - logger.warning(' capa cannot handle obfuscation well. This means the results may be misleading or incomplete.') - logger.warning(' If possible, you should try to unpack this input file before analyzing it with capa.') - logger.warning('-' * 80) - if args.vverbose: render_capabilities_vverbose(capabilities) elif args.verbose: @@ -759,7 +760,9 @@ def ida_main(): import capa.features.extractors.ida capabilities = find_capabilities(rules, capa.features.extractors.ida.IdaFeatureExtractor()) - render_capabilities_default(rules, capabilities) + + if not is_file_limitation(rules, capabilities): + render_capabilities_default(rules, capabilities) def is_runtime_ida(): From a5004b201421e12e7217929b45709fcd3aef517e Mon Sep 17 00:00:00 2001 From: Michael Hunhoff Date: Thu, 25 Jun 2020 10:05:19 -0600 Subject: [PATCH 2/3] adding support checks for AMD64/binary files in capa explorer and capa main --- capa/ida/helpers/__init__.py | 47 +++++++++++++++++++++++++++++++++++ capa/ida/ida_capa_explorer.py | 16 +++--------- capa/main.py | 11 ++++++-- 3 files changed, 59 insertions(+), 15 deletions(-) diff --git a/capa/ida/helpers/__init__.py b/capa/ida/helpers/__init__.py index 78f0d93e..00636c9b 100644 --- a/capa/ida/helpers/__init__.py +++ b/capa/ida/helpers/__init__.py @@ -1,6 +1,53 @@ +import logging + import idaapi import idc +logger = logging.getLogger() + +# file type names as returned by idaapi.get_file_type_name() +SUPPORTED_FILE_TYPES = [ + 'Portable executable for 80386 (PE)', + 'Portable executable for AMD64 (PE)', + 'Binary file' # x86/AMD64 shellcode support +] + + +def inform_user_ida_ui(message): + idaapi.info('%s. Please refer to IDA Output window for more information.' % message) + + +def is_supported_file_type(): + file_type = idaapi.get_file_type_name() + + if file_type not in SUPPORTED_FILE_TYPES: + logger.error('-' * 80) + logger.error(' Input file does not appear to be a PE file.') + logger.error(' ') + logger.error(' capa currently only supports analyzing PE files (or x86/AMD64 shellcode).') + logger.error(' If you don\'t know the input file type, you can try using the `file` utility to guess it.') + logger.error('-' * 80) + + inform_user_ida_ui('capa does not support the format of this file') + + return False + + # support binary files specifically for x86/AMD64 shellcode + # warn user binary file is loaded but still allow capa to process it + # TODO: check specific architecture of binary files based on how user configured IDA processors + if file_type == 'Binary file': + logger.warning('-' * 80) + logger.warning(' Input file appears to be a binary file.') + logger.warning(' ') + logger.warning(' capa currently only supports analyzing binary files containing x86/AMD64 shellcode.') + logger.warning(' This means the results may be misleading or incomplete if the binary file is not x86/AMD64.') + logger.warning(' If you don\'t know the input file type, you can try using the `file` utility to guess it.') + logger.warning('-' * 80) + + inform_user_ida_ui('capa encountered warnings during analysis') + + return True + def get_disasm_line(va): ''' ''' diff --git a/capa/ida/ida_capa_explorer.py b/capa/ida/ida_capa_explorer.py index 5a17d661..079d6d49 100644 --- a/capa/ida/ida_capa_explorer.py +++ b/capa/ida/ida_capa_explorer.py @@ -26,18 +26,14 @@ import idaapi import capa.main import capa.rules import capa.features.extractors.ida +import capa.ida.helpers from capa.ida.explorer.view import CapaExplorerQtreeView from capa.ida.explorer.model import CapaExplorerDataModel from capa.ida.explorer.proxy import CapaExplorerSortFilterProxyModel - PLUGIN_NAME = 'capa explorer' -SUPPORTED_FILE_TYPES = [ - 'Portable executable for 80386 (PE)', -] - logger = logging.getLogger(PLUGIN_NAME) @@ -332,7 +328,7 @@ class CapaExplorerForm(idaapi.PluginForm): capabilities = capa.main.find_capabilities(rules, capa.features.extractors.ida.IdaFeatureExtractor(), True) if capa.main.is_file_limitation(rules, capabilities): - idaapi.info('capa encountered warnings during analysis. Please refer to the IDA Output window for more information.') + capa.ida.helpers.inform_user_ida_ui('capa encountered warnings during analysis') logger.info('analysis completed.') @@ -447,13 +443,7 @@ def main(): ''' TODO: move to idaapi.plugin_t class ''' logging.basicConfig(level=logging.INFO) - if idaapi.get_file_type_name() not in SUPPORTED_FILE_TYPES: - logger.error('-' * 80) - logger.error(' Input file does not appear to be a PE file.') - logger.error(' ') - logger.error(' capa explorer currently only supports analyzing PE files.') - logger.error('-' * 80) - idaapi.info('capa does not support the format of this file. Please refer to the IDA output window for more information.') + if not capa.ida.helpers.is_supported_file_type(): return -1 global CAPA_EXPLORER_FORM diff --git a/capa/main.py b/capa/main.py index 1fa5fdfb..9d29107f 100644 --- a/capa/main.py +++ b/capa/main.py @@ -761,13 +761,20 @@ def ida_main(): import capa.features.extractors.ida capabilities = find_capabilities(rules, capa.features.extractors.ida.IdaFeatureExtractor()) - if not is_file_limitation(rules, capabilities): - render_capabilities_default(rules, capabilities) + import capa.ida.helpers + if not capa.ida.helpers.is_supported_file_type(): + return -1 + + if is_file_limitation(rules, capabilities): + capa.ida.helpers.inform_user_ida_ui('capa encountered warnings during analysis') + + render_capabilities_default(rules, capabilities) def is_runtime_ida(): try: import idc + import idaapi except ImportError: return False else: From 25f026274856fc866ee9aee80fef50c88ab4b0bb Mon Sep 17 00:00:00 2001 From: Michael Hunhoff Date: Thu, 25 Jun 2020 11:09:47 -0600 Subject: [PATCH 3/3] moving code around for supported file type dialouge in standalone tool and capa explorer --- capa/ida/helpers/__init__.py | 23 ++------- capa/ida/ida_capa_explorer.py | 20 +++++++- capa/main.py | 94 ++++++++++++++++------------------- 3 files changed, 65 insertions(+), 72 deletions(-) diff --git a/capa/ida/helpers/__init__.py b/capa/ida/helpers/__init__.py index 00636c9b..be6402e3 100644 --- a/capa/ida/helpers/__init__.py +++ b/capa/ida/helpers/__init__.py @@ -3,7 +3,7 @@ import logging import idaapi import idc -logger = logging.getLogger() +logger = logging.getLogger('capa') # file type names as returned by idaapi.get_file_type_name() SUPPORTED_FILE_TYPES = [ @@ -19,33 +19,16 @@ def inform_user_ida_ui(message): def is_supported_file_type(): file_type = idaapi.get_file_type_name() - if file_type not in SUPPORTED_FILE_TYPES: logger.error('-' * 80) logger.error(' Input file does not appear to be a PE file.') logger.error(' ') - logger.error(' capa currently only supports analyzing PE files (or x86/AMD64 shellcode).') + logger.error( + ' capa currently only supports analyzing PE files (or binary files containing x86/AMD64 shellcode) with IDA.') logger.error(' If you don\'t know the input file type, you can try using the `file` utility to guess it.') logger.error('-' * 80) - inform_user_ida_ui('capa does not support the format of this file') - return False - - # support binary files specifically for x86/AMD64 shellcode - # warn user binary file is loaded but still allow capa to process it - # TODO: check specific architecture of binary files based on how user configured IDA processors - if file_type == 'Binary file': - logger.warning('-' * 80) - logger.warning(' Input file appears to be a binary file.') - logger.warning(' ') - logger.warning(' capa currently only supports analyzing binary files containing x86/AMD64 shellcode.') - logger.warning(' This means the results may be misleading or incomplete if the binary file is not x86/AMD64.') - logger.warning(' If you don\'t know the input file type, you can try using the `file` utility to guess it.') - logger.warning('-' * 80) - - inform_user_ida_ui('capa encountered warnings during analysis') - return True diff --git a/capa/ida/ida_capa_explorer.py b/capa/ida/ida_capa_explorer.py index 079d6d49..c322a274 100644 --- a/capa/ida/ida_capa_explorer.py +++ b/capa/ida/ida_capa_explorer.py @@ -34,7 +34,7 @@ from capa.ida.explorer.proxy import CapaExplorerSortFilterProxyModel PLUGIN_NAME = 'capa explorer' -logger = logging.getLogger(PLUGIN_NAME) +logger = logging.getLogger('capa') class CapaExplorerIdaHooks(idaapi.UI_Hooks): @@ -327,7 +327,23 @@ class CapaExplorerForm(idaapi.PluginForm): rules = capa.rules.RuleSet(rules) capabilities = capa.main.find_capabilities(rules, capa.features.extractors.ida.IdaFeatureExtractor(), True) - if capa.main.is_file_limitation(rules, capabilities): + # support binary files specifically for x86/AMD64 shellcode + # warn user binary file is loaded but still allow capa to process it + # TODO: check specific architecture of binary files based on how user configured IDA processors + if idaapi.get_file_type_name() == 'Binary file': + logger.warning('-' * 80) + logger.warning(' Input file appears to be a binary file.') + logger.warning(' ') + logger.warning( + ' capa currently only supports analyzing binary files containing x86/AMD64 shellcode with IDA.') + logger.warning( + ' This means the results may be misleading or incomplete if the binary file loaded in IDA is not x86/AMD64.') + logger.warning(' If you don\'t know the input file type, you can try using the `file` utility to guess it.') + logger.warning('-' * 80) + + capa.ida.helpers.inform_user_ida_ui('capa encountered warnings during analysis') + + if capa.main.is_file_limitation(rules, capabilities, is_standalone=False): capa.ida.helpers.inform_user_ida_ui('capa encountered warnings during analysis') logger.info('analysis completed.') diff --git a/capa/main.py b/capa/main.py index 9d29107f..a67afd5f 100644 --- a/capa/main.py +++ b/capa/main.py @@ -404,62 +404,57 @@ def appears_rule_cat(rules, capabilities, rule_cat): return False -def is_file_limitation(rules, capabilities): - if appears_rule_cat(rules, capabilities, 'other-features/installer/'): - logger.warning('-' * 80) - logger.warning(' This sample appears to be an installer.') - logger.warning(' ') - logger.warning(' capa cannot handle installers well. This means the results may be misleading or incomplete.') - logger.warning(' You should try to understand the install mechanism and analyze created files with capa.') - logger.warning(' ') - logger.warning(' Use -v or -vv if you really want to see the capabilities identified by capa.') - logger.warning('-' * 80) +def is_file_limitation(rules, capabilities, is_standalone=True): + file_limitations = { # capa will likely detect installer specific functionality. # this is probably not what the user wants. - return True - - if appears_rule_cat(rules, capabilities, 'other-features/compiled-to-dot-net'): - logger.warning('-' * 80) - logger.warning(' This sample appears to be a .NET module.') - logger.warning(' ') - logger.warning(' .NET is a cross-platform framework for running managed applications.') - logger.warning( - ' capa cannot handle non-native files. This means that the results may be misleading or incomplete.') - logger.warning(' You may have to analyze the file manually, using a tool like the .NET decompiler dnSpy.') - logger.warning(' ') - logger.warning(' Use -v or -vv if you really want to see the capabilities identified by capa.') - logger.warning('-' * 80) + 'other-features/installer/': [ + ' This sample appears to be an installer.', + ' ', + ' capa cannot handle installers well. This means the results may be misleading or incomplete.' + ' You should try to understand the install mechanism and analyze created files with capa.' + ], # capa won't detect much in .NET samples. # it might match some file-level things. # for consistency, bail on things that we don't support. - return True - - if appears_rule_cat(rules, capabilities, 'other-features/compiled-with-autoit'): - logger.warning('-' * 80) - logger.warning(' This sample appears to be compiled with AutoIt.') - logger.warning(' ') - logger.warning(' AutoIt is a freeware BASIC-like scripting language designed for automating the Windows GUI.') - logger.warning( - ' capa cannot handle AutoIt scripts. This means that the results will be misleading or incomplete.') - logger.warning(' You may have to analyze the file manually, using a tool like the AutoIt decompiler MyAut2Exe.') - logger.warning(' ') - logger.warning(' Use -v or -vv if you really want to see the capabilities identified by capa.') - logger.warning('-' * 80) + 'other-features/compiled-to-dot-net': [ + ' This sample appears to be a .NET module.', + ' ', + ' .NET is a cross-platform framework for running managed applications.', + ' capa cannot handle non-native files. This means that the results may be misleading or incomplete.', + ' You may have to analyze the file manually, using a tool like the .NET decompiler dnSpy.' + ], # capa will detect dozens of capabilities for AutoIt samples, # but these are due to the AutoIt runtime, not the payload script. # so, don't confuse the user with FP matches - bail instead - return True + 'other-features/compiled-with-autoit': [ + ' This sample appears to be compiled with AutoIt.', + ' ', + ' AutoIt is a freeware BASIC-like scripting language designed for automating the Windows GUI.', + ' capa cannot handle AutoIt scripts. This means that the results will be misleading or incomplete.', + ' You may have to analyze the file manually, using a tool like the AutoIt decompiler MyAut2Exe.' + ], + # capa won't detect much in packed samples + 'anti-analysis/packing/': [ + ' This sample appears to be packed.', + ' ', + ' Packed samples have often been obfuscated to hide their logic.', + ' capa cannot handle obfuscation well. This means the results may be misleading or incomplete.', + ' If possible, you should try to unpack this input file before analyzing it with capa.' + ] + } - if appears_rule_cat(rules, capabilities, 'anti-analysis/packing/'): + for category, dialogue in file_limitations.items(): + if not appears_rule_cat(rules, capabilities, category): + continue logger.warning('-' * 80) - logger.warning(' This sample appears packed.') - logger.warning(' ') - logger.warning(' Packed samples have often been obfuscated to hide their logic.') - logger.warning(' capa cannot handle obfuscation well. This means the results may be misleading or incomplete.') - logger.warning(' If possible, you should try to unpack this input file before analyzing it with capa.') + for line in dialogue: + logger.warning(line) + if is_standalone: + logger.warning(' ') + logger.warning(' Use -v or -vv if you really want to see the capabilities identified by capa.') logger.warning('-' * 80) return True - return False @@ -738,6 +733,10 @@ def ida_main(): logging.basicConfig(level=logging.INFO) logging.getLogger().setLevel(logging.INFO) + import capa.ida.helpers + if not capa.ida.helpers.is_supported_file_type(): + return -1 + logger.info('-' * 80) logger.info(' Using default embedded rules.') logger.info(' ') @@ -761,11 +760,7 @@ def ida_main(): import capa.features.extractors.ida capabilities = find_capabilities(rules, capa.features.extractors.ida.IdaFeatureExtractor()) - import capa.ida.helpers - if not capa.ida.helpers.is_supported_file_type(): - return -1 - - if is_file_limitation(rules, capabilities): + if is_file_limitation(rules, capabilities, is_standalone=False): capa.ida.helpers.inform_user_ida_ui('capa encountered warnings during analysis') render_capabilities_default(rules, capabilities) @@ -774,7 +769,6 @@ def ida_main(): def is_runtime_ida(): try: import idc - import idaapi except ImportError: return False else: