viv: insn: string: handle viv bug around substrings (#1273)

* viv: insn: string: handle viv bug around substrings

closes #1271

* use minimum string length 4

* update overlapping string test and fixup vivisect elf analysis missing function

Co-authored-by: Moritz <mr-tz@users.noreply.github.com>

CHANGELOG.md

@@ -88,6 +88,7 @@
 - render: fix verbose rendering of scopes #1263 @williballenthin
 - rules: better detect invalid rules #1282 @williballenthin
 - show-features: better render strings with embedded whitespace #1267 @williballenthin
+- handle vivisect bug around strings at instruction level, use min length 4 #1271 @williballenthin @mr-tz
 - extractor: guard against invalid "calls from" features #1177 @mr-tz
 
 ### capa explorer IDA Pro plugin

capa/features/extractors/viv/insn.py

@@ -281,7 +281,12 @@ def read_string(vw, offset: int) -> str:
         pass
     else:
         if alen > 0:
-            return read_memory(vw, offset, alen).decode("utf-8")
+            buf = read_memory(vw, offset, alen)
+            if b"\x00" in buf:
+                # account for bug #1271.
+                # remove when vivisect is fixed.
+                buf = buf.partition(b"\x00")[0]
+            return buf.decode("utf-8")
 
     try:
         ulen = vw.detectUnicode(offset)
@@ -300,7 +305,9 @@ def read_string(vw, offset: int) -> str:
                 # vivisect seems to mis-detect the end unicode strings
                 # off by two, too short
                 ulen += 2
-            return read_memory(vw, offset, ulen).decode("utf-16")
+            # partition to account for bug #1271.
+            # remove when vivisect is fixed.
+            return read_memory(vw, offset, ulen).decode("utf-16").partition("\x00")[0]
 
     raise ValueError("not a string", offset)
 
@@ -665,11 +672,12 @@ def extract_op_string_features(
 
     for v in derefs(f.vw, v):
         try:
-            s = read_string(f.vw, v)
+            s = read_string(f.vw, v).rstrip("\x00")
         except ValueError:
             continue
         else:
-            yield String(s.rstrip("\x00")), ih.address
+            if len(s) > 4:
+                yield String(s), ih.address
 
 
 def extract_operand_features(f: FunctionHandle, bb, insn: InsnHandle) -> Iterator[Tuple[Feature, Address]]:

tests/fixtures.py

@@ -118,6 +118,9 @@ def fixup_viv(path, extractor):
     if "3b13b" in path:
         # vivisect only recognizes calling thunk function at 0x10001573
         extractor.vw.makeFunction(0x10006860)
+    if "294b8d" in path:
+        # see vivisect/#561
+        extractor.vw.makeFunction(0x404970)
 
 
 @lru_cache(maxsize=1)
@@ -277,6 +280,8 @@ def get_data_path_by_name(name):
         return os.path.join(CD, "data", "b5f0524e69b3a3cf636c7ac366ca57bf5e3a8fdc8a9f01caf196c611a7918a87.elf_")
     elif name.startswith("bf7a9c"):
         return os.path.join(CD, "data", "bf7a9c8bdfa6d47e01ad2b056264acc3fd90cf43fe0ed8deec93ab46b47d76cb.elf_")
+    elif name.startswith("294b8d"):
+        return os.path.join(CD, "data", "294b8db1f2702b60fb2e42fdc50c2cee6a5046112da9a5703a548a4fa50477bc.elf_")
     else:
         raise ValueError("unexpected sample fixture: %s" % name)
 
@@ -627,6 +632,8 @@ FEATURE_PRESENCE_TESTS = sorted(
         ("mimikatz", "function=0x40105D", capa.features.common.String("ACR  > "), True),
         ("mimikatz", "function=0x40105D", capa.features.common.String("nope"), False),
         ("773290...", "function=0x140001140", capa.features.common.String(r"%s:\\OfficePackagesForWDAG"), True),
+        # overlapping string, see #1271
+        ("294b8d...", "function=0x404970,bb=0x404970,insn=0x40499F", capa.features.common.String("\r\n\x00:ht"), False),
         # insn/regex
         ("pma16-01", "function=0x4021B0", capa.features.common.Regex("HTTP/1.0"), True),
         ("pma16-01", "function=0x402F40", capa.features.common.Regex("www.practicalmalwareanalysis.com"), True),