From ac7f37dbe8dc790329dbd7f05eaa2ae892166d6e Mon Sep 17 00:00:00 2001 From: Amelia Shaw Date: Wed, 22 Jul 2026 10:04:10 +0100 Subject: [PATCH 1/6] Ran AutoPep8 on the python scripts --- bin/yara_rules/parse_clamav.py | 13 +++++++------ bin/yara_rules/update_clamav_signatures.py | 5 ++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/bin/yara_rules/parse_clamav.py b/bin/yara_rules/parse_clamav.py index 8e36936..8bb63b5 100755 --- a/bin/yara_rules/parse_clamav.py +++ b/bin/yara_rules/parse_clamav.py @@ -46,6 +46,7 @@ class TargetType: FLASH = 11 JAVA = 12 + yara_rule_template = """rule %s { meta: @@ -192,15 +193,15 @@ def _translate_offset(self, offset, index): x = int(split[0]) y = int(split[1]) if match.group(2) == '+': - self._conditions.append("$a%d in (%s+%d .. %s + %d)" \ - % (index, base_yara_offset, x, base_yara_offset, x + y)) + self._conditions.append("$a%d in (%s+%d .. %s + %d)" + % (index, base_yara_offset, x, base_yara_offset, x + y)) else: if y < x: - self._conditions.append("$a%d in (%s - %d .. %s - %d)" \ - % (index, base_yara_offset, x, base_yara_offset, x - y)) + self._conditions.append("$a%d in (%s - %d .. %s - %d)" + % (index, base_yara_offset, x, base_yara_offset, x - y)) elif y > x: - self._conditions.append("$a%d in (%s - %d .. %s + %d)" \ - % (index, base_yara_offset, x, base_yara_offset, y - x)) + self._conditions.append("$a%d in (%s - %d .. %s + %d)" + % (index, base_yara_offset, x, base_yara_offset, y - x)) else: # x == y self._conditions.append("$a%d in (%s - %d .. %s)" % (index, base_yara_offset, x, base_yara_offset)) else: diff --git a/bin/yara_rules/update_clamav_signatures.py b/bin/yara_rules/update_clamav_signatures.py index b5a5051..598e40a 100755 --- a/bin/yara_rules/update_clamav_signatures.py +++ b/bin/yara_rules/update_clamav_signatures.py @@ -56,7 +56,7 @@ def download_file(url): if file_size is None: outfile.write(r.content) return - + downloaded = 0 chunk_size = 8192 file_size = int(file_size) @@ -69,8 +69,6 @@ def download_file(url): print(status, end="") - - def zlib_decompress(path, outpath): d = zlib.decompressobj(zlib.MAX_WBITS + 16) input = open(path, "rb") @@ -140,6 +138,7 @@ def update_signatures(url, download): os.remove("%s.tar" % file_basename) os.remove("%s.ndb" % file_basename) + # Work in the script's directory if os.path.dirname(sys.argv[0]) != "": os.chdir(os.path.dirname(sys.argv[0])) From 8ec38c6d91dd917251c4732a112d23a36fdf7ebb Mon Sep 17 00:00:00 2001 From: Amelia Shaw Date: Wed, 22 Jul 2026 10:08:32 +0100 Subject: [PATCH 2/6] Adjusted regex strings to use raw strings --- bin/yara_rules/parse_clamav.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/bin/yara_rules/parse_clamav.py b/bin/yara_rules/parse_clamav.py index 8bb63b5..d095168 100755 --- a/bin/yara_rules/parse_clamav.py +++ b/bin/yara_rules/parse_clamav.py @@ -22,14 +22,14 @@ from string import hexdigits # Two groups of number separated by a comma, i.e. 200,400 -range_offset_pattern = re.compile("(\d+),(\d+)") +range_offset_pattern = re.compile(r"(\d+),(\d+)") # One capital letter followed by one or two letters/numbers (i.e S3, EOF, ...), then possibly a sign and a mix of # comma / numbers. This matches offsets like S0+123, EOF-2, SE2, EP+5,10 etc. extended_offset_pattern = re.compile("([A-Z][A-Z0-9]{1,2})(([+-])([0-9,]+))?") # Matches {-number} -floating_jump_pattern = re.compile("\{\-([0-9]+)\}") +floating_jump_pattern = re.compile(r"\{\-([0-9]+)\}") class TargetType: @@ -109,8 +109,8 @@ def _translate_signature(self, sig, index): s = sig s = s.replace("*", " [-] ") # Unbounded jump s = s.replace("{0}", "") # Skipping no bytes. Useless but appears in one signature. - s = re.sub("\{\d+\}$", "", s, count=1) # Remove byte skips at the end of signatures. - s = floating_jump_pattern.sub(" {0-\g<1>} ", s) # Yara doesn't support [-X] jumps, we need [0-X] + s = re.sub(r"\{\d+\}$", "", s, count=1) # Remove byte skips at the end of signatures. + s = floating_jump_pattern.sub(r" {0-\g<1>} ", s) # Yara doesn't support [-X] jumps, we need [0-X] s = s.replace("{", "[").replace("}", "]") # Byte skips # Try to guess if it isn't an hexadecimal pattern. if any(c in "ghijljmnopqrstuvwxyzGHIJKLMNOPQRSTUVWXYZ" for c in s): @@ -230,7 +230,7 @@ def __str__(self): else: conditions += "any of them" else: - tokens = re.findall("([=,<>\(\)&\|]|\d+)", self._logical_expression) + tokens = re.findall(r"([=,<>\(\)&\|]|\d+)", self._logical_expression) i = 0 while i < len(tokens): t = tokens[i] From f18b7d1c4024ac0637cb610c512e2b9c4a840c96 Mon Sep 17 00:00:00 2001 From: Amelia Shaw Date: Wed, 22 Jul 2026 11:51:29 +0100 Subject: [PATCH 3/6] Added in check for PCRE subsignatures to resolve one of the yara compiling errors --- bin/yara_rules/parse_clamav.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/bin/yara_rules/parse_clamav.py b/bin/yara_rules/parse_clamav.py index d095168..e90884c 100755 --- a/bin/yara_rules/parse_clamav.py +++ b/bin/yara_rules/parse_clamav.py @@ -112,11 +112,22 @@ def _translate_signature(self, sig, index): s = re.sub(r"\{\d+\}$", "", s, count=1) # Remove byte skips at the end of signatures. s = floating_jump_pattern.sub(r" {0-\g<1>} ", s) # Yara doesn't support [-X] jumps, we need [0-X] s = s.replace("{", "[").replace("}", "]") # Byte skips + # Try to guess if it isn't an hexadecimal pattern. if any(c in "ghijljmnopqrstuvwxyzGHIJKLMNOPQRSTUVWXYZ" for c in s): raise MalformedRuleError("Malformed rule: %s (%s)" % (self._meta_signature, s)) - else: - self._signatures.append("$a%d = { %s }" % (index, s)) + + # This checks for PCRE subsignatures. These consist of a trigger and a regex to perform when the trigger is met + # https://docs.clamav.net/manual/Signatures/LogicalSignatures.html#pcre-subsignatures + # Example found on in Win_dot_Trojan_dot_Zebrocy_dash_6743852_dash_2: + # 0|(1&2)/6#?87474703A2F2F(3[0-9])[1,3]2E(3[0-9])[1,3]2E(3[0-9])[1,3]2E(3[0-9])[1,3]2F(3[0-9]|[46][1-9A-F]|[57][0-9]|5A|7A|5F|2F|2D)+2E706870/ + # This rule checks if subsignatures 0 or 1 and 2 are met, and if so it runs the regex + # This could be applied to the yara rules, but it would require rewriting the conditions so that rule 3 is the regex on its own, and the conditions are (0 | (1 & 2)) & 3 + # These were often caught by the previous check that looks for non-hexadecimal alphanumerics, but that doesnt trigger in cases like above + if re.match(r"[\d()<>&|=,]*\/.*\/", s): + raise MalformedRuleError("Malformed rule: %s (%s)" % (self._meta_signature, s)) + + self._signatures.append("$a%d = { %s }" % (index, s)) def _translate_offset(self, offset, index): # Handle simple cases first: find pattern anywhere. From da7461e1f2e5ed4319e20caf9cb790646b112da5 Mon Sep 17 00:00:00 2001 From: Amelia Shaw Date: Wed, 22 Jul 2026 13:12:19 +0100 Subject: [PATCH 4/6] Added q flag to both scripts in order to hide warnings when the rules are being parsed. Added a section to rewrite logical expressions that start with 0:, as the colon is not in any documentation, and the second half is valid --- bin/yara_rules/parse_clamav.py | 36 ++++++++++++++++++---- bin/yara_rules/update_clamav_signatures.py | 11 ++++--- 2 files changed, 36 insertions(+), 11 deletions(-) diff --git a/bin/yara_rules/parse_clamav.py b/bin/yara_rules/parse_clamav.py index e90884c..c700f3d 100755 --- a/bin/yara_rules/parse_clamav.py +++ b/bin/yara_rules/parse_clamav.py @@ -21,6 +21,8 @@ import sys from string import hexdigits +quiet = False + # Two groups of number separated by a comma, i.e. 200,400 range_offset_pattern = re.compile(r"(\d+),(\d+)") @@ -255,7 +257,8 @@ def __str__(self): # If they haven't been detected while looking ahead, it means that # they arrive after a full expression (i.e. (0&1)>X,Y), which can't # be translated to a Yara rule as far as I know. - print("Unable to translate a logical signature for %s. Skipping..." % self._meta_signature) + if not quiet: + print("Unable to translate a logical signature for %s. Skipping..." % self._meta_signature) return "" else: try: @@ -263,7 +266,8 @@ def __str__(self): # Check for a negation or a count if i + 2 < len(tokens) and (tokens[i+1] == "=" or tokens[i+1] == ">" or tokens[i+1] == "<"): if i + 3 < len(tokens) and tokens[i+2] == ",": - print("Unable to translate a logical signature for %s. Skipping..." % self._meta_signature) + if not quiet: + print("Unable to translate a logical signature for %s. Skipping..." % self._meta_signature) return "" if tokens[i+1] == "=" and tokens[i+2] == "0": # Negation conditions += "not %s" % self._conditions[index] @@ -293,7 +297,10 @@ def __str__(self): return yara_rule_template % (self._rulename, self._meta_signature, signatures, conditions) -def parse_ndb(input, output, is_daily=False): +def parse_ndb(input, output, is_daily=False, is_quiet=False): + global quiet + if is_quiet: + quiet = True with open(input) as f: with open(output, 'ab') as g: for line in f: @@ -315,7 +322,8 @@ def parse_ndb(input, output, is_daily=False): try: rule = YaraRule(malware_name, [[signature, offset]], is_daily=is_daily) except MalformedRuleError: - print("Rule %s seems to be malformed. Skipping..." % malware_name) + if not quiet: + print("Rule %s seems to be malformed. Skipping..." % malware_name) continue if not rule.get_meta_signature() in RULES: @@ -325,7 +333,10 @@ def parse_ndb(input, output, is_daily=False): print("Rule %s already exists!" % rule.get_meta_signature()) -def parse_ldb(input, output, is_daily=False): +def parse_ldb(input, output, is_daily=False, is_quiet=False): + global quiet + if is_quiet: + quiet = True with open(input) as f: with open(output, 'ab') as g: for line in f: @@ -352,6 +363,15 @@ def parse_ldb(input, output, is_daily=False): if any('!' in r for r in rules): # Skip rules containing "!" such as "...!(01|02|03)..." continue # which do not translate to Yara rules. + # I cannot find any documentation about why there would be a colon in the logical expression, + # but it occurs in only one daily case, Win.Trojan.Agent-6825810-0-6852456-0 + # 0:0&((1>20&2>10&3)|(4)) + # This causes some malformed yara where the 0 rule specifier is duplicated + # I'm not sure what possible configurations it could be, but since the logical expression generally cannot accept a colon, + # it makes sense to take the second group if it has one + if ":" in logical_expression: + logical_expression = logical_expression.split(":")[1] + signatures = [] for r in rules: r_split = r.split(":") @@ -364,7 +384,8 @@ def parse_ldb(input, output, is_daily=False): rule = YaraRule(malware_name, signatures, logical_expression=logical_expression, is_daily=is_daily) translated_rule = rule.__str__() except MalformedRuleError: - print("Rule %s seems to be malformed. Skipping..." % malware_name) + if not quiet: + print("Rule %s seems to be malformed. Skipping..." % malware_name) continue if not rule.get_meta_signature() in RULES and translated_rule: @@ -378,7 +399,10 @@ def main(): parser = argparse.ArgumentParser(description="Parses ClamAV signatures and translates them to Yara rules.") parser.add_argument("-i", "--input", dest="input", help="The file to parse.") parser.add_argument("-o", "--output", dest="output", help="The destination file for the Yara rules.") + parser.add_argument("-q", "--quiet", dest="quiet", help="Suppresses \"Malformed\" and \"Unable to translate logical signature\" messages", action="store_true") args = parser.parse_args() + if args.quiet: + quiet = True if args.input.endswith(".ndb"): parse_ndb(args.input, args.output) elif args.input.endswith(".ldb"): diff --git a/bin/yara_rules/update_clamav_signatures.py b/bin/yara_rules/update_clamav_signatures.py index 598e40a..0a37c9d 100755 --- a/bin/yara_rules/update_clamav_signatures.py +++ b/bin/yara_rules/update_clamav_signatures.py @@ -89,7 +89,7 @@ def zlib_decompress(path, outpath): os.remove(path) -def update_signatures(url, download): +def update_signatures(url, download, quiet): # Download CVD file if necessary if download: download_file(url) @@ -131,9 +131,9 @@ def update_signatures(url, download): tar.extract("%s.ldb" % file_basename) os.chmod("%s.ldb" % file_basename, 0o644) tar.close() - parse_ndb("%s.ndb" % file_basename, "clamav.yara", file_basename != "main") + parse_ndb("%s.ndb" % file_basename, "clamav.yara", file_basename != "main", is_quiet=quiet) if os.path.exists("%s.ldb" % file_basename): - parse_ldb("%s.ldb" % file_basename, "clamav.yara", file_basename != "main") + parse_ldb("%s.ldb" % file_basename, "clamav.yara", file_basename != "main", is_quiet=quiet) os.remove("%s.ldb" % file_basename) os.remove("%s.tar" % file_basename) os.remove("%s.ndb" % file_basename) @@ -147,6 +147,7 @@ def update_signatures(url, download): parser.add_argument("--main", action="store_true", help="Update ClamAV's main signature file.") parser.add_argument("--skip-download", dest="skipdownload", action="store_false", help="Work with local copies of ClamAV signature files.") +parser.add_argument("-q", "--quiet", dest="quiet", help="Suppresses \"Malformed\" and \"Unable to translate logical signature\" messages", action="store_true") args = parser.parse_args() try: @@ -162,13 +163,13 @@ def update_signatures(url, download): os.remove("clamav.main.yara") with open("clamav.yara", "wb") as f: f.write(b'import "manape"\n\n') # Do not forget to import our module. - update_signatures(URL_MAIN, args.skipdownload) + update_signatures(URL_MAIN, args.skipdownload, args.quiet) shutil.copy("clamav.yara", "clamav.main.yara") # Keep a copy to which we can append future daily signature files. else: # Use the old clamav.main.yara as a base and append the daily rules to it. shutil.copy("clamav.main.yara", "clamav.yara") -update_signatures(URL_DAILY, args.skipdownload) +update_signatures(URL_DAILY, args.skipdownload, args.quiet) try: os.remove("clamav.yarac") From b44fcd078811ee5d11806ca19dd52287523e21ad Mon Sep 17 00:00:00 2001 From: Amelia Shaw Date: Wed, 22 Jul 2026 13:46:09 +0100 Subject: [PATCH 5/6] Added handling for when a rule does not include all of the expressions in the conditions --- bin/yara_rules/parse_clamav.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/bin/yara_rules/parse_clamav.py b/bin/yara_rules/parse_clamav.py index c700f3d..37ff87a 100755 --- a/bin/yara_rules/parse_clamav.py +++ b/bin/yara_rules/parse_clamav.py @@ -294,6 +294,12 @@ def __str__(self): sys.exit(1) i += 1 + # In one case, Win.Trojan.AgentTesla-9846789-0, rule 7 is unused, and yara needs all rules to be used + # If there are any numbers in the conditions that are skipped, then the rule should be rejected + rule_numbers = set([int(x[2:]) for x in re.findall(r"\$a\d+", conditions)]) + if len(set(range(0, max(rule_numbers)+1)).difference(rule_numbers)) != 0: + raise MalformedRuleError("Rule missing from conditions") + return yara_rule_template % (self._rulename, self._meta_signature, signatures, conditions) @@ -363,7 +369,7 @@ def parse_ldb(input, output, is_daily=False, is_quiet=False): if any('!' in r for r in rules): # Skip rules containing "!" such as "...!(01|02|03)..." continue # which do not translate to Yara rules. - # I cannot find any documentation about why there would be a colon in the logical expression, + # I cannot find any documentation about why there would be a colon in the logical expression, # but it occurs in only one daily case, Win.Trojan.Agent-6825810-0-6852456-0 # 0:0&((1>20&2>10&3)|(4)) # This causes some malformed yara where the 0 rule specifier is duplicated From ddef25236282756f3860363008ba9a29a8073304 Mon Sep 17 00:00:00 2001 From: Amelia Shaw Date: Wed, 22 Jul 2026 14:07:42 +0100 Subject: [PATCH 6/6] Added check for invalid rule alternate groups. Adjusted previous comments --- bin/yara_rules/parse_clamav.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/bin/yara_rules/parse_clamav.py b/bin/yara_rules/parse_clamav.py index 37ff87a..422b2d5 100755 --- a/bin/yara_rules/parse_clamav.py +++ b/bin/yara_rules/parse_clamav.py @@ -122,13 +122,21 @@ def _translate_signature(self, sig, index): # This checks for PCRE subsignatures. These consist of a trigger and a regex to perform when the trigger is met # https://docs.clamav.net/manual/Signatures/LogicalSignatures.html#pcre-subsignatures # Example found on in Win_dot_Trojan_dot_Zebrocy_dash_6743852_dash_2: - # 0|(1&2)/6#?87474703A2F2F(3[0-9])[1,3]2E(3[0-9])[1,3]2E(3[0-9])[1,3]2E(3[0-9])[1,3]2F(3[0-9]|[46][1-9A-F]|[57][0-9]|5A|7A|5F|2F|2D)+2E706870/ + # 0|(1&2)/6#?87474703A2F2F(3[0-9])[1,3]2E(3[0-9])[1,3]2E(3[0-9])[1,3]2E(3[0-9])[1,3]2F(3[0-9]|[46][1-9A-F]|[57][0-9]|5A|7A|5F|2F|2D)+2E706870/ # This rule checks if subsignatures 0 or 1 and 2 are met, and if so it runs the regex # This could be applied to the yara rules, but it would require rewriting the conditions so that rule 3 is the regex on its own, and the conditions are (0 | (1 & 2)) & 3 - # These were often caught by the previous check that looks for non-hexadecimal alphanumerics, but that doesnt trigger in cases like above + # These were often caught by the previous check that looks for non-hexadecimal alphanumerics, but that doesn't trigger in cases like above if re.match(r"[\d()<>&|=,]*\/.*\/", s): raise MalformedRuleError("Malformed rule: %s (%s)" % (self._meta_signature, s)) + # This excludes rules that for some reason have an alternate hex value group, but no alternates. I can only think this must be malformed + # I've only seen this in one rule, Win.Backdoor.CrimsonRAT-9953760-0, where + # 11046f2f00000a2526031f4028650000065a11046f2f00000a(2526|)5b5a1f4428650000065b130503031f4828650000065a11046f3000000a5b035a1f4c2865000006 + # contains (2526|), which is meant to have hex values after the |, but doesn't. I can't find anything in the docs saying this is valid + # The regex finds parenthesis pairs ending in |), which is invalid + if re.findall(r"\([^)]+\|\)", s): + raise MalformedRuleError("Malformed rule: %s (%s)" % (self._meta_signature, s)) + self._signatures.append("$a%d = { %s }" % (index, s)) def _translate_offset(self, offset, index): @@ -294,7 +302,7 @@ def __str__(self): sys.exit(1) i += 1 - # In one case, Win.Trojan.AgentTesla-9846789-0, rule 7 is unused, and yara needs all rules to be used + # In one case, Win.Trojan.AgentTesla-9846789-0, rule 7 isn't used in the conditions anywhere, and yara needs all rules to be used # If there are any numbers in the conditions that are skipped, then the rule should be rejected rule_numbers = set([int(x[2:]) for x in re.findall(r"\$a\d+", conditions)]) if len(set(range(0, max(rule_numbers)+1)).difference(rule_numbers)) != 0: @@ -373,7 +381,7 @@ def parse_ldb(input, output, is_daily=False, is_quiet=False): # but it occurs in only one daily case, Win.Trojan.Agent-6825810-0-6852456-0 # 0:0&((1>20&2>10&3)|(4)) # This causes some malformed yara where the 0 rule specifier is duplicated - # I'm not sure what possible configurations it could be, but since the logical expression generally cannot accept a colon, + # I'm not sure what possible values it could be, but since the logical expression generally cannot accept a colon, # it makes sense to take the second group if it has one if ":" in logical_expression: logical_expression = logical_expression.split(":")[1]