diff --git a/bin/clean_utf8.py b/bin/clean_utf8.py index 1b62f63..f448a50 100755 --- a/bin/clean_utf8.py +++ b/bin/clean_utf8.py @@ -38,7 +38,6 @@ __version__ = "1.1" - class CleanUTF8: """ Clean up spaces, control characters, hyphen and such in utf8 corpora. @@ -142,7 +141,9 @@ def clean_line(self, line: str) -> str: # Basic wide punctuation mapping if self.wide_punct: line = self.re_wide.sub(r" \g<1> ", line) - line = line.translate(str.maketrans(",。:)(;?﹗.﹪﹡﹟", ",.:)(;?!.%*#")) + line = line.translate( + str.maketrans(",。:)(;?﹗.﹪﹡﹟", ",.:)(;?!.%*#") + ) # Collapse multiple spaces to a single space line = self.re_mspace.sub(" ", line) @@ -161,7 +162,6 @@ def progress(*args): print("\r", *args, sep="", end="", file=sys.stderr) - @click.command() @click.option( "-v", @@ -232,9 +232,10 @@ def main( normalization_type=normalization_type, ) - with open(str(infile), mode="r", encoding="UTF-8", newline="\n") as cin, open( - str(outfile), mode="w", encoding="UTF-8" - ) as cout: + with ( + open(str(infile), mode="r", encoding="UTF-8", newline="\n") as cin, + open(str(outfile), mode="w", encoding="UTF-8") as cout, + ): cin = map(str.strip, cin) for count, line in enumerate(cin, 1): if verbose and count % 1000 == 0: @@ -242,8 +243,5 @@ def main( print(clean(line), file=cout) - - - if __name__ == "__main__": main() diff --git a/bin/filter-parallel.py b/bin/filter-parallel.py index 2ab4bfb..4644d09 100755 --- a/bin/filter-parallel.py +++ b/bin/filter-parallel.py @@ -12,39 +12,42 @@ # Copyright 2015, Sa Majeste la Reine du Chef du Canada / # Copyright 2015, Her Majesty in Right of Canada -import sys -import os.path from argparse import ArgumentParser, FileType, Action from portage_utils import ( - open, - printCopyright, - DebugAction, - HelpAction, - VerboseAction, + fatal_error, + open, + printCopyright, + DebugAction, + HelpAction, + VerboseAction, ) class Op(object): - none = 0 - gt = 1 - ge = 2 - lt = 3 - le = 4 + none = 0 + gt = 1 + ge = 2 + lt = 3 + le = 4 + class OpAction(Action): - """A custom action is needed to store both the operator and threshold.""" - def __init__(self, option_strings, dest, **kwargs): - super(OpAction, self).__init__(option_strings, dest, **kwargs) - def __call__(self, parser, namespace, values, option_string=None): - setattr(namespace, self.dest, self.const) - setattr(namespace, self.dest+"_threshold", values) + """A custom action is needed to store both the operator and threshold.""" + + def __init__(self, option_strings, dest, **kwargs): + super(OpAction, self).__init__(option_strings, dest, **kwargs) + + def __call__(self, parser, namespace, values, option_string=None): + setattr(namespace, self.dest, self.const) + setattr(namespace, self.dest + "_threshold", values) + def get_args(): - """Command line argument processing.""" + """Command line argument processing.""" - usage="filter-parallel.py [options] scores_file in_file1 [in_file2 ...]" - help=""" + usage = "filter-parallel.py [options] scores_file in_file1 [in_file2 ...]" + help = """ Filter lines in parallel from multiple line-aligned files according to a score in the provided , removing those lines whose score fails to satisfy a specified threshold test. Write output to , where @@ -52,62 +55,113 @@ def get_args(): All files, including the scores file, must contain the same number of lines. """ - # Use the argparse module, not the deprecated optparse module. - parser = ArgumentParser(usage=usage, description=help, add_help=False) - - # Use our standard help, verbose and debug support. - parser.add_argument("-h", "-help", "--help", action=HelpAction) - parser.add_argument("-v", "--verbose", action=VerboseAction) - parser.add_argument("-d", "--debug", action=DebugAction) - - parser.add_argument("-ext", dest="ext", type=str, default=".filt", - help="extension for output files [%(default)s]") + # Use the argparse module, not the deprecated optparse module. + parser = ArgumentParser(usage=usage, description=help, add_help=False) + + # Use our standard help, verbose and debug support. + parser.add_argument("-h", "-help", "--help", action=HelpAction) + parser.add_argument("-v", "--verbose", action=VerboseAction) + parser.add_argument("-d", "--debug", action=DebugAction) + + parser.add_argument( + "-ext", + dest="ext", + type=str, + default=".filt", + help="extension for output files [%(default)s]", + ) + + grp_op = parser.add_argument_group( + "Threshold operator selection options (one required)" + ) + ops = grp_op.add_mutually_exclusive_group(required=True) + ops.add_argument( + "-gt", + dest="op", + action=OpAction, + const=Op.gt, + type=float, + metavar="THRESHOLD", + help="""Keep if greater than threshold""", + ) + ops.add_argument( + "-ge", + dest="op", + action=OpAction, + const=Op.ge, + type=float, + metavar="THRESHOLD", + help="""Keep if greater than or equal to threshold""", + ) + ops.add_argument( + "-lt", + dest="op", + action=OpAction, + const=Op.lt, + type=float, + metavar="THRESHOLD", + help="""Keep if less than threshold""", + ) + ops.add_argument( + "-le", + dest="op", + action=OpAction, + const=Op.le, + type=float, + metavar="THRESHOLD", + help="""Keep if less than or equal to threshold""", + ) + + parser.add_argument( + "scores_file", + type=FileType("r", encoding="utf8"), + help="files to strip lines from in parallel", + ) + + parser.add_argument( + "in_files", + nargs="+", + type=FileType("r", encoding="utf8"), + help="file of scores to use for filtering", + ) + + try: + cmd_args = parser.parse_args() + except IOError as e: + fatal_error("cannot open: '{0}': {1}".format(e.filename, e)) + + return cmd_args - grp_op = parser.add_argument_group("Threshold operator selection options (one required)") - ops = grp_op.add_mutually_exclusive_group(required=True) - ops.add_argument('-gt', dest="op", action=OpAction, const=Op.gt, type=float, - metavar="THRESHOLD", help='''Keep if greater than threshold''') - ops.add_argument('-ge', dest="op", action=OpAction, const=Op.ge, type=float, - metavar="THRESHOLD", help='''Keep if greater than or equal to threshold''') - ops.add_argument('-lt', dest="op", action=OpAction, const=Op.lt, type=float, - metavar="THRESHOLD", help='''Keep if less than threshold''') - ops.add_argument('-le', dest="op", action=OpAction, const=Op.le, type=float, - metavar="THRESHOLD", help='''Keep if less than or equal to threshold''') - - parser.add_argument("scores_file", type=FileType('r', encoding="utf8"), - help="files to strip lines from in parallel") - - parser.add_argument("in_files", nargs="+", type=FileType('r', encoding="utf8"), - help="file of scores to use for filtering") - - try: - cmd_args = parser.parse_args() - except IOError as e: - fatal_error("cannot open: '{0}': {1}".format(e.filename, e)) - - return cmd_args def main(): - printCopyright("filter-parallel.py", 2015); - - cmd_args = get_args() - out_files = tuple(open(f.name+cmd_args.ext, 'w', encoding="utf_8") for f in cmd_args.in_files) - - for score_line in cmd_args.scores_file: - score = float(score_line) - lines = [] - for f in cmd_args.in_files: - lines.append(f.readline()) - if cmd_args.op is Op.gt and score > cmd_args.op_threshold or \ - cmd_args.op is Op.ge and score >= cmd_args.op_threshold or \ - cmd_args.op is Op.lt and score < cmd_args.op_threshold or \ - cmd_args.op is Op.le and score <= cmd_args.op_threshold: + printCopyright("filter-parallel.py", 2015) + cmd_args = get_args() + out_files = tuple( + open(f.name + cmd_args.ext, "w", encoding="utf_8") for f in cmd_args.in_files + ) + + for score_line in cmd_args.scores_file: + score = float(score_line) + lines = [] + for f in cmd_args.in_files: + lines.append(f.readline()) + if ( + cmd_args.op is Op.gt + and score > cmd_args.op_threshold + or cmd_args.op is Op.ge + and score >= cmd_args.op_threshold + or cmd_args.op is Op.lt + and score < cmd_args.op_threshold + or cmd_args.op is Op.le + and score <= cmd_args.op_threshold + ): for i in range(len(lines)): - print(lines[i], file=out_files[i], end='') + print(lines[i], file=out_files[i], end="") + + for f in cmd_args.in_files: + if len(f.readline()) != 0: + fatal_error("File", f.name, "contains more lines than some other files.") - for f in cmd_args.in_files: - if len(f.readline()) != 0: - fatal_error("File", f.name, "contains more lines than some other files.") -if __name__ == '__main__': - main() +if __name__ == "__main__": + main() diff --git a/bin/lines.py b/bin/lines.py index ae4af9a..b2a9932 100755 --- a/bin/lines.py +++ b/bin/lines.py @@ -13,18 +13,19 @@ # Copyright 2008, Sa Majeste la Reine du Chef du Canada / # Copyright 2008, Her Majesty in Right of Canada -import gzip import io import sys from portage_utils import open if len(sys.argv) != 3: - sys.stderr.write("Usage: lines.py \n\ + sys.stderr.write( + "Usage: lines.py \n\ \n\n\ Extracts lines specified in first file from second file.\n\ Line numbers have to start with 1 (not 0) and may contain repetitions.\n\ Output will be sorted by line numbers.\n\ -") +" + ) sys.exit(1) ### Read arguments @@ -42,15 +43,15 @@ line = txtFile.readline() done = False while (line != "") and (not done): - #print("%",n1,n2) + # print("%",n1,n2) while n1 == n2: - #sys.stderr.write("Line %i: %s\n" % (n2,line)) - #sys.stdout.write(str(line)) - print(line, end='') + # sys.stderr.write("Line %i: %s\n" % (n2,line)) + # sys.stdout.write(str(line)) + print(line, end="") if len(nums) == 0: done = True break n2 = nums.pop(0) - #print("#", n1, line, end=' ') - n1 = n1+1 + # print("#", n1, line, end=' ') + n1 = n1 + 1 line = txtFile.readline() diff --git a/bin/select-lines.py b/bin/select-lines.py index c8120e3..0e9cafb 100755 --- a/bin/select-lines.py +++ b/bin/select-lines.py @@ -29,132 +29,175 @@ def get_args(): - """Command line argument processing.""" + """Command line argument processing.""" - usage = "select-lines.py [options] indexfile [infile [outfile]]" - help = """ + usage = "select-lines.py [options] indexfile [infile [outfile]]" + help = """ Select a set of lines by index from a file. indexfile contains 1-based integer indicies of lines to be extracted. indexfile is assumed to be sorted. """ - parser = ArgumentParser(usage=usage, description=help, - formatter_class=RawDescriptionHelpFormatter, add_help=False) - parser.add_argument("-h", "-help", "--help", action=HelpAction) - parser.add_argument("-v", "--verbose", action=VerboseAction) - parser.add_argument("-d", "--debug", action=DebugAction) - parser.add_argument("-a", "--alignment-column", dest="alignment_column", default=0, type=int, - help="indexfile is an alignment info file from ssal -a; process given column: 1 or 2") - parser.add_argument("--joiner", dest="joiner", default=" ", type=str, - help="with -a, join lines in a range with given joiner [one space]") - parser.add_argument("--separator", dest="separator", default="\n", type=str, - help="with -a, separate ranges with given separator [one newline]") + parser = ArgumentParser( + usage=usage, + description=help, + formatter_class=RawDescriptionHelpFormatter, + add_help=False, + ) + parser.add_argument("-h", "-help", "--help", action=HelpAction) + parser.add_argument("-v", "--verbose", action=VerboseAction) + parser.add_argument("-d", "--debug", action=DebugAction) + parser.add_argument( + "-a", + "--alignment-column", + dest="alignment_column", + default=0, + type=int, + help="indexfile is an alignment info file from ssal -a; process given column: 1 or 2", + ) + parser.add_argument( + "--joiner", + dest="joiner", + default=" ", + type=str, + help="with -a, join lines in a range with given joiner [one space]", + ) + parser.add_argument( + "--separator", + dest="separator", + default="\n", + type=str, + help="with -a, separate ranges with given separator [one newline]", + ) + + parser.add_argument( + "indexfile", + type=lambda f: open(f, "r", encoding="utf-8"), + help="sorted index file", + ) + + parser.add_argument( + "infile", + nargs="?", + type=lambda f: open(f, "r", encoding="utf-8"), + default=io.TextIOWrapper(sys.stdin.buffer, encoding="utf-8"), + help="input file [sys.stdin]", + ) + + parser.add_argument( + "outfile", + nargs="?", + type=lambda f: open(f, "w", encoding="utf-8"), + default=io.TextIOWrapper(sys.stdout.buffer, encoding="utf-8"), + help="output file [sys.stdout]", + ) + + cmd_args = parser.parse_args() + + return cmd_args - parser.add_argument("indexfile", - type=lambda f: open(f, "r", encoding="utf-8"), - help="sorted index file") - parser.add_argument("infile", nargs='?', - type=lambda f: open(f, "r", encoding="utf-8"), - default=io.TextIOWrapper(sys.stdin.buffer, encoding="utf-8"), - help="input file [sys.stdin]") - - parser.add_argument("outfile", nargs='?', - type=lambda f: open(f, "w", encoding="utf-8"), - default=io.TextIOWrapper(sys.stdout.buffer, encoding="utf-8"), - help="output file [sys.stdout]") - - cmd_args = parser.parse_args() - - return cmd_args +def parse_alignment_line(line, column): + tokens = line.split() + try: + (start, end) = tokens[column - 1].split("-", 1) + start = int(start) + end = int(end) + except (ValueError, TypeError): + fatal_error("Invalid alignment info line:", line.strip()) + if end < start: + fatal_error("Invalid alignment has end index: - fatal_error("Index file out of sort order at index:", index, "input line:", line_number) - - if index_line: - fatal_error("Out of input before end of index file at index:", index) - - elif cmd_args.alignment_column == 1 or cmd_args.alignment_column == 2: - col = cmd_args.alignment_column - index_line = indexfile.readline() - if index_line: - (start, end) = parse_alignment_line(index_line, col) - if start < 0: - fatal_error("Alignment file specifies negative line number at:", index_line.strip()) - - for in_line in infile: - if not index_line: - break - if line_number >= start and line_number < end: - print(in_line.strip('\n'), file=outfile, end='') - if line_number+1 < end: - print(cmd_args.joiner, file=outfile, end='') - - line_number += 1 - while line_number == end: - print(cmd_args.separator, file=outfile, end='') - index_line = indexfile.readline() - if index_line: - (start, end) = parse_alignment_line(index_line, col) - if start < line_number: - fatal_error("Alignment file out of order at:", index_line.strip()) - else: - break - - if index_line: - fatal_error("Out of input before end of alignment index file at:", index_line.strip()) - - else: - fatal_error("invalid -a/--alignment-column value: use 1 or 2 (or 0 for none).") - - indexfile.close() - infile.close() - outfile.close() - -if __name__ == '__main__': + printCopyright("select-lines.py", 2018) + os.environ["PORTAGE_INTERNAL_CALL"] = "1" + + cmd_args = get_args() + + indexfile = cmd_args.indexfile + infile = cmd_args.infile + outfile = cmd_args.outfile + + # The following allows stderr to handle non-ascii characters: + sys.stderr = codecs.getwriter("utf-8")(sys.stderr.detach()) + + line_number = 0 + + if cmd_args.alignment_column == 0: + index_line = indexfile.readline() + if index_line: + index = int(index_line) + + for in_line in infile: + if not index_line: + break + line_number += 1 + if line_number == index: + print(in_line, file=outfile, end="") + index_line = indexfile.readline() + if index_line: + index = int(index_line) + elif line_number > index: + fatal_error( + "Index file out of sort order at index:", + index, + "input line:", + line_number, + ) + + if index_line: + fatal_error("Out of input before end of index file at index:", index) + + elif cmd_args.alignment_column == 1 or cmd_args.alignment_column == 2: + col = cmd_args.alignment_column + index_line = indexfile.readline() + if index_line: + (start, end) = parse_alignment_line(index_line, col) + if start < 0: + fatal_error( + "Alignment file specifies negative line number at:", + index_line.strip(), + ) + + for in_line in infile: + if not index_line: + break + if line_number >= start and line_number < end: + print(in_line.strip("\n"), file=outfile, end="") + if line_number + 1 < end: + print(cmd_args.joiner, file=outfile, end="") + + line_number += 1 + while line_number == end: + print(cmd_args.separator, file=outfile, end="") + index_line = indexfile.readline() + if index_line: + (start, end) = parse_alignment_line(index_line, col) + if start < line_number: + fatal_error( + "Alignment file out of order at:", index_line.strip() + ) + else: + break + + if index_line: + fatal_error( + "Out of input before end of alignment index file at:", + index_line.strip(), + ) + + else: + fatal_error("invalid -a/--alignment-column value: use 1 or 2 (or 0 for none).") + + indexfile.close() + infile.close() + outfile.close() + + +if __name__ == "__main__": main() diff --git a/bin/select-random-chunks.py b/bin/select-random-chunks.py index f119bf1..3e6673e 100755 --- a/bin/select-random-chunks.py +++ b/bin/select-random-chunks.py @@ -13,22 +13,27 @@ # Copyright 2020, Her Majesty in Right of Canada import sys -# import codecs -# import re + from argparse import ArgumentParser, RawDescriptionHelpFormatter import os -import os.path import subprocess import random -from portage_utils import * +from portage_utils import ( + HelpAction, + VerboseAction, + DebugAction, + fatal_error, + printCopyright, + verbose, +) def get_args(): - """Command line argument processing.""" + """Command line argument processing.""" -# usage = "select-random-chunks.py [options] [outfile]" - help = """ + # usage = "select-random-chunks.py [options] [outfile]" + help = """ Select a number of random chunks of a specified size producing an index file. The generated indicies are 1-based. @@ -36,88 +41,131 @@ def get_args(): outfile can be used as an indexfile for select-lines.py. """ -# parser = ArgumentParser(usage=usage, description=help, add_help=False, -# formatter_class=RawDescriptionHelpFormatter) - parser = ArgumentParser(description=help, add_help=False, - formatter_class=RawDescriptionHelpFormatter) - parser.add_argument("-h", "-help", "--help", action=HelpAction) - parser.add_argument("-v", "--verbose", action=VerboseAction) - parser.add_argument("-d", "--debug", action=DebugAction) - - parser.add_argument("-c", "--chunk-size", dest="chunk_size", default=1, type=int, - help="Size of chunks to select [%(default)s]") - - group1 = parser.add_mutually_exclusive_group(required=True) - group1.add_argument("-n", "--number-chunks", dest="num_chunks", type=int, - help="Number of chunks to select") - group1.add_argument("-o", "--outsize", dest="output_size", type=int, - help="Target size for outfile [num_chunks * chunk_size]") - - group2 = parser.add_mutually_exclusive_group(required=True) - group2.add_argument("-m", "--max-index", dest="max_index", type=int, - help="Number of chunks to select [%(default)s]") - group2.add_argument("-f", "--infile", dest="infile", type=str, - help="File whose size determines max_index") - - parser.add_argument("-s", "--seed", dest="seed", default=2020, type=int, - help="Seed for random number generator. [%(default)s]") - - parser.add_argument("outfile", nargs='?', type=lambda f: open(f,'w'), default=sys.stdout, - help="output file [sys.stdout]") - - cmd_args = parser.parse_args() - - return cmd_args - - -def file_size( filename ): - result = subprocess.run(['wc', '-l', filename], stdout=subprocess.PIPE) - if result.returncode != 0: - fatal_error("Cannot access (using wc -l):", filename) - return int(result.stdout.decode().split()[0]) + # parser = ArgumentParser(usage=usage, description=help, add_help=False, + # formatter_class=RawDescriptionHelpFormatter) + parser = ArgumentParser( + description=help, add_help=False, formatter_class=RawDescriptionHelpFormatter + ) + parser.add_argument("-h", "-help", "--help", action=HelpAction) + parser.add_argument("-v", "--verbose", action=VerboseAction) + parser.add_argument("-d", "--debug", action=DebugAction) + + parser.add_argument( + "-c", + "--chunk-size", + dest="chunk_size", + default=1, + type=int, + help="Size of chunks to select [%(default)s]", + ) + + group1 = parser.add_mutually_exclusive_group(required=True) + group1.add_argument( + "-n", + "--number-chunks", + dest="num_chunks", + type=int, + help="Number of chunks to select", + ) + group1.add_argument( + "-o", + "--outsize", + dest="output_size", + type=int, + help="Target size for outfile [num_chunks * chunk_size]", + ) + + group2 = parser.add_mutually_exclusive_group(required=True) + group2.add_argument( + "-m", + "--max-index", + dest="max_index", + type=int, + help="Number of chunks to select [%(default)s]", + ) + group2.add_argument( + "-f", + "--infile", + dest="infile", + type=str, + help="File whose size determines max_index", + ) + + parser.add_argument( + "-s", + "--seed", + dest="seed", + default=2020, + type=int, + help="Seed for random number generator. [%(default)s]", + ) + + parser.add_argument( + "outfile", + nargs="?", + type=lambda f: open(f, "w"), + default=sys.stdout, + help="output file [sys.stdout]", + ) + + cmd_args = parser.parse_args() + + return cmd_args + + +def file_size(filename): + result = subprocess.run(["wc", "-l", filename], stdout=subprocess.PIPE) + if result.returncode != 0: + fatal_error("Cannot access (using wc -l):", filename) + return int(result.stdout.decode().split()[0]) def main(): - printCopyright("select-random-chunks.py", 2020); - os.environ['PORTAGE_INTERNAL_CALL'] = '1'; + printCopyright("select-random-chunks.py", 2020) + os.environ["PORTAGE_INTERNAL_CALL"] = "1" + cmd_args = get_args() - cmd_args = get_args() + random.seed(cmd_args.seed) - random.seed(cmd_args.seed) + if cmd_args.output_size is not None: + num_chunks = cmd_args.output_size // cmd_args.chunk_size + else: + num_chunks = cmd_args.num_chunks - if cmd_args.output_size is not None: - num_chunks = cmd_args.output_size // cmd_args.chunk_size - else: - num_chunks = cmd_args.num_chunks + if cmd_args.infile is not None: + max_index = file_size(cmd_args.infile) + else: + max_index = cmd_args.max_index - if cmd_args.infile is not None: - max_index = file_size(cmd_args.infile) - else: - max_index = cmd_args.max_index + verbose("seed:", cmd_args.seed) + verbose("chunk_size:", cmd_args.chunk_size) + if cmd_args.output_size is not None: + verbose("output_size: ", cmd_args.output_size) + verbose("num_chunks:", num_chunks) + if cmd_args.infile is not None: + verbose("infile:", cmd_args.infile) + verbose("max_index: ", max_index) + verbose("outfile: ", cmd_args.outfile) - verbose("seed:", cmd_args.seed) - verbose("chunk_size:", cmd_args.chunk_size) - if cmd_args.output_size is not None: - verbose("output_size: ", cmd_args.output_size) - verbose("num_chunks:", num_chunks) - if cmd_args.infile is not None: - verbose("infile:", cmd_args.infile) - verbose("max_index: ", max_index) - verbose("outfile: ", cmd_args.outfile) + if num_chunks * cmd_args.chunk_size > max_index: + fatal_error( + "num_chunks * chunk_size (", + num_chunks * cmd_args.chunk_size, + ") must be <= max_index (", + max_index, + ").", + ) - if num_chunks * cmd_args.chunk_size > max_index: - fatal_error("num_chunks * chunk_size (", num_chunks * cmd_args.chunk_size, - ") must be <= max_index (", max_index, ").") + max_range = max_index - (cmd_args.chunk_size - 1) + 1 + chunks = sorted(random.sample(range(1, max_range, cmd_args.chunk_size), num_chunks)) - max_range = max_index - (cmd_args.chunk_size-1) + 1 - chunks = sorted(random.sample(range(1, max_range, cmd_args.chunk_size), num_chunks)) + for index in chunks: + for i in range(cmd_args.chunk_size): + print(index + i, file=cmd_args.outfile) - for index in chunks: - for i in range(cmd_args.chunk_size): - print(index+i, file=cmd_args.outfile) + cmd_args.outfile.close() - cmd_args.outfile.close() -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/bin/strip-parallel-blank-lines.py b/bin/strip-parallel-blank-lines.py index 36318e7..5c873dd 100755 --- a/bin/strip-parallel-blank-lines.py +++ b/bin/strip-parallel-blank-lines.py @@ -54,24 +54,25 @@ ofiles = [] for file in args: ifiles.append(open(file, "r")) - ofiles.append(open(re.sub(r'(.gz$|$)', r'.no-blanks\g<1>', file, count=1), "w")) + ofiles.append(open(re.sub(r"(.gz$|$)", r".no-blanks\g<1>", file, count=1), "w")) second = 1 -if len(ifiles) < 2: second = 0 # only use file1 if only file1 given +if len(ifiles) < 2: + second = 0 # only use file1 if only file1 given lines = [""] * len(ifiles) for lines[0] in ifiles[0]: - for i in range(1, len(ifiles)): lines[i] = ifiles[i].readline() - if (lines[i] == ""): + if lines[i] == "": sys.stderr.write("file " + ifiles[i].name + " too short!\n") sys.exit(1) if replace: rep = blankline.match(lines[0]) for i in range(0, len(ifiles)): - if rep and blankline.match(lines[i]): lines[i] = ".\n" + if rep and blankline.match(lines[i]): + lines[i] = ".\n" ofiles[i].write(lines[i]) else: if not blankline.match(lines[0]) and not blankline.match(lines[second]): diff --git a/bin/strip-parallel-duplicates.py b/bin/strip-parallel-duplicates.py index 955e1fe..2717823 100755 --- a/bin/strip-parallel-duplicates.py +++ b/bin/strip-parallel-duplicates.py @@ -25,10 +25,10 @@ def get_args(): - """Command line argument processing.""" + """Command line argument processing.""" - usage = "strip-parallel-duplicates.py [options] file1 file2 [file3 ...]" - help = """ + usage = "strip-parallel-duplicates.py [options] file1 file2 [file3 ...]" + help = """ Strip lines in parallel from multiple line-aligned files if the lines from the compared files are identical. Write output to , where defaults to .dedup. @@ -37,55 +37,80 @@ def get_args(): the identical line comparison. """ - # Use the argparse module, not the deprecated optparse module. - parser = ArgumentParser(usage=usage, description=help, add_help=False) + # Use the argparse module, not the deprecated optparse module. + parser = ArgumentParser(usage=usage, description=help, add_help=False) + + # Use our standard help, verbose and debug support. + parser.add_argument("-h", "-help", "--help", action=HelpAction) + parser.add_argument("-v", "--verbose", action=VerboseAction) + parser.add_argument("-d", "--debug", action=DebugAction) + + parser.add_argument( + "-c", + dest="compare", + type=int, + default=2, + help="number of files to compare [%(default)s]", + ) + parser.add_argument( + "-ext", + dest="ext", + type=str, + default=".dedup", + help="extension for output files [%(default)s]", + ) + + parser.add_argument( + "in_files", + nargs="*", + type=FileType("r", encoding="utf-8"), + help="files to strip lines from in parallel", + ) + + cmd_args = parser.parse_args() + if cmd_args.compare < 2: + fatal_error("Number of files to compare (-c) must be >= 2: ", cmd_args.compare) + if len(cmd_args.in_files) < cmd_args.compare: + fatal_error(cmd_args.compare, "files required for comparison.") + + return cmd_args - # Use our standard help, verbose and debug support. - parser.add_argument("-h", "-help", "--help", action=HelpAction) - parser.add_argument("-v", "--verbose", action=VerboseAction) - parser.add_argument("-d", "--debug", action=DebugAction) - - parser.add_argument("-c", dest="compare", type=int, default=2, - help="number of files to compare [%(default)s]") - parser.add_argument("-ext", dest="ext", type=str, default=".dedup", - help="extension for output files [%(default)s]") - - parser.add_argument("in_files", nargs="*", type=FileType('r', encoding="utf-8"), - help="files to strip lines from in parallel") - - cmd_args = parser.parse_args() - if cmd_args.compare < 2: - fatal_error("Number of files to compare (-c) must be >= 2: ", cmd_args.compare) - if len(cmd_args.in_files) < cmd_args.compare: - fatal_error(cmd_args.compare, "files required for comparison.") - - return cmd_args def main(): - printCopyright("strip-parallel-duplicates.py", 2012) - - cmd_args = get_args() - out_files = tuple(open(f.name+cmd_args.ext, 'w', encoding="utf8") for f in cmd_args.in_files) - - eof = False - while True: - lines = [] - for f in cmd_args.in_files: - lines.append(f.readline()) - if len(lines[-1]) == 0: eof = True - if eof: break - for i in range(1, cmd_args.compare): - if lines[i] != lines[0]: identical = False; break - else: - identical = True - if not identical: - for i in range(len(lines)): - print(lines[i], file=out_files[i], end='') - - for i in range(len(cmd_args.in_files)): - if len(lines[i]) != 0: - fatal_error("File", cmd_args.in_files[i].name, - "contains more lines than some other files.") - -if __name__ == '__main__': - main() + printCopyright("strip-parallel-duplicates.py", 2012) + + cmd_args = get_args() + out_files = tuple( + open(f.name + cmd_args.ext, "w", encoding="utf8") for f in cmd_args.in_files + ) + + eof = False + while True: + lines = [] + for f in cmd_args.in_files: + lines.append(f.readline()) + if len(lines[-1]) == 0: + eof = True + if eof: + break + for i in range(1, cmd_args.compare): + if lines[i] != lines[0]: + identical = False + break + else: + identical = True + if not identical: + for i in range(len(lines)): + print(lines[i], file=out_files[i], end="") + + for i in range(len(cmd_args.in_files)): + if len(lines[i]) != 0: + fatal_error( + "File", + cmd_args.in_files[i].name, + "contains more lines than some other files.", + ) + + +if __name__ == "__main__": + main()