Default to higher performacs gzip.  Fix file extention problem.
"- `wikidata_input_path`: a folder containing the Wikidata KGTK edges to partition. \n",
"- `wikidata_parts_path`: a folder containing the part files of Wikidata, including files such as `part.wikibase-item.tsv.gz`\n",
"- `temp_folder_path`: a folder that may be used for temporary files. \n",
- "- `gzip_command`: The compression command for sorting. (default: gzip). \n",
+ "- `gzip_command`: The compression command for sorting. (default: pigz). \n",
"- `sort_extras`: Extra parameters for the sort program. \n",
"- `unsorted_extension`: The file extension for unsorted files. (default: .unsorted.tsv.gz) \n",
"- `sorted_extension`: The file extension for sorted files. (default: .tsv.gz) \n",
"wikidata_input_path = '/data4/rogers/elicit/cache/datasets/wikidata-20200803/data/everything.tsv.gz'\n",
"wikidata_parts_path = '/data4/rogers/elicit/cache/datasets/wikidata-20200803/parts'\n",
"temp_folder_path = wikidata_parts_path + '/temp'\n",
- "gzip_command = 'gzip'\n",
+ "gzip_command = 'pigz'\n",
"sort_extras = '-T ' + wikidata_parts_path\n",
"unsorted_extension = 'unsorted.tsv.gz'\n",
"sorted_extension = 'tsv.gz'\n",
- "use_mgzip = 'False'\n",
+ "use_mgzip = 'True'\n",
"verbose = 'True'\n"
]
},
"# The unsorted file extension.\n",
"os.environ['UNSORTED_EXTENSION'] = unsorted_extension\n",
"# The sorted file extension.\n",
- "os.environ['SORTED_EXTENSION'] = unsorted_extension\n",
+ "os.environ['SORTED_EXTENSION'] = sorted_extension\n",
"# The use_mgzip flag.\n",
"os.environ['USE_MGZIP'] = use_mgzip\n",
"# The verbose flag.\n",
