Started docstring updates for QA utils.
@@ -28,10 +28,8 @@ import jsonlines
import torch
from torch.utils.data import TensorDataset, SequentialSampler, DataLoader
-
-from pytorch_transformers.tokenization_bert import BasicTokenizer
+from pytorch_transformers.tokenization_bert import BasicTokenizer, whitespace_tokenize
from pytorch_transformers import XLNetTokenizer
-from pytorch_transformers.tokenization_bert import whitespace_tokenize
from pytorch_transformers.modeling_bert import (
BertConfig,
@@ -55,7 +53,8 @@ CONFIG_CLASS = {}
CONFIG_CLASS.update({k: BertConfig for k in BERT_PRETRAINED_MODEL_ARCHIVE_MAP})
CONFIG_CLASS.update({k: XLNetConfig for k in XLNET_PRETRAINED_MODEL_ARCHIVE_MAP})
-
+# cached files during preprocessing
+# these are used in postprocessing to generate the final answer texts
CACHED_EXAMPLES_TRAIN_FILE = "cached_examples_train.jsonl"
CACHED_FEATURES_TRAIN_FILE = "cached_features_train.jsonl"
@@ -70,7 +69,26 @@ def _list_supported_models():
class QAProcessor:
- def __init__(self, model_name, custom_tokenize=None, to_lower=False, cache_dir="."):
+ """
+ Class for preprocessing raw question answering data to generate train/test features.
+
+ Args:
+ model_name (str): Name of the model. Call QAProcessor.list_supported_models() to
+ get all supported models.
+ to_lower (bool, optional): Whether to convert all letters to lower case during
+ tokenization. This is determined by if a cased model is used. Defaults to False,
+ which corresponds to a cased model.
+ custom_tokenize (function, optional): A custom tokenize function used to tokenize the
+ input text. If not provided, the default tokenizer corresponding to the model_name
+ is loaded and its `tokenize` method is used. NOTE that even this function is
+ provided, the numerical token ids are still generated by the `convert_tokens_to_ids`
+ method of the default tokenizer, so there is a risk that tokens generated by the
+ custom_tokenize function don't have correponding token ids in the default toeknizer.
+ Defaults to None.
+ cache_dir (str, otpional): Directory to cache the tokenizer. Defaults to ".".
+ """
+
+ def __init__(self, model_name, to_lower=False, custom_tokenize=None, cache_dir="."):
self.tokenizer = TOKENIZER_CLASS[model_name].from_pretrained(
model_name, do_lower_case=to_lower, cache_dir=cache_dir
)
@@ -134,6 +152,12 @@ class QAProcessor:
doc_stride=128,
cache_dir="./cached_qa_features",
):
+ """
+ Preprocesses raw question answering data and generates train/test features.
+
+ Args:
+ qa_dataset ():
+ """
if not os.path.exists(cache_dir):
os.makedirs(cache_dir)
