sentence resolution tests
@@ -12,7 +12,7 @@ class SentenceResolverTrainingTests(unittest.TestCase):
it will be used instead of the default glove
"""
import pandas as pd
- cols = ["y","_term","text"]
+ cols = ["y","_y","text"]
p='/home/ckl/Documents/freelance/jsl/nlu/nlu4realgit2/tests/datasets/AskAPatient.fold-0.train.txt'
dataset = pd.read_csv(p,sep="\t",encoding="ISO-8859-1",header=None)
dataset.columns = cols
@@ -42,13 +42,11 @@ class SentenceResolverTrainingTests(unittest.TestCase):
Alternatively, if a Word Embedding is specified in the load command before the train.chunk_resolver,
it will be used instead of the default glove
"""
- dataset = pd.DataFrame({
- 'text': ['super sleepy', 'bleeding from ears','bleeding from nose','bleeding from mouth'],
- '_extra_info': ['bad disease', 'bad disease!', 'very bad', ' super bad '],
- # 'y': [1,33,44,66]
- # 'label': ['lol','kek','lol','kek']
- 'label': ['lol','kek','lol','kek']
- })
+ import pandas as pd
+ cols = ["y","_y","text"]
+ p='/home/ckl/Documents/freelance/jsl/nlu/nlu4realgit2/tests/datasets/AskAPatient.fold-0.train.txt'
+ dataset = pd.read_csv(p,sep="\t",encoding="ISO-8859-1",header=None)
+ dataset.columns = cols
SPARK_NLP_LICENSE = sct.SPARK_NLP_LICENSE
AWS_ACCESS_KEY_ID = sct.AWS_ACCESS_KEY_ID
@@ -58,11 +56,7 @@ class SentenceResolverTrainingTests(unittest.TestCase):
nlu.auth(SPARK_NLP_LICENSE,AWS_ACCESS_KEY_ID,AWS_SECRET_ACCESS_KEY,JSL_SECRET)
# trainable_pipe = nlu.load('glove train.resolve_chunks', verbose=True)
- trainable_pipe = nlu.load('en.embed.glove.biovec train.resolve_chunks', verbose=True)
- # TODO bert/elmo give wierd storage ref errors...
- # TODO WRITE ISSUE IN HEALTHCARE LIB ABOUT THIS!!!
- # ONLY GLOVE WORKS!!
- # trainable_pipe = nlu.load('bert train.resolve_chunks', verbose=True)
+ trainable_pipe = nlu.load('embed_sentence.bert train.resolve_sentence', verbose=True)
trainable_pipe.print_info()
fitted_pipe = trainable_pipe.fit(dataset)
res = fitted_pipe.predict(dataset, multithread=False)
