Updated decode script to support two dumps versions
@@ -110,15 +110,36 @@ def levenshtein(a, b):
current[j] = min(add, delete, change)
return current[n]
+
def load_dump(pickle_file):
with open(pickle_file, 'rb') as f:
data = pickle.load(f, encoding='bytes')
return data
+
+def get_logits(data, labels):
+ '''
+ Get logits from pickled data.
+ There are two versions of pickle file (and data):
+ 1. raw logits NumPy array
+ 2. dictionary with logits and additional meta information
+ '''
+ if isinstance(data, np.ndarray):
+ # convert NumPy array to dict format
+ logits = {}
+ for idx, line in enumerate(labels):
+ audio_filename = line[0]
+ logits[audio_filename] = data[idx]
+ else:
+ logits = data['logits']
+ return logits
+
+
def load_labels(csv_file):
labels = np.loadtxt(csv_file, skiprows=1, delimiter=',', dtype=str)
return labels
+
def load_vocab(vocab_file):
vocab = []
with open(vocab_file, 'r') as f:
@@ -127,6 +148,7 @@ def load_vocab(vocab_file):
vocab.append('_')
return vocab
+
def greedy_decoder(logits, vocab, merge=True):
s = ''
c = ''
@@ -140,11 +162,13 @@ def greedy_decoder(logits, vocab, merge=True):
s = s.replace('_', '')
return s
+
def softmax(x):
m = np.expand_dims(np.max(x, axis=-1), -1)
e = np.exp(x - m)
return e / np.expand_dims(e.sum(axis=-1), -1)
+
def evaluate_wer(data, labels, vocab, decoder):
total_dist = 0.0
total_count = 0.0
@@ -167,12 +191,14 @@ def evaluate_wer(data, labels, vocab, decoder):
data = load_dump(args.logits)
labels = load_labels(args.labels)
+logits = get_logits(data, labels)
vocab = load_vocab(args.vocab)
vocab[-1] = '_'
probs_batch = []
-for idx in range(len(data)):
- probs_batch.append(softmax(data[idx]))
+for line in labels:
+ audio_filename = line[0]
+ probs_batch.append(softmax(logits[audio_filename]))
if args.mode == 'eval':
wer, _ = evaluate_wer(data, labels, vocab, greedy_decoder)
@@ -181,7 +207,6 @@ if args.mode == 'eval':
for alpha in np.arange(args.alpha, args.alpha_max, args.alpha_step):
for beta in np.arange(args.beta, args.beta_max, args.beta_step):
scorer = Scorer(alpha, beta, model_path=args.lm, vocabulary=vocab[:-1])
-
res = ctc_beam_search_decoder_batch(probs_batch, vocab[:-1],
beam_size=args.beam_width,
num_processes=num_cpus,
@@ -213,7 +238,6 @@ if args.mode == 'eval':
f.write('{} 0.0 0.0 {}\n'.format(pred[0], pred[1]))
f.write('E=>>>>>>>>\n')
-
elif args.mode == 'infer':
scorer = Scorer(args.alpha, args.beta, model_path=args.lm, vocabulary=vocab[:-1])
res = ctc_beam_search_decoder_batch(probs_batch, vocab[:-1],
@@ -229,3 +253,4 @@ elif args.mode == 'infer':
np.savetxt(args.infer_output_file, infer_preds, fmt='%s', delimiter=',',
header='wav_filename,transcript')
+
