Wav2vec 2.0 (de, en, pl, ru, multi)

.gitattributes

@@ -33,3 +33,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+wav2vec2-end-of-speech-detection/5sec_audio.wav filter=lfs diff=lfs merge=lfs -text
+wav2vec2-lv-60-espeak-cv-ft-js/Simple[[:space:]]and[[:space:]]Effective[[:space:]]Zero-shot[[:space:]]Cross-lingual[[:space:]]Phoneme[[:space:]]Recognition.pdf filter=lfs diff=lfs merge=lfs -text
+wav2vec2-lv-60-espeak-cv-ft-ONNX/Simple[[:space:]]and[[:space:]]Effective[[:space:]]Zero-shot[[:space:]]Cross-lingual[[:space:]]Phoneme[[:space:]]Recognition.pdf filter=lfs diff=lfs merge=lfs -text

distil-wav2vec2-onnx/.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

distil-wav2vec2-onnx/README.md

@@ -0,0 +1,25 @@
+---
+language: en
+base_model:
+- OthmaneJ/distil-wav2vec2
+datasets:
+- librispeech_asr
+tags:
+- onnx
+- speech
+- audio
+- automatic-speech-recognition
+license: apache-2.0
+---
+# Distil-wav2vec2 ONNX
+
+This repository hosts ONNX exports of the Distil-wav2vec2 model.
+
+## Contents
+
+- `onnx/distil-wav2vec2_fp32.onnx`, `onnx/distil-wav2vec2_fp16.onnx`, `onnx/distil-wav2vec2_int8.onnx`
+
+## Upstream
+
+Original project:
+https://huggingface.co/OthmaneJ/distil-wav2vec2

distil-wav2vec2-onnx/onnx/distil-wav2vec2_fp16.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:26048304a2f60cf74d0ba3e58d53b0cc01ef24b3fda90ec3eeacff28dcd6443e
+size 104613131

distil-wav2vec2-onnx/onnx/distil-wav2vec2_fp32.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:51df0aae6a9314a4fdde8e664334d2529fa983a46e9a6c60facb9532c86e8532
+size 207542006

distil-wav2vec2-onnx/onnx/distil-wav2vec2_int8.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e994a38dcf2c4c24740a6afead518c6b7ea9add9c52d6c455a773b868b145e48
+size 52161794

distil-wav2vec2-onnx/source.txt

@@ -0,0 +1 @@
+https://huggingface.co/KevinAHM/distil-wav2vec2-onnx

wav2vec2-alignment/.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

wav2vec2-alignment/README.md

@@ -0,0 +1,44 @@
+---
+language:
+- en
+- multilingual
+license: apache-2.0
+tags:
+- onnx
+- audio
+- automatic-speech-recognition
+- phoneme-recognition
+- wav2vec2
+base_model: facebook/wav2vec2-lv-60-espeak-cv-ft
+---
+
+# Wav2Vec2-LV-60-Espeak-CV-FT (ONNX)
+
+This is an **ONNX export** of the [facebook/wav2vec2-lv-60-espeak-cv-ft](https://huggingface.co/facebook/wav2vec2-lv-60-espeak-cv-ft) model.
+
+It is designed for client-side inference in the **UltrClick ContentPro** application to perform forced alignment of lyrics to audio.
+
+## Model Details
+
+-   **Original Model**: `facebook/wav2vec2-lv-60-espeak-cv-ft`
+-   **Format**: ONNX (Open Neural Network Exchange)
+-   **Precision**: FP16 (Float16)
+-   **Output**: IPA Phoneme logits (392 vocab size)
+-   **Sample Rate**: 16kHz
+
+## Usage
+
+This model is intended to be used with the ONNX Runtime (e.g., via `ort` in Rust or `onnxruntime` in Python).
+
+### Input
+-   **Name**: `audio`
+-   **Shape**: `[batch_size, samples]`
+-   **Type**: Float32 tensor
+
+### Output
+-   **Name**: `logits`
+-   **Shape**: `[batch_size, frames, 392]` (392 is the vocab size)
+
+## License
+
+This model is a derivative of the original `facebook/wav2vec2-lv-60-espeak-cv-ft` model and retains the **Apache 2.0** license.

wav2vec2-alignment/languages.txt

@@ -0,0 +1,57 @@
+Abkhaz
+Arabic
+Assamese
+Breton
+Catalan
+Hakha Chin
+Czech
+Chuvash
+Welsh
+German
+Divehi
+Greek
+English
+Esperanto
+Spanish
+Estonian
+Basque
+Persian
+Finnish
+French
+Western Frisian
+Irish
+Hindi
+Upper Sorbian
+Hungarian
+Interlingua
+Indonesian
+Italian
+Japanese
+Georgian
+Kabyle
+Kyrgyz
+Ganda
+Lithuanian
+Latvian
+Mongolian
+Maltese
+Dutch
+Oriya
+Panjabi
+Polish
+Portuguese
+Romansh
+Romanian
+Russian
+Kinyarwanda
+Yakut
+Slovenian
+Swedish
+Tamil
+Thai
+Turkish
+Tatar
+Ukrainian
+Vietnamese
+Votic
+Chinese

wav2vec2-alignment/source.txt

@@ -0,0 +1 @@
+https://huggingface.co/Hochien/wav2vec2-alignment

wav2vec2-alignment/wav2vec2-lv60-espeak-fp16.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a65177b47f8b304dc11d962ce91a6943f54e55dca69caecaaa35beab80f49925
+size 632239986

wav2vec2-base-10k-voxpopuli-ft-pl-ONNX/.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

wav2vec2-base-10k-voxpopuli-ft-pl-ONNX/README.md

@@ -0,0 +1,89 @@
+---
+language: pl
+tags:
+- audio
+- automatic-speech-recognition
+- voxpopuli
+license: cc-by-nc-4.0
+library_name: transformers.js
+base_model:
+- facebook/wav2vec2-base-10k-voxpopuli-ft-pl
+pipeline_tag: automatic-speech-recognition
+---
+
+
+
+# wav2vec2-base-10k-voxpopuli-ft-pl (ONNX)
+
+
+This is an ONNX version of [facebook/wav2vec2-base-10k-voxpopuli-ft-pl](https://huggingface.co/facebook/wav2vec2-base-10k-voxpopuli-ft-pl). It was automatically converted and uploaded using [this Hugging Face Space](https://huggingface.co/spaces/onnx-community/convert-to-onnx).
+
+
+## Usage with Transformers.js
+
+
+See the pipeline documentation for `automatic-speech-recognition`: https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.AutomaticSpeechRecognitionPipeline
+
+
+---
+
+
+# Wav2Vec2-Base-VoxPopuli-Finetuned
+
+[Facebook's Wav2Vec2](https://ai.facebook.com/blog/wav2vec-20-learning-the-structure-of-speech-from-raw-audio/) base model pretrained on the 10K unlabeled subset of [VoxPopuli corpus](https://arxiv.org/abs/2101.00390) and fine-tuned on the transcribed data in pl (refer to Table 1 of paper for more information).
+
+**Paper**: *[VoxPopuli: A Large-Scale Multilingual Speech Corpus for Representation
+Learning, Semi-Supervised Learning and Interpretation](https://arxiv.org/abs/2101.00390)*
+
+**Authors**: *Changhan Wang, Morgane Riviere, Ann Lee, Anne Wu, Chaitanya Talnikar, Daniel Haziza, Mary Williamson, Juan Pino, Emmanuel Dupoux* from *Facebook AI*
+
+See the official website for more information, [here](https://github.com/facebookresearch/voxpopuli/)
+
+
+# Usage for inference
+
+In the following it is shown how the model can be used in inference on a sample of the [Common Voice dataset](https://commonvoice.mozilla.org/en/datasets)
+
+```python
+#!/usr/bin/env python3
+from transformers import Wav2Vec2Processor, Wav2Vec2ForCTC
+from datasets import load_dataset
+import torchaudio
+import torch
+
+# resample audio
+
+# load model & processor
+model = Wav2Vec2ForCTC.from_pretrained("facebook/wav2vec2-base-10k-voxpopuli-ft-pl")
+processor = Wav2Vec2Processor.from_pretrained("facebook/wav2vec2-base-10k-voxpopuli-ft-pl")
+
+# load dataset
+ds = load_dataset("common_voice", "pl", split="validation[:1%]")
+
+# common voice does not match target sampling rate
+common_voice_sample_rate = 48000
+target_sample_rate = 16000
+
+resampler = torchaudio.transforms.Resample(common_voice_sample_rate, target_sample_rate)
+
+
+# define mapping fn to read in sound file and resample
+def map_to_array(batch):
+    speech, _ = torchaudio.load(batch["path"])
+    speech = resampler(speech)
+    batch["speech"] = speech[0]
+    return batch
+
+
+# load all audio files
+ds = ds.map(map_to_array)
+
+# run inference on the first 5 data samples
+inputs = processor(ds[:5]["speech"], sampling_rate=target_sample_rate, return_tensors="pt", padding=True)
+
+# inference
+logits = model(**inputs).logits
+predicted_ids = torch.argmax(logits, axis=-1)
+
+print(processor.batch_decode(predicted_ids))
+```

wav2vec2-base-10k-voxpopuli-ft-pl-ONNX/config.json

@@ -0,0 +1,111 @@
+{
+  "_attn_implementation_autoset": true,
+  "_name_or_path": "facebook/wav2vec2-base-10k-voxpopuli-ft-pl",
+  "activation_dropout": 0.1,
+  "adapter_attn_dim": null,
+  "adapter_kernel_size": 3,
+  "adapter_stride": 2,
+  "add_adapter": false,
+  "apply_spec_augment": true,
+  "architectures": [
+    "Wav2Vec2ForCTC"
+  ],
+  "attention_dropout": 0.1,
+  "bos_token_id": 0,
+  "classifier_proj_size": 256,
+  "codevector_dim": 256,
+  "contrastive_logits_temperature": 0.1,
+  "conv_bias": false,
+  "conv_dim": [
+    512,
+    512,
+    512,
+    512,
+    512,
+    512,
+    512
+  ],
+  "conv_kernel": [
+    10,
+    3,
+    3,
+    3,
+    3,
+    2,
+    2
+  ],
+  "conv_stride": [
+    5,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2
+  ],
+  "ctc_loss_reduction": "sum",
+  "ctc_zero_infinity": false,
+  "diversity_loss_weight": 0.1,
+  "do_stable_layer_norm": false,
+  "eos_token_id": 2,
+  "feat_extract_activation": "gelu",
+  "feat_extract_dropout": 0.0,
+  "feat_extract_norm": "group",
+  "feat_proj_dropout": 0.1,
+  "feat_quantizer_dropout": 0.0,
+  "final_dropout": 0.1,
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout": 0.1,
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-05,
+  "layerdrop": 0.1,
+  "mask_feature_length": 10,
+  "mask_feature_min_masks": 0,
+  "mask_feature_prob": 0.0,
+  "mask_time_length": 10,
+  "mask_time_min_masks": 2,
+  "mask_time_prob": 0.05,
+  "model_type": "wav2vec2",
+  "num_adapter_layers": 3,
+  "num_attention_heads": 12,
+  "num_codevector_groups": 2,
+  "num_codevectors_per_group": 320,
+  "num_conv_pos_embedding_groups": 16,
+  "num_conv_pos_embeddings": 128,
+  "num_feat_extract_layers": 7,
+  "num_hidden_layers": 12,
+  "num_negatives": 100,
+  "output_hidden_size": 768,
+  "pad_token_id": 1,
+  "proj_codevector_dim": 256,
+  "tdnn_dilation": [
+    1,
+    2,
+    3,
+    1,
+    1
+  ],
+  "tdnn_dim": [
+    512,
+    512,
+    512,
+    512,
+    1500
+  ],
+  "tdnn_kernel": [
+    5,
+    3,
+    3,
+    1,
+    1
+  ],
+  "torch_dtype": "float32",
+  "transformers_version": "4.49.0",
+  "use_weighted_layer_sum": false,
+  "vocab_size": 41,
+  "xvector_output_dim": 512
+}

wav2vec2-base-10k-voxpopuli-ft-pl-ONNX/onnx/model.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9309e96f5bb53ba957a7646cc9c5a3d61bd47d930a6998d1863e0ec081c3b199
+size 377939575

wav2vec2-base-10k-voxpopuli-ft-pl-ONNX/onnx/model_bnb4.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b5a5d1dd62965a54abd99edf07f0bf99bd01811b4fe3fcbec94579ddbae8c016
+size 84528286

wav2vec2-base-10k-voxpopuli-ft-pl-ONNX/onnx/model_fp16.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4724fe007a6aabb2f2a8d63318d28e533491e29b3877752f5e80da92b34c15e4
+size 189132785

wav2vec2-base-10k-voxpopuli-ft-pl-ONNX/onnx/model_int8.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8f066fac7d0662feec0e64002a4a969032a02ee5a4048bb1959d324801ead168
+size 95219724

wav2vec2-base-10k-voxpopuli-ft-pl-ONNX/onnx/model_q4.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9265d2e19425b2240ac9c251e963d4a2fcda05741dd775076f4098b5e806b797
+size 89862702

wav2vec2-base-10k-voxpopuli-ft-pl-ONNX/onnx/model_q4f16.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e104f81133126cb900fdc017e1daecf45e3663668ab9666d1c40cb8ce3057745
+size 66439769

wav2vec2-base-10k-voxpopuli-ft-pl-ONNX/onnx/model_quantized.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb33c88ee1fc2010aebbb9a8eef0034638a1d418e3bb57cab01ba70c1a520ad3
+size 95219762

wav2vec2-base-10k-voxpopuli-ft-pl-ONNX/onnx/model_uint8.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb33c88ee1fc2010aebbb9a8eef0034638a1d418e3bb57cab01ba70c1a520ad3
+size 95219762

wav2vec2-base-10k-voxpopuli-ft-pl-ONNX/preprocessor_config.json

@@ -0,0 +1,9 @@
+{
+  "do_normalize": true,
+  "feature_extractor_type": "Wav2Vec2FeatureExtractor",
+  "feature_size": 1,
+  "padding_side": "right",
+  "padding_value": 0,
+  "return_attention_mask": false,
+  "sampling_rate": 16000
+}

wav2vec2-base-10k-voxpopuli-ft-pl-ONNX/quantize_config.json

@@ -0,0 +1,18 @@
+{
+    "modes": [
+        "fp16",
+        "q8",
+        "int8",
+        "uint8",
+        "q4",
+        "q4f16",
+        "bnb4"
+    ],
+    "per_channel": false,
+    "reduce_range": false,
+    "block_size": null,
+    "is_symmetric": true,
+    "accuracy_level": null,
+    "quant_type": 1,
+    "op_block_list": null
+}

wav2vec2-base-10k-voxpopuli-ft-pl-ONNX/source.txt

@@ -0,0 +1 @@
+https://huggingface.co/onnx-community/wav2vec2-base-10k-voxpopuli-ft-pl-ONNX

wav2vec2-base-10k-voxpopuli-ft-pl-ONNX/special_tokens_map.json

@@ -0,0 +1,6 @@
+{
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "pad_token": "<pad>",
+  "unk_token": "<unk>"
+}

wav2vec2-base-10k-voxpopuli-ft-pl-ONNX/tokenizer.json

@@ -0,0 +1,110 @@
+{
+    "version": "1.0",
+    "truncation": null,
+    "padding": null,
+    "added_tokens": [
+        {
+            "id": 1,
+            "content": "<s>",
+            "single_word": false,
+            "lstrip": true,
+            "rstrip": true,
+            "normalized": false,
+            "special": true
+        },
+        {
+            "id": 0,
+            "content": "<pad>",
+            "single_word": false,
+            "lstrip": true,
+            "rstrip": true,
+            "normalized": false,
+            "special": true
+        },
+        {
+            "id": 2,
+            "content": "</s>",
+            "single_word": false,
+            "lstrip": true,
+            "rstrip": true,
+            "normalized": false,
+            "special": true
+        },
+        {
+            "id": 3,
+            "content": "<unk>",
+            "single_word": false,
+            "lstrip": true,
+            "rstrip": true,
+            "normalized": false,
+            "special": true
+        }
+    ],
+    "normalizer": {
+        "type": "Replace",
+        "pattern": {
+            "String": " "
+        },
+        "content": "|"
+    },
+    "pre_tokenizer": {
+        "type": "Split",
+        "pattern": {
+            "Regex": ""
+        },
+        "behavior": "Isolated",
+        "invert": false
+    },
+    "post_processor": null,
+    "decoder": {
+        "type": "CTC",
+        "pad_token": "<pad>",
+        "word_delimiter_token": "|",
+        "cleanup": true
+    },
+    "model": {
+        "vocab": {
+            "<s>": 1,
+            "<pad>": 0,
+            "</s>": 2,
+            "<unk>": 3,
+            "|": 4,
+            "e": 5,
+            "a": 6,
+            "i": 7,
+            "o": 8,
+            "n": 9,
+            "z": 10,
+            "r": 11,
+            "w": 12,
+            "s": 13,
+            "t": 14,
+            "c": 15,
+            "y": 16,
+            "p": 17,
+            "d": 18,
+            "k": 19,
+            "m": 20,
+            "j": 21,
+            "u": 22,
+            "l": 23,
+            "b": 24,
+            "g": 25,
+            "\u0142": 26,
+            "h": 27,
+            "\u0105": 28,
+            "\u0119": 29,
+            "\u017c": 30,
+            "\u00f3": 31,
+            "\u015b": 32,
+            "\u0107": 33,
+            "f": 34,
+            "\u0144": 35,
+            "\u017a": 36,
+            "v": 37,
+            "x": 38,
+            "q": 39,
+            "1": 40
+        }
+    }
+}

wav2vec2-base-10k-voxpopuli-ft-pl-ONNX/tokenizer_config.json

@@ -0,0 +1,49 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<pad>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": false
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": false
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": false
+    },
+    "3": {
+      "content": "<unk>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "do_lower_case": false,
+  "eos_token": "</s>",
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "processor_class": "Wav2Vec2Processor",
+  "replace_word_delimiter_char": " ",
+  "target_lang": null,
+  "tokenizer_class": "Wav2Vec2CTCTokenizer",
+  "unk_token": "<unk>",
+  "word_delimiter_token": "|"
+}

wav2vec2-base-10k-voxpopuli-ft-pl-ONNX/vocab.json

@@ -0,0 +1,43 @@
+{
+  "1": 40,
+  "</s>": 2,
+  "<pad>": 0,
+  "<s>": 1,
+  "<unk>": 3,
+  "a": 6,
+  "b": 24,
+  "c": 15,
+  "d": 18,
+  "e": 5,
+  "f": 34,
+  "g": 25,
+  "h": 27,
+  "i": 7,
+  "j": 21,
+  "k": 19,
+  "l": 23,
+  "m": 20,
+  "n": 9,
+  "o": 8,
+  "p": 17,
+  "q": 39,
+  "r": 11,
+  "s": 13,
+  "t": 14,
+  "u": 22,
+  "v": 37,
+  "w": 12,
+  "x": 38,
+  "y": 16,
+  "z": 10,
+  "|": 4,
+  "ó": 31,
+  "ą": 28,
+  "ć": 33,
+  "ę": 29,
+  "ł": 26,
+  "ń": 35,
+  "ś": 32,
+  "ź": 36,
+  "ż": 30
+}

wav2vec2-end-of-speech-detection/.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

wav2vec2-end-of-speech-detection/5sec_audio.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6ddddf2ffec49c3bf14f91967b7f66cecebd6c913fdd5e8a610bf45744eb4716
+size 311930

wav2vec2-end-of-speech-detection/README.md

@@ -0,0 +1,119 @@
+---
+license: apache-2.0
+datasets:
+- mozilla-foundation/common_voice_16_0
+language:
+- en
+- de
+- pl
+- fr
+- it
+base_model:
+- facebook/wav2vec2-base-960h
+pipeline_tag: audio-classification
+---
+
+# End of Speech Detection with Wav2Vec 2.0
+
+The End-of-Speech model is based on the open-source Wav2Vec 2.0 model from Meta AI. It uses convolutional feature encoders, which translate chunks of raw audio input into latent speech representations and a transformer to capture the information throughout this sequence of representations. This helps the model distinguish different pitch declines, as well as final lengthening (and the following pause) in the intonation and therefore distinguish when an end of speech event occurs - the same way us humans do.
+
+# Training Data
+
+The training data is constructed from the Common voice 16.0 English Audio dataset by the Mozilla Firefox foundation. It is under a permissive license CC0 1.0.
+
+In order to train the wav2vec 2.0 model for end of speech, we would need a large enough dataset that consists of both end of speech and not end of speech samples. Since there weren’t any open source datasets that contained such ready samples, we needed to construct one. The common voice dataset consists of audio samples that contain only one spoken sentence each. 
+
+Unfortunately, there is additional noisy/empty audio in the beginning and end of the audio samples. To remove those and capture only the audio that corresponds to the spoken sentence, we would need the timestamp of the sentence, or better yet, the word level timestamps. This is achieved with the help of whisperX. This way we capture when the sentence starts and finishes and remove anything before and after.
+
+After cleaning the samples, we ran through random samples to validate the correctness of the procedure. Afterwards we label the last 700/704ms of the audio samples as end of speech events and all before that as not end of speech.
+
+Finally, in addition, we added overlapping segments to the dataset by moving the 700/704ms window in both directions.
+
+# Input
+
+The model is trained at 700 and 704ms (11x64ms) inputs of raw audio. The sample rate is 16kHz. During experiments different lengths have been tested (300ms, 500ms and 1 sec) and 700/704ms proved to be the middle ground between good enough performance and shortest chunk.
+
+# Output
+
+The model classifies each audio input into 2 classes - eos (id: 0) and not_eos (id: 1).
+
+# Usage
+
+```python
+from transformers import Wav2Vec2Processor, AutoConfig
+import onnxruntime as rt
+import torch
+import torch.nn.functional as F
+import numpy as np
+import os
+import torchaudio
+
+
+class EndOfSpeechDetection:
+    processor: Wav2Vec2Processor
+    config: AutoConfig
+    session: rt.InferenceSession
+
+    def load_model(self, path, use_gpu=False):
+        processor = Wav2Vec2Processor.from_pretrained(path)
+        config = AutoConfig.from_pretrained(path)
+
+        sess_options = rt.SessionOptions()
+        sess_options.graph_optimization_level = rt.GraphOptimizationLevel.ORT_ENABLE_ALL
+
+        providers = ["ROCMExecutionProvider"] if use_gpu else ["CPUExecutionProvider"]
+        session = rt.InferenceSession(
+            os.path.join(path, "model.onnx"), sess_options, providers=providers
+        )
+        return processor, config, session
+
+    def predict(self, segment, file_type="pcm"):
+        if file_type == "pcm":
+            # pcm files
+            speech_array = np.memmap(segment, dtype="float32", mode="r").astype(
+                np.float32
+            )
+        else:
+            # wave files
+            speech_array, _ = torchaudio.load(segment)
+            speech_array = speech_array[0].numpy()
+
+        features = self.processor(
+            speech_array, sampling_rate=16000, return_tensors="pt", padding=True
+        )
+        input_values = features.input_values
+        outputs = self.session.run(
+            [self.session.get_outputs()[-1].name],
+            {self.session.get_inputs()[-1].name: input_values.detach().cpu().numpy()},
+        )[0]
+        softmax_output = F.softmax(torch.tensor(outputs), dim=1)
+
+        both_classes_with_prob = {
+            self.config.id2label[i]: softmax_output[0][i].item()
+            for i in range(len(softmax_output[0]))
+        }
+
+        return both_classes_with_prob
+
+
+if __name__ == "__main__":
+    eos = EndOfSpeechDetection()
+    eos.processor, eos.config, eos.session = eos.load_model("eos-model-onnx")
+    print(eos.predict("some.pcm", file_type="pcm"))
+
+```
+
+# Latency (& Memory) Optimization
+- Knowledge Distillation
+- Onnx format weights
+    - The weights are converted in the Onnx format (in order to optimize CPU & GPU Performance)
+    - As tested on an AMD Instinct MI100 GPU - sub 10ms inference per 704ms audio chunk
+
+# Evaluation
+
+Accuracy at 0.95 with 8120 samples tested.
+
+| classes | precision | recall | f1-score | support |
+|---|---|---|---|---|
+| eos | 0.94 | 0.95 | 0.95 | 4060 |
+| not_eos | 0.95 | 0.94 | 0.95 | 4060 |

wav2vec2-end-of-speech-detection/eos-model-onnx/config.json

@@ -0,0 +1,125 @@
+{
+  "_name_or_path": "eos-det/model_07/checkpoint-2283",
+  "activation_dropout": 0.0,
+  "adapter_attn_dim": null,
+  "adapter_kernel_size": 3,
+  "adapter_stride": 2,
+  "add_adapter": false,
+  "apply_spec_augment": true,
+  "architectures": [
+    "Wav2Vec2ForSequenceClassification"
+  ],
+  "attention_dropout": 0.1,
+  "bos_token_id": 1,
+  "classifier_proj_size": 256,
+  "codevector_dim": 256,
+  "contrastive_logits_temperature": 0.1,
+  "conv_bias": false,
+  "conv_dim": [
+    512,
+    512,
+    512,
+    512,
+    512,
+    512,
+    512
+  ],
+  "conv_kernel": [
+    10,
+    3,
+    3,
+    3,
+    3,
+    2,
+    2
+  ],
+  "conv_stride": [
+    5,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2
+  ],
+  "ctc_loss_reduction": "sum",
+  "ctc_zero_infinity": false,
+  "diversity_loss_weight": 0.1,
+  "do_stable_layer_norm": false,
+  "eos_token_id": 2,
+  "feat_extract_activation": "gelu",
+  "feat_extract_norm": "group",
+  "feat_proj_dropout": 0.1,
+  "feat_quantizer_dropout": 0.0,
+  "final_dropout": 0.0,
+  "freeze_feat_extract_train": true,
+  "hidden_act": "gelu",
+  "hidden_dropout": 0.1,
+  "hidden_size": 768,
+  "id2label": {
+    "0": "eos",
+    "1": "not_eos"
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "label2id": {
+    "eos": 0,
+    "not_eos": 1
+  },
+  "layer_norm_eps": 1e-05,
+  "layerdrop": 0.0,
+  "mask_channel_length": 10,
+  "mask_channel_min_space": 1,
+  "mask_channel_other": 0.0,
+  "mask_channel_prob": 0.0,
+  "mask_channel_selection": "static",
+  "mask_feature_length": 10,
+  "mask_feature_min_masks": 0,
+  "mask_feature_prob": 0.0,
+  "mask_time_length": 10,
+  "mask_time_min_masks": 2,
+  "mask_time_min_space": 1,
+  "mask_time_other": 0.0,
+  "mask_time_prob": 0.05,
+  "mask_time_selection": "static",
+  "model_type": "wav2vec2",
+  "no_mask_channel_overlap": false,
+  "no_mask_time_overlap": false,
+  "num_adapter_layers": 3,
+  "num_attention_heads": 12,
+  "num_codevector_groups": 2,
+  "num_codevectors_per_group": 320,
+  "num_conv_pos_embedding_groups": 16,
+  "num_conv_pos_embeddings": 128,
+  "num_feat_extract_layers": 7,
+  "num_hidden_layers": 12,
+  "num_negatives": 100,
+  "output_hidden_size": 768,
+  "pad_token_id": 0,
+  "proj_codevector_dim": 256,
+  "tdnn_dilation": [
+    1,
+    2,
+    3,
+    1,
+    1
+  ],
+  "tdnn_dim": [
+    512,
+    512,
+    512,
+    512,
+    1500
+  ],
+  "tdnn_kernel": [
+    5,
+    3,
+    3,
+    1,
+    1
+  ],
+  "transformers_version": "4.38.2",
+  "use_weighted_layer_sum": false,
+  "vocab_size": 32,
+  "xvector_output_dim": 512
+}

wav2vec2-end-of-speech-detection/eos-model-onnx/model.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d526a75c63ea501292f463f6de28c209fee2ccf733ada042155c01c1c5bc31a9
+size 378578988

wav2vec2-end-of-speech-detection/eos-model-onnx/preprocessor_config.json

@@ -0,0 +1,9 @@
+{
+  "do_normalize": true,
+  "feature_extractor_type": "Wav2Vec2FeatureExtractor",
+  "feature_size": 1,
+  "padding_side": "right",
+  "padding_value": 0.0,
+  "return_attention_mask": false,
+  "sampling_rate": 16000
+}

wav2vec2-end-of-speech-detection/eos-model-onnx/special_tokens_map.json

@@ -0,0 +1 @@
+{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "<unk>", "pad_token": "<pad>"}

wav2vec2-end-of-speech-detection/eos-model-onnx/tokenizer_config.json

@@ -0,0 +1 @@
+{"unk_token": "<unk>", "bos_token": "<s>", "eos_token": "</s>", "pad_token": "<pad>", "do_lower_case": false, "return_attention_mask": false, "do_normalize": true}

wav2vec2-end-of-speech-detection/eos-model-onnx/vocab.json

@@ -0,0 +1 @@
+{"<pad>": 0, "<s>": 1, "</s>": 2, "<unk>": 3, "|": 4, "E": 5, "T": 6, "A": 7, "O": 8, "N": 9, "I": 10, "H": 11, "S": 12, "R": 13, "D": 14, "L": 15, "U": 16, "M": 17, "W": 18, "C": 19, "F": 20, "G": 21, "Y": 22, "P": 23, "B": 24, "V": 25, "K": 26, "'": 27, "X": 28, "J": 29, "Q": 30, "Z": 31}

wav2vec2-end-of-speech-detection/inference.py

@@ -0,0 +1,80 @@
+from transformers import Wav2Vec2Processor, AutoConfig
+import onnxruntime as rt
+import torch
+import torch.nn.functional as F
+import numpy as np
+import os
+import torchaudio
+import soundfile as sf
+
+
+class EndOfSpeechDetection:
+    processor: Wav2Vec2Processor
+    config: AutoConfig
+    session: rt.InferenceSession
+
+    def load_model(self, path, use_gpu=False):
+        processor = Wav2Vec2Processor.from_pretrained(path)
+        config = AutoConfig.from_pretrained(path)
+
+        sess_options = rt.SessionOptions()
+        sess_options.graph_optimization_level = rt.GraphOptimizationLevel.ORT_ENABLE_ALL
+
+        providers = ["ROCMExecutionProvider"] if use_gpu else ["CPUExecutionProvider"]
+        session = rt.InferenceSession(
+            os.path.join(path, "model.onnx"), sess_options, providers=providers
+        )
+        return processor, config, session
+
+    def predict(self, segment, file_type="pcm"):
+        if file_type == "pcm":
+            # pcm files
+            speech_array = np.memmap(segment, dtype="float32", mode="r").astype(
+                np.float32
+            )
+        else:
+            # wave files
+            speech_array, _ = torchaudio.load(segment)
+            speech_array = speech_array[0].numpy()
+
+        features = self.processor(
+            speech_array, sampling_rate=16000, return_tensors="pt", padding=True
+        )
+        input_values = features.input_values
+        outputs = self.session.run(
+            [self.session.get_outputs()[-1].name],
+            {self.session.get_inputs()[-1].name: input_values.detach().cpu().numpy()},
+        )[0]
+        softmax_output = F.softmax(torch.tensor(outputs), dim=1)
+
+        both_classes_with_prob = {
+            self.config.id2label[i]: softmax_output[0][i].item()
+            for i in range(len(softmax_output[0]))
+        }
+
+        return both_classes_with_prob
+
+
+if __name__ == "__main__":
+    eos = EndOfSpeechDetection()
+    eos.processor, eos.config, eos.session = eos.load_model("eos-model-onnx")
+
+    audio_file = "5sec_audio.wav"
+    audio, sr = torchaudio.load(audio_file)
+    audio = audio[0].numpy()
+    audio_len = len(audio)
+    segment_len = 700 * sr // 1000
+    segments = []
+    for i in range(0, audio_len, segment_len):
+        if i + segment_len < audio_len:
+            segment = audio[i : i + segment_len]
+        else:
+            segment = audio[i:]
+
+        segments.append(segment)
+
+    if not os.path.exists("segments"):
+        os.makedirs("segments")
+    for i, segment in enumerate(segments):
+        sf.write(f"segments/segment_{i}.wav", segment, sr)
+        print(eos.predict(f"segments/segment_{i}.wav", file_type="wav"))

wav2vec2-end-of-speech-detection/languages.txt

@@ -0,0 +1,5 @@
+English
+German
+Polish
+French
+Italian

wav2vec2-end-of-speech-detection/source.txt

@@ -0,0 +1 @@
+https://huggingface.co/telnyx/wav2vec2-end-of-speech-detection

wav2vec2-large-xlsr-53-german-cv9/.gitattributes

@@ -0,0 +1,28 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zstandard filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+model.safetensors filter=lfs diff=lfs merge=lfs -text