From 292282867dde49d6d2460312e9a078aec1cb0851 Mon Sep 17 00:00:00 2001
From: anton-l <aglozhkov@gmail.com>
Date: Tue, 17 Aug 2021 16:11:39 +0300
Subject: [PATCH] Tokenizer-less model

---
 .gitignore               | 3 +++
 preprocessor_config.json | 2 +-
 special_tokens_map.json  | 1 -
 tokenizer_config.json    | 1 -
 vocab.json               | 1 -
 5 files changed, 4 insertions(+), 4 deletions(-)
 create mode 100644 .gitignore
 delete mode 100644 special_tokens_map.json
 delete mode 100644 tokenizer_config.json
 delete mode 100644 vocab.json

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..d0c418c
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,3 @@
+special_tokens_map.json
+tokenizer_config.json
+vocab.json
diff --git a/preprocessor_config.json b/preprocessor_config.json
index 8df8da1..a0b7227 100644
--- a/preprocessor_config.json
+++ b/preprocessor_config.json
@@ -3,7 +3,7 @@
   "feature_extractor_type": "Wav2Vec2FeatureExtractor",
   "feature_size": 1,
   "padding_side": "right",
-  "padding_value": 0,
+  "padding_value": 0.0,
   "return_attention_mask": false,
   "sampling_rate": 16000
 }
diff --git a/special_tokens_map.json b/special_tokens_map.json
deleted file mode 100644
index 25bc396..0000000
--- a/special_tokens_map.json
+++ /dev/null
@@ -1 +0,0 @@
-{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "<unk>", "pad_token": "<pad>"}
\ No newline at end of file
diff --git a/tokenizer_config.json b/tokenizer_config.json
deleted file mode 100644
index 928e2ed..0000000
--- a/tokenizer_config.json
+++ /dev/null
@@ -1 +0,0 @@
-{"unk_token": "<unk>", "bos_token": "<s>", "eos_token": "</s>", "pad_token": "<pad>", "do_lower_case": false, "word_delimiter_token": "|", "return_attention_mask": false, "do_normalize": true, "model_max_length": 9223372036854775807, "special_tokens_map_file": "/home/crocopie/.cache/huggingface/transformers/208086b2429fa2ba5b196810c1bcd7d61e2c8d4afd65d05d0670096d735fd5bb.9d6cd81ef646692fb1c169a880161ea1cb95f49694f220aced9b704b457e51dd", "tokenizer_file": null, "name_or_path": "facebook/wav2vec2-base-960h", "tokenizer_class": "Wav2Vec2CTCTokenizer"}
\ No newline at end of file
diff --git a/vocab.json b/vocab.json
deleted file mode 100644
index 88181b9..0000000
--- a/vocab.json
+++ /dev/null
@@ -1 +0,0 @@
-{"<pad>": 0, "<s>": 1, "</s>": 2, "<unk>": 3, "|": 4, "E": 5, "T": 6, "A": 7, "O": 8, "N": 9, "I": 10, "H": 11, "S": 12, "R": 13, "D": 14, "L": 15, "U": 16, "M": 17, "W": 18, "C": 19, "F": 20, "G": 21, "Y": 22, "P": 23, "B": 24, "V": 25, "K": 26, "'": 27, "X": 28, "J": 29, "Q": 30, "Z": 31}
\ No newline at end of file