tokenizer.ja.json

{
 "<h1>GPT-NeoX Tokenizer</h1>\n<p>This initializes a Hugging Face tokenizer from the downloaded vocabulary.</p>\n": "<h1>GPT-\u30cd\u30aa\u30c3\u30af\u30b9\u30c8\u30fc\u30af\u30ca\u30a4\u30b6\u30fc</h1>\n<p>\u3053\u308c\u306b\u3088\u308a\u3001\u30c0\u30a6\u30f3\u30ed\u30fc\u30c9\u3057\u305f\u30dc\u30ad\u30e3\u30d6\u30e9\u30ea\u30fc\u304b\u3089 Hugging Face \u30c8\u30fc\u30af\u30ca\u30a4\u30b6\u30fc\u304c\u521d\u671f\u5316\u3055\u308c\u307e\u3059\u3002</p>\n",
 "<h3>Load NeoX Tokenizer</h3>\n<ul><p><em>Returns</em>  the tokenizer</p></ul>\n": "<h3>NeoX \u30c8\u30fc\u30af\u30ca\u30a4\u30b6\u30fc\u3092\u30ed\u30fc\u30c9</h3>\n<ul><p><em>\u30c8\u30fc\u30af\u30ca\u30a4\u30b6\u30fc\u3092\u8fd4\u3057\u307e\u3059</em></p></ul>\n",
 "GPT-NeoX Tokenizer": "GPT-\u30cd\u30aa\u30c3\u30af\u30b9\u30c8\u30fc\u30af\u30ca\u30a4\u30b6\u30fc",
 "Loads the GPT-NeoX tokenizer": "GPT-Neox \u30c8\u30fc\u30af\u30ca\u30a4\u30b6\u30fc\u3092\u30ed\u30fc\u30c9\u3057\u307e\u3059"
}