experiment.zh.json

{
 "<h2>Configurations</h2>\n<p>This inherits from <a href=\"../../experiments/nlp_autoregression.html#NLPAutoRegressionConfigs\"><span translate=no>_^_0_^_</span></a></p>\n": "<h2>Configurations</h2>\n<p>This inherits from <a href=\"../../experiments/nlp_autoregression.html#NLPAutoRegressionConfigs\"><span translate=no>_^_0_^_</span></a></p>\n",
 "<h3>RWKV configurations</h3>\n": "<h3>RWKV configurations</h3>\n",
 "<p> </p>\n": "<p> </p>\n",
 "<p> Create RWKV model and initialize weights</p>\n": "<p> Create RWKV model and initialize weights</p>\n",
 "<p>Apply custom weight initialization </p>\n": "<p>Apply custom weight initialization </p>\n",
 "<p>Batch size <span translate=no>_^_0_^_</span> </p>\n": "<p>Batch size <span translate=no>_^_0_^_</span> </p>\n",
 "<p>Create AdamW optimizer and use the fused version if it is available </p>\n": "<p>Create AdamW optimizer and use the fused version if it is available </p>\n",
 "<p>Create configs </p>\n": "<p>Create configs </p>\n",
 "<p>Create experiment </p>\n": "<p>Create experiment </p>\n",
 "<p>Custom optimizer </p>\n": "<p>Custom optimizer </p>\n",
 "<p>Override configurations </p>\n": "<p>Override configurations </p>\n",
 "<p>Prompt separator is blank </p>\n": "<p>Prompt separator is blank </p>\n",
 "<p>RWKV model </p>\n": "<p>RWKV model </p>\n",
 "<p>Run training </p>\n": "<p>Run training </p>\n",
 "<p>Set models for saving and loading </p>\n": "<p>Set models for saving and loading </p>\n",
 "<p>Set the vocabulary sizes for embeddings and generating logits </p>\n": "<p>Set the vocabulary sizes for embeddings and generating logits </p>\n",
 "<p>Start the experiment </p>\n": "<p>Start the experiment </p>\n",
 "<p>Starting prompt for sampling </p>\n": "<p>Starting prompt for sampling </p>\n",
 "<p>Switch between training and validation for <span translate=no>_^_0_^_</span> times per epoch </p>\n": "<p>Switch between training and validation for <span translate=no>_^_0_^_</span> times per epoch </p>\n",
 "<p>Train for <span translate=no>_^_0_^_</span> epochs </p>\n": "<p>Train for <span translate=no>_^_0_^_</span> epochs </p>\n",
 "<p>Use Tiny Shakespeare dataset </p>\n": "<p>Use Tiny Shakespeare dataset </p>\n",
 "<p>Use a context size of <span translate=no>_^_0_^_</span> </p>\n": "<p>Use a context size of <span translate=no>_^_0_^_</span> </p>\n",
 "<p>Use character level tokenizer </p>\n": "<p>Use character level tokenizer </p>\n",
 "<p>We use our <a href=\"../configs.html#RWKVConfigs\">configurable RWKV implementation</a> </p>\n": "<p>We use our <a href=\"../configs.html#RWKVConfigs\">configurable RWKV implementation</a> </p>\n",
 "<p>create optim groups. Any parameters that is 2D will be weight decayed, otherwise no. i.e. all weight tensors in matmuls + embeddings decay, all biases and layernorms don&#x27;t. </p>\n": "<p>create optim groups. Any parameters that is 2D will be weight decayed, otherwise no. i.e. all weight tensors in matmuls + embeddings decay, all biases and layernorms don&#x27;t. </p>\n",
 "<p>filter out those that do not require grad </p>\n": "<p>filter out those that do not require grad </p>\n",
 "<p>initialize Vector Parameters in TimeMixing </p>\n": "<p>initialize Vector Parameters in TimeMixing </p>\n",
 "<p>model </p>\n": "<p>model </p>\n",
 "<p>number of warmup iterations </p>\n": "<p>number of warmup iterations </p>\n",
 "<p>start with all of the candidate parameters </p>\n": "<p>start with all of the candidate parameters </p>\n",
 "<p>total number of training iterations </p>\n": "<p>total number of training iterations </p>\n",
 "<p>weight decay </p>\n": "<p>weight decay </p>\n",
 "experiment.py": "experiment.py"
}