{"id":4401,"date":"2023-05-27T15:16:35","date_gmt":"2023-05-27T07:16:35","guid":{"rendered":"http:\/\/www.sniper97.cn\/?p=4401"},"modified":"2023-05-27T15:17:14","modified_gmt":"2023-05-27T07:17:14","slug":"%e3%80%90%e6%b7%b1%e5%ba%a6%e5%ad%a6%e4%b9%a0%e7%ac%94%e8%ae%b0%e3%80%91pre-norm%e3%80%81post-norm%e4%b8%8edeepnorm","status":"publish","type":"post","link":"http:\/\/www.sniper97.cn\/index.php\/note\/deep-learning\/note-deep-learning\/4401\/","title":{"rendered":"\u3010\u6df1\u5ea6\u5b66\u4e60\u7b14\u8bb0\u3011Pre-Norm\u3001Post-Norm\u4e0eDeepNorm"},"content":{"rendered":"\n<p>\u4f20\u7edf\u7684Bert\u91c7\u7528\u4e86Post-Norm\u7684\u65b9\u6cd5\uff0c\u800c\u4f8b\u5982\u6700\u8fd1\u7684Bloom\u3001LLama\u90fd\u4f7f\u7528\u4e86Pre-Norm\u7684\u65b9\u6cd5\u3002\u90a3\u4e48\u8fd9\u4e24\u4e2a\u6709\u4ec0\u4e48\u533a\u522b\u5462\uff1f<\/p>\n\n\n\n<p>\u9996\u5148\u67e5\u770bBert\u7684Post-Norm\uff0c\u662f\u5728Add\u64cd\u4f5c\u540e\u8fdb\u884cNorm\u64cd\u4f5c\uff0c\u56e0\u6b64\u53eb\u505aPost-Norm\u3002<\/p>\n\n\n<div class=\"wp-block-image\">\n<figure class=\"aligncenter size-full is-resized\"><img loading=\"lazy\" decoding=\"async\" src=\"http:\/\/www.sniper97.cn\/wp-content\/uploads\/2023\/05\/image-30.png\" alt=\"\" class=\"wp-image-4403\" width=\"263\" height=\"311\" srcset=\"http:\/\/www.sniper97.cn\/wp-content\/uploads\/2023\/05\/image-30.png 656w, http:\/\/www.sniper97.cn\/wp-content\/uploads\/2023\/05\/image-30-254x300.png 254w\" sizes=\"(max-width: 263px) 100vw, 263px\" \/><\/figure><\/div>\n\n\n<p>\u800cPre-Norm\u5219\u662fNorm\u4e4b\u540e\u518dAdd\uff0c\u6240\u4ee5\u53ebPre-Norm\u3002<\/p>\n\n\n<div class=\"wp-block-image\">\n<figure class=\"aligncenter size-large is-resized\"><img loading=\"lazy\" decoding=\"async\" src=\"http:\/\/www.sniper97.cn\/wp-content\/uploads\/2023\/05\/image-29-1024x531.png\" alt=\"\" class=\"wp-image-4402\" width=\"451\" height=\"234\" srcset=\"http:\/\/www.sniper97.cn\/wp-content\/uploads\/2023\/05\/image-29-1024x531.png 1024w, http:\/\/www.sniper97.cn\/wp-content\/uploads\/2023\/05\/image-29-300x156.png 300w, http:\/\/www.sniper97.cn\/wp-content\/uploads\/2023\/05\/image-29-768x398.png 768w, http:\/\/www.sniper97.cn\/wp-content\/uploads\/2023\/05\/image-29.png 1254w\" sizes=\"(max-width: 451px) 100vw, 451px\" \/><\/figure><\/div>\n\n\n<h2 class=\"wp-block-heading\">Pre-Norm\u548cPost-Norm\u6709\u4ec0\u4e48\u533a\u522b\uff1f<\/h2>\n\n\n\n<p>\u7b80\u5355\u8bf4\u5c31\u662fPost-Norm\u7531\u4e8e\u662f\u5728\u6b8b\u5dee\u4e4b\u540e\u8fdb\u884c\u5f52\u4e00\u5316\uff0c\u56e0\u6b64\u5f52\u4e00\u5316\u7684\u6548\u679c\u66f4\u597d\uff0c\u4f7f\u5f97\u6a21\u578b\u7684<strong>\u9c81\u68d2\u6027\u66f4\u5f3a<\/strong>\u3002<\/p>\n\n\n\n<p>\u800cPre-Norm\u7531\u4e8e\u5e76\u4e0d\u662f\u6240\u6709\u7684\u53c2\u6570\u90fd\u53c2\u4e0e\u6b63\u5219\u5316\uff0c\u56e0\u6b64\u6574\u4f53\u6765\u8bf4\u66f4<strong>\u4e0d\u5bb9\u6613\u53d1\u751f\u68af\u5ea6\u6d88\u5931<\/strong>\u7684\u95ee\u9898\uff0c\u6a21\u578b\u8bad\u7ec3\u7684\u7a33\u5b9a\u6027\u66f4\u5f3a\u3002<\/p>\n\n\n\n<p>\u56e0\u6b64\uff0c\u5728Bert\u65f6\u4ee3\u7531\u4e8e\u5c42\u6570\u8f83\u6d45\uff0c\u5f80\u5f80\u91c7\u7528\u7684\u662fPost-Norm\uff0c\u800c\u5230\u4e86\u5927\u6a21\u578b\u65f6\u4ee3\uff0c\u7531\u4e8etransformer\u7684\u5c42\u6570\u5f00\u59cb\u52a0\u6df1\uff0c\u4e3a\u4e86\u8bad\u7ec3\u7a33\u5b9a\u6027\u5f00\u59cb\u4f7f\u7528Pre-Norm\u3002<\/p>\n\n\n\n<p>\u90a3\u4e48\u6709\u6ca1\u6709\u65e2\u8981\u53c8\u8981\u7684\u64cd\u4f5c\u5462\uff1f<\/p>\n\n\n\n<h2 class=\"wp-block-heading\">DeepNorm<\/h2>\n\n\n\n<p>\u5fae\u8f6f\u7684\u4e00\u4e2a\u5b9e\u4e60\u751f\u63d0\u51fa\u6765\u4e86\u4e00\u4e2aDeepNorm\uff0c\u5b83\u53ef\u4ee5\u7a33\u5b9a\u7684\u8bad\u7ec3\u5343\u5c42\u7684transformer\u3002\u800cGLM\u4e5f\u662f\u4f7f\u7528\u4e86\u8fd9\u79cdNorm\uff0c\u4f7f\u5f97\u767e\u4ebf\u6a21\u578b\u5728\u6574\u4e2a\u8bad\u7ec3\u8fc7\u7a0b\u4e2d\u53ea\u51fa\u73b0\u4e863\u6b21loss\u5f02\u5e38\uff08\u5f53\u7136\u4e5f\u6709\u522b\u7684\u63aa\u65bd\uff09\u3002<\/p>\n\n\n\n<p>\u8be6\u89c1<a href=\"http:\/\/www.sniper97.cn\/index.php\/uncategorized\/4391\/\" title=\"\u3010\u8bba\u6587\u3011DeepNet: Scaling Transformers to 1,000 Layers\">\u3010\u8bba\u6587\u3011DeepNet: Scaling Transformers to 1,000 Layers<\/a><\/p>\n","protected":false},"excerpt":{"rendered":"<p>\u4f20\u7edf\u7684Bert\u91c7\u7528\u4e86Post-Norm\u7684\u65b9\u6cd5\uff0c\u800c\u4f8b\u5982\u6700\u8fd1\u7684Bloom\u3001LLama\u90fd\u4f7f\u7528\u4e86Pre-No [&hellip;]<\/p>\n","protected":false},"author":2,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"om_disable_all_campaigns":false,"_mi_skip_tracking":false,"_monsterinsights_sitenote_active":false,"_monsterinsights_sitenote_note":"","_monsterinsights_sitenote_category":0,"footnotes":""},"categories":[9],"tags":[],"views":10101,"_links":{"self":[{"href":"http:\/\/www.sniper97.cn\/index.php\/wp-json\/wp\/v2\/posts\/4401"}],"collection":[{"href":"http:\/\/www.sniper97.cn\/index.php\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"http:\/\/www.sniper97.cn\/index.php\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"http:\/\/www.sniper97.cn\/index.php\/wp-json\/wp\/v2\/users\/2"}],"replies":[{"embeddable":true,"href":"http:\/\/www.sniper97.cn\/index.php\/wp-json\/wp\/v2\/comments?post=4401"}],"version-history":[{"count":2,"href":"http:\/\/www.sniper97.cn\/index.php\/wp-json\/wp\/v2\/posts\/4401\/revisions"}],"predecessor-version":[{"id":4405,"href":"http:\/\/www.sniper97.cn\/index.php\/wp-json\/wp\/v2\/posts\/4401\/revisions\/4405"}],"wp:attachment":[{"href":"http:\/\/www.sniper97.cn\/index.php\/wp-json\/wp\/v2\/media?parent=4401"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"http:\/\/www.sniper97.cn\/index.php\/wp-json\/wp\/v2\/categories?post=4401"},{"taxonomy":"post_tag","embeddable":true,"href":"http:\/\/www.sniper97.cn\/index.php\/wp-json\/wp\/v2\/tags?post=4401"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}