{"id":3897,"date":"2021-06-12T17:18:35","date_gmt":"2021-06-12T09:18:35","guid":{"rendered":"http:\/\/www.sniper97.cn\/?p=3897"},"modified":"2021-06-12T17:18:35","modified_gmt":"2021-06-12T09:18:35","slug":"%e3%80%90%e6%b7%b1%e5%ba%a6%e5%ad%a6%e4%b9%a0%e7%ac%94%e8%ae%b0%e3%80%91%e6%bf%80%e6%b4%bb%e5%87%bd%e6%95%b0%e9%9b%86%e5%90%88","status":"publish","type":"post","link":"http:\/\/www.sniper97.cn\/index.php\/note\/deep-learning\/base\/3897\/","title":{"rendered":"\u3010\u6df1\u5ea6\u5b66\u4e60\u7b14\u8bb0\u3011\u6fc0\u6d3b\u51fd\u6570\u96c6\u5408"},"content":{"rendered":"\n<p>\u672c\u7bc7\u5199\u4e00\u4e9b\u5bf9\u6fc0\u6d3b\u51fd\u6570\u7684\u4e00\u4e9b\u7406\u89e3\u3002\u5982\u679c\u6709\u9519\u8bef\u6b22\u8fce\u5404\u4f4d\u5927\u4f6c\u6307\u51fa\u3002<\/p>\n\n\n<p>\u9996\u5148\u662f\u635f\u5931\u51fd\u6570\u7684\u4e00\u822c\u8981\u6c42\uff1a<\/p>\n\n\n<ul><li><strong>\u975e\u7ebf\u6027<\/strong>\u3002\u975e\u7ebf\u6027\u4fdd\u8bc1\u4e86\u7f51\u7edc\u7684\u6df1\u5ea6\uff1b\u4f7f\u7528\u7ebf\u6027\u6fc0\u6d3b\u51fd\u6570\u4f1a\u8ba9\u591a\u5c42\u7f51\u7edc\u574d\u584c\u6210\u4e00\u5c42\u3002<\/li><li><strong>\uff08\u51e0\u4e4e\uff09\u5904\u5904\u53ef\u5fae<\/strong>\u3002\u53ef\u5fae\u6027\u4fdd\u8bc1\u4e86\u5728\u4f18\u5316\u4e2d\u7684\u68af\u5ea6\u53ef\u8ba1\u7b97\u6027\u3002<\/li><li><strong>\u8ba1\u7b97\u7b80\u5355<\/strong>\u3002\u8fd9\u4e5f\u5c31\u662f\u4e3a\u4ec0\u4e48Relu\u6bd4exp\u7c7b\u6fc0\u6d3b\u51fd\u6570\u8981\u66f4\u53d7\u6b22\u8fce\u3002<\/li><li><strong>\u975e\u9971\u548c\u6027<\/strong>\u3002\u9971\u548c\u6307\u7684\u662f\u5728\u67d0\u4e9b\u533a\u95f4\u68af\u5ea6\u63a5\u8fd10\uff0c\u4ece\u800c\u5bfc\u81f4\u68af\u5ea6\u6d88\u5931\u3002\u6700\u5178\u578b\u7684\u4f8b\u5b50\u662fsigmoid\u3002<\/li><\/ul>\n\n\n<h2 class=\"wp-block-heading\">sigmoid<\/h2>\n\n\n<p>\u56fe\u50cf\u5982\u4e0b\uff1a<\/p>\n\n\n<figure class=\"wp-block-image\"><img decoding=\"async\" src=\"https:\/\/pic2.zhimg.com\/80\/v2-8d9c99a123ba8cb2629106660e8bf6d5_1440w.jpg\" alt=\"\"\/><\/figure>\n\n\n<p>\u516c\u5f0f\u5982\u4e0b\uff1a<\/p>\n\n\n<div class=\"wp-block-image\"><figure class=\"aligncenter\"><img decoding=\"async\" src=\"https:\/\/pic4.zhimg.com\/80\/v2-e92c44a96313ffe1cdd4594c52466d97_1440w.jpg\" alt=\"\"\/><\/figure><\/div>\n\n\n<p>sigmoid\u975e\u5e38\u9002\u5408\u5c06\u6982\u7387\u4f5c\u4e3a\u8f93\u51fa\u7684\u6a21\u578b\uff0c\u56e0\u4e3a\u6982\u7387\u7684\u53d6\u503c\u8303\u56f4\u662f0\uff5e1\u3002<\/p>\n\n\n<p><strong>\u4f18\u70b9\uff1a<\/strong><\/p>\n\n\n<ul><li>\u68af\u5ea6\u5e73\u6ed1\uff0c\u907f\u514d\u300c\u8df3\u8dc3\u300d\u7684\u8f93\u51fa\u503c\uff1b<\/li><li>\u660e\u786e\u7684\u9884\u6d4b\uff0c\u5373\u975e\u5e38\u63a5\u8fd1 1 \u6216 0\u3002<\/li><\/ul>\n\n\n<p><strong>\u7f3a\u70b9\uff1a<\/strong><\/p>\n\n\n<ul><li>\u503e\u5411\u4e8e\u68af\u5ea6\u6d88\u5931\uff08\u7531\u4e8e\u4e24\u4fa7\u7684\u9971\u548c\u533a\u57df\uff09<\/li><li>\u6307\u6570\u8fd0\u7b97\u7684\u8ba1\u7b97\u4ee3\u4ef7\u8f83\u5927<\/li><li>\u8f93\u51fa\u4e0d\u4ee50\u4e3a\u4e2d\u5fc3\uff0c\u964d\u4f4e\u4e86\u6743\u91cd\u66f4\u65b0\u7684\u6548\u7387\u3002<\/li><\/ul>\n\n\n<h2 class=\"wp-block-heading\">tanh<\/h2>\n\n\n<p>\u56fe\u50cf\u4e3a\uff1a<\/p>\n\n\n<div class=\"wp-block-image\"><figure class=\"aligncenter\"><img decoding=\"async\" src=\"https:\/\/pic1.zhimg.com\/80\/v2-3b8753a9956a673554d5e992c398f334_1440w.jpg\" alt=\"\"\/><\/figure><\/div>\n\n\n<p>\u516c\u5f0f\u4e3a\uff1a<\/p>\n\n\n<div class=\"wp-block-image\"><figure class=\"aligncenter\"><img decoding=\"async\" src=\"https:\/\/pic4.zhimg.com\/80\/v2-e74cf9339fba4fb7fdf2003fd8ea737f_1440w.jpg\" alt=\"\"\/><\/figure><\/div>\n\n\n<p>tanh \u662f\u4e00\u4e2a\u53cc\u66f2\u6b63\u5207\u51fd\u6570\u3002tanh \u51fd\u6570\u548c sigmoid \u51fd\u6570\u7684\u66f2\u7ebf\u76f8\u5bf9\u76f8\u4f3c\u3002\u4f46\u662f\u5b83\u6bd4 sigmoid \u51fd\u6570\u66f4\u6709\u4e00\u4e9b\u4f18\u52bf\u3002<\/p>\n\n\n<p><strong>\u4f18\u70b9\uff1a<\/strong><\/p>\n\n\n<ul><li>\u68af\u5ea6\u5e73\u6ed1\uff0c\u907f\u514d\u300c\u8df3\u8dc3\u300d\u7684\u8f93\u51fa\u503c\uff1b<\/li><li>\u8f93\u51fa\u4ee50\u4e3a\u4e2d\u5fc3\u3002<\/li><\/ul>\n\n\n<p><strong>\u7f3a\u70b9\uff1a<\/strong><\/p>\n\n\n<ul><li>\u503e\u5411\u4e8e\u68af\u5ea6\u6d88\u5931\uff08\u7531\u4e8e\u4e24\u4fa7\u7684\u9971\u548c\u533a\u57df\uff09<\/li><li>\u6307\u6570\u8fd0\u7b97\u7684\u8ba1\u7b97\u4ee3\u4ef7\u8f83\u5927<\/li><\/ul>\n\n\n<h2 class=\"wp-block-heading\">ReLU<\/h2>\n\n\n<p>\u56fe\u50cf\u4e3a\uff1a<\/p>\n\n\n<div class=\"wp-block-image\"><figure class=\"aligncenter\"><img decoding=\"async\" src=\"https:\/\/pic1.zhimg.com\/80\/v2-30c8465016babfc2e6440c43aee81528_1440w.jpg\" alt=\"\"\/><\/figure><\/div>\n\n\n<p>\u516c\u5f0f\u4e3a\uff1a<\/p>\n\n\n<div class=\"wp-block-image\"><figure class=\"aligncenter\"><img decoding=\"async\" src=\"https:\/\/pic1.zhimg.com\/80\/v2-0cbe7272f0bd246052f99e136e422220_1440w.jpg\" alt=\"\"\/><\/figure><\/div>\n\n\n<p>ReLU\u76ee\u524d\u5728\u6df1\u5ea6\u5b66\u4e60\u4e2d\u6bd4\u8f83\u6d41\u884c\uff0c\u76f8\u6bd4sigmoid\u548ctanh\u6709\u5f88\u591a\u4f18\u70b9\uff0c\u4f46\u662f\u4e5f\u6709\u7f3a\u70b9\u3002\uff08PS\uff1a\u5173\u4e8eReLU\u66f4\u591a\u7684\u601d\u8003\u8be6\u89c1<a href=\"http:\/\/www.sniper97.cn\/index.php\/note\/deep-learning\/base\/3900\/\">ReLU\u7684\u601d\u8003<\/a>\uff09<\/p>\n\n\n<p><strong>\u4f18\u70b9\uff1a<\/strong><\/p>\n\n\n<ul><li>\u8ba1\u7b97\u7b80\u5355\uff0c\u6ca1\u6709\u590d\u6742\u7684\u6307\u6570\u8fd0\u7b97<\/li><li>\u5f53\u8f93\u5165\u4e3a\u6b63\u65f6\uff0c\u4e0d\u5b58\u5728\u68af\u5ea6\u9971\u548c<\/li><\/ul>\n\n\n<p><strong>\u7f3a\u70b9\uff1a<\/strong><\/p>\n\n\n<ul><li>ReLU\u4e0d\u662f\u4ee50\u4e3a\u4e2d\u5fc3\u7684<\/li><li>Dead ReLU\u95ee\u9898\u3002\u53cd\u5411\u4f20\u64ad\u8fc7\u7a0b\u4e2d\u53ef\u80fd\u4f1a\u4ea7\u751f0\u68af\u5ea6\uff08\u6b63\u5411\u4e2d\u76840\u53ef\u4ee5\u7406\u89e3\uff0c\u6bd5\u7adf\u6709\u4e9b\u5730\u65b9\u654f\u611f\u6709\u4e9b\u5730\u65b9\u4e0d\u654f\u611f\uff0c\u4f46\u662f\u53cd\u5411\u4f20\u64ad\u5982\u679c\u8f93\u5165\u8d1f\u6570\u5219\u4f1a\u4ea7\u751f0\u68af\u5ea6\uff09<\/li><\/ul>\n\n\n<h2 class=\"wp-block-heading\">Leaky ReLU<\/h2>\n\n\n<p>\u4e3a\u4e86\u89e3\u51b3Dead ReLU\u95ee\u9898\uff0c\u63d0\u51fa\u4e86Leaky ReLU<\/p>\n\n\n<div class=\"wp-block-image\"><figure class=\"aligncenter\"><img decoding=\"async\" src=\"https:\/\/pic2.zhimg.com\/80\/v2-4d592b88fe164d0ca1fdd42a79f8b4a1_1440w.jpg\" alt=\"\"\/><\/figure><\/div>\n\n\n<div class=\"wp-block-image\"><figure class=\"aligncenter\"><img decoding=\"async\" src=\"https:\/\/pic1.zhimg.com\/80\/v2-9069d07c1aaf672f91678502e0aab92c_1440w.jpg\" alt=\"\"\/><\/figure><\/div>\n\n\n<p>Leaky ReLU\u901a\u8fc7\u7ed9\u8d1f\u533a\u95f4\u4e00\u4e2a\u5f88\u5c0f\u7684\u659c\u7387\uff080.01\uff09\u6765\u89e3\u51b3Dead ReLU\u95ee\u9898\u3002<\/p>\n\n\n<p>\u7406\u8bba\u4e0a\u8bb2\uff0cLeaky ReLU\u62e5\u6709ReLU\u6240\u6709\u7684\u4f18\u70b9\uff0c\u540c\u65f6\u53c8\u4e0d\u4f1a\u6709ReLU\u7684\u95ee\u9898\u3002\u4f46\u662f\u5c1a\u672a\u8bc1\u660eLeaky ReLU<strong>\u603b<\/strong>\u6bd4ReLU\u597d\u3002<\/p>\n\n\n<h2 class=\"wp-block-heading\">GeLU<\/h2>\n\n\n<p>\u516c\u5f0f\u4e3a\uff1a<\/p>\n\n\n<figure class=\"wp-block-image\"><img decoding=\"async\" src=\"https:\/\/pic4.zhimg.com\/80\/v2-c17c66cbb168f7e16cecdb5f16c2023f_1440w.png\" alt=\"\"\/><\/figure>\n\n\n<p>\u8fd9\u4e2a\u662f\u65e0\u6cd5\u8ba1\u7b97\u7684\uff0c\u901a\u8fc7\u6570\u5b66\u516c\u5f0f\u903c\u8fd1\u4e3a\uff1a<\/p>\n\n\n<div class=\"wp-block-image\"><figure class=\"aligncenter\"><img decoding=\"async\" src=\"https:\/\/pic4.zhimg.com\/80\/v2-caceaa950884848a3b772340dbace50f_1440w.jpg\" alt=\"\"\/><\/figure><\/div>\n\n\n<p>\u4e5f\u53ef\u4ee5\u8bf4\u662fx*sigmoid(x)<\/p>\n\n\n<p>\u56fe\u50cf\u4e3a\uff1a<\/p>\n\n\n<div class=\"wp-block-image\"><figure class=\"aligncenter\"><img decoding=\"async\" src=\"https:\/\/pic3.zhimg.com\/80\/v2-2d9cd68d5d839ce613c1d044d02a4a2e_1440w.jpg\" alt=\"\"\/><\/figure><\/div>\n\n\n<p>\u76ee\u524dBerts\uff0cGPT-2\u7b49\u6a21\u578b\u4e2d\u5747\u4f7f\u7528\u4e86GELU\u6fc0\u6d3b\u51fd\u6570\u3002<\/p>\n\n\n<hr class=\"wp-block-separator\"\/>\n\n\n<p>\u6700\u540e\uff0c\u8c37\u6b4c\u5927\u8111\u8fd8\u5c1d\u8bd5\u4e86\u5f88\u591a\u7b80\u5355\u51fd\u6570\u7ec4\u5408\u6765\u4f5c\u4e3a\u6fc0\u6d3b\u51fd\u6570\uff08<a rel=\"noreferrer noopener\" aria-label=\"arXiv:1710.05941\uff08\u5728\u65b0\u7a97\u53e3\u6253\u5f00\uff09\" href=\"https:\/\/arxiv.org\/abs\/1710.05941\" target=\"_blank\">arXiv:1710.05941<\/a>\uff09\uff0c\u6709\u5174\u8da3\u7684\u540c\u5b66\u53ef\u4ee5\u53bb\u8bfb\u4e00\u8bfb\u3002<\/p>\n","protected":false},"excerpt":{"rendered":"<p>\u672c\u7bc7\u5199\u4e00\u4e9b\u5bf9\u6fc0\u6d3b\u51fd\u6570\u7684\u4e00\u4e9b\u7406\u89e3\u3002\u5982\u679c\u6709\u9519\u8bef\u6b22\u8fce\u5404\u4f4d\u5927\u4f6c\u6307\u51fa\u3002 \u9996\u5148\u662f\u635f\u5931\u51fd\u6570\u7684\u4e00\u822c\u8981\u6c42\uff1a \u975e\u7ebf\u6027\u3002\u975e [&hellip;]<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"om_disable_all_campaigns":false,"_mi_skip_tracking":false,"_monsterinsights_sitenote_active":false,"_monsterinsights_sitenote_note":"","_monsterinsights_sitenote_category":0,"footnotes":""},"categories":[8,9],"tags":[33,37,39],"views":4374,"_links":{"self":[{"href":"http:\/\/www.sniper97.cn\/index.php\/wp-json\/wp\/v2\/posts\/3897"}],"collection":[{"href":"http:\/\/www.sniper97.cn\/index.php\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"http:\/\/www.sniper97.cn\/index.php\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"http:\/\/www.sniper97.cn\/index.php\/wp-json\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"http:\/\/www.sniper97.cn\/index.php\/wp-json\/wp\/v2\/comments?post=3897"}],"version-history":[{"count":0,"href":"http:\/\/www.sniper97.cn\/index.php\/wp-json\/wp\/v2\/posts\/3897\/revisions"}],"wp:attachment":[{"href":"http:\/\/www.sniper97.cn\/index.php\/wp-json\/wp\/v2\/media?parent=3897"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"http:\/\/www.sniper97.cn\/index.php\/wp-json\/wp\/v2\/categories?post=3897"},{"taxonomy":"post_tag","embeddable":true,"href":"http:\/\/www.sniper97.cn\/index.php\/wp-json\/wp\/v2\/tags?post=3897"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}