\uc190\uc2e4\ud568\uc218\uc5d0 \uc5b4\ub5a4 \uc81c\uc57d \uc870\uac74\uc744 \uc801\uc6a9\ud574 \uc624\ubc84\ud53c\ud305\uc744 \ucd5c\uc18c\ud654\ud558\ub294 \ubc29\ubc95\uc73c\ub85c L1 \uc815\ud615\ud654\uc640 L2 \uc815\ud615\ud654\uac00 \uc788\uc2b5\ub2c8\ub2e4. \uc624\ubc84\ud53c\ud305\uc740 \ud2b9\uc815 \uac00\uc911\uce58\uac12\uc774 \ucee4\uc9c8\uc218\ub85d \ubc1c\uc0dd\ud560 \uac00\ub2a5\uc131\uc774 \ub192\uc544\uc9c0\ubbc0\ub85c \uc774\ub97c \ud574\uc18c\ud558\uae30 \uc704\ud574 \ud2b9\uc815\uac12\uc744 \uc190\uc2e4\ud568\uc218\uc5d0 \ub354\ud574\uc8fc\ub294 \uac83\uc774 \uc815\ud615\ud654 \uc911 \uac00\uc911\uce58 \uac10\uc18c(Weight Decay)\uc774\uba70, \ub354\ud574\uc8fc\ub294 \ud2b9\uc815\uac12\uc744 \uacb0\uc815\ud558\ub294 \uac83\uc774 L1 \uc815\ud615\ud654\uc640 L2 \uc815\ud615\ud654\uc785\ub2c8\ub2e4. \ud30c\uc774\ud1a0\uce58\uc5d0\uc11c \uc774 Weight Decay\ub294 \ub2e4\uc74c \ucf54\ub4dc\ucc98\ub7fc \uc801\uc6a9\ud560 \uc218 \uc788\uc2b5\ub2c8\ub2e4.<\/p>\n
\r\noptimizer = torch.optim.SGD(model.parameters(), lr=learning_rate, weight_decay=0.1)\r\n<\/pre>\n\uacb0\uacfc\uc801\uc73c\ub85c weight_decay\uc758 \uac12\uc774 \ucee4\uc9c8\uc218\ub85d \uac00\uc911\uce58 \uac12\uc774 \uc791\uc5b4\uc9c0\uac8c \ub418\uace0, \uc624\ubc84\ud53c\ud305 \ud604\uc0c1\uc744 \ud574\uc18c\ud560 \uc218 \uc788\uc9c0\ub9cc, weight_decay \uac12\uc744 \ub108\ubb34 \ud06c\uac8c \ud558\uba74 \uc5b8\ub354\ud53c\ud305 \ud604\uc0c1\uc774 \ubc1c\uc0dd\ud558\ubbc0\ub85c \uc801\ub2f9\ud55c \uac12\uc744 \uc0ac\uc6a9\ud574\uc57c \ud569\ub2c8\ub2e4.<\/p>\n
2. \ub370\uc774\ud130 \uc99d\uac15(Data Augmentation)<\/h3>\n
\ud6c8\ub828 \ub370\uc774\ud130\ub97c \uc54c\uace0\ub9ac\uc998\uc5d0 \uc758\ud574 \uadf8 \uc218\ub97c \ub298\ub9ac\ub294 \ubc29\ubc95\uc73c\ub85c, \uc624\ubc84\ud53c\ud305 \ud574\uc18c\ub294 \ubb3c\ub860 \ud6c8\ub828 \ubc0f \ud14c\uc2a4\ud2b8 \ub370\uc774\ud130\uc5d0 \ub300\ud55c \uc815\ud655\ub3c4\ub97c \ub192\uc77c \uc218 \uc788\ub294 \ubc29\ubc95\uc785\ub2c8\ub2e4. \ub2e4\uc74c \ucf54\ub4dc\uc758 \uc608\ucc98\ub7fc \ub370\uc774\ud130\uc14b\uc5d0 \ub300\ud55c transform \uc778\uc790\uc5d0 \uc9c0\uc815\ud558\uc5ec \uc801\uc6a9\ud560 \uc218 \uc788\uc2b5\ub2c8\ub2e4.<\/p>\n
\r\nmnist_train = dset.MNIST(\r\n '.\/', \r\n train=True, \r\n transform = transforms.Compose([\r\n transforms.Resize(34), # \uc6d0\ub798 28x28\uc778 \uc774\ubbf8\uc9c0\ub97c 34x34\ub85c \ub298\ub9bd\ub2c8\ub2e4.\r\n transforms.CenterCrop(28), # \uc911\uc559 28x28\ub97c \ubf51\uc544\ub0c5\ub2c8\ub2e4.\r\n transforms.RandomHorizontalFlip(), # \ub79c\ub364\ud558\uac8c \uc88c\uc6b0\ubc18\uc804 \ud569\ub2c8\ub2e4.\r\n transforms.Lambda(lambda x: x.rotate(90)), # \ub78c\ub2e4\ud568\uc218\ub97c \uc774\uc6a9\ud574 90\ub3c4 \ud68c\uc804\ud574\uc90d\ub2c8\ub2e4.\r\n transforms.ToTensor(), # \uc774\ubbf8\uc9c0\ub97c \ud150\uc11c\ub85c \ubcc0\ud615\ud569\ub2c8\ub2e4.\r\n ]),\r\n target_transform=None,\r\n download=True\r\n)\r\n<\/pre>\n3. \uac00\uc911\uce58 \ucd08\uae30\ud654(Weight Initialization)<\/h3>\n
\uc2e0\uacbd\ub9dd\uc774 \uae4a\uc5b4\uc9c8 \uc218\ub85d \uac01 \uc2e0\uacbd\ub9dd\uc758 \uac00\uc911\uce58\uac12\ub4e4\uc758 \ubd84\ud3ec\uac00 \ud55c\ucabd\uc73c\ub85c \uc3e0\ub9ac\uac70\ub098, \ud2b9\uc815 \uac12 \ubd80\ubd84\uc73c\ub85c \uc2ec\ud558\uac8c \ubaa8\uc77c \uc218 \uc788\ub294 \ud604\uc0c1\uc774 \ubc1c\uc0dd\ud569\ub2c8\ub2e4. \uc774\ub7f0 \ud604\uc0c1\uc774 \ubc1c\uc0dd\ud558\uba74 \uae30\uc6b8\uae30 \uc18c\uc2e4(Gradient Vanishing)\uc774 \ubc1c\uc0dd\ud560 \uc218 \uc788\uace0, \uc2e0\uacbd\ub9dd\uc758 \ud45c\ud604\ub825\uc5d0 \uc81c\ud55c\uc774 \uc0dd\uaca8 \uc2e0\uacbd\ub9dd\uc744 \uae4a\uac8c \uad6c\uc131\ud558\ub294 \uc758\ubbf8\uac00 \uc0ac\ub77c\uc9d1\ub2c8\ub2e4. \uc774\ub97c \uc704\ud574 \ud559\uc2b5\ud558\uae30 \uc804\uc5d0 \uac00\uc911\uce58\ub97c \uc801\ub2f9\ud558\uac8c \ucd08\uae30\ud558\ub294 \uac83\uc774 \ud544\uc694\ud569\ub2c8\ub2e4. \uc544\ub798\uc758 \ucf54\ub4dc\ub294 \ud30c\uc774\ud1a0\uce58\uc5d0\uc11c \ud754\ud788\ud558\uac8c \uac00\uc911\uce58\ub97c \ucd08\uae30\ud558\ub294 \ucf54\ub4dc\uc758 \uc608\uc785\ub2c8\ub2e4.<\/p>\n
\r\nclass aDNN(nn.Module):\r\n def __init__(self):\r\n super(aDNN,self).__init__()\r\n self.layer = nn.Sequential(\r\n # ...\r\n ) \r\n \r\n for m in self.modules():\r\n if isinstance(m, nn.Conv2d):\r\n '''\r\n m.weight.data.normal_(0.0, 0.1) # \uac00\uc911\uce58\ub97c \ud3c9\uade0 0, \ud3b8\ucc28 0.1\ub85c \ucd08\uae30\ud654\r\n m.bias.data.fill_(0) # \ud3b8\ucc28\ub97c 0\uc73c\ub85c \ucd08\uae30\ud654\r\n '''\r\n\r\n '''\r\n init.xavier_normal(m.weight.data) # Xavier Initialization\r\n m.bias.data.fill_(0) # \ud3b8\ucc28\ub97c 0\uc73c\ub85c \ucd08\uae30\ud654 \r\n '''\r\n\r\n ''' \r\n init.kaiming_normal_(m.weight.data) # Kaming He Initialization\r\n m.bias.data.fill_(0) # \ud3b8\ucc28\ub97c 0\uc73c\ub85c \ucd08\uae30\ud654 \r\n '''\r\n elif isinstance(m, nn.Linear):\r\n # \uc704\uc758 \ucf54\ub4dc\uc640 \uc720\uc0ac\ud568\r\n<\/pre>\nXavier \ucd08\uae30\ud654\ub294 \ud65c\uc131\ud654\ud568\uc218\uac00 Sigmoid\ub098 Tanh\uc77c\ub54c \uc801\ub2f9\ud558\uba70, Kaming He \ucd08\uae30\ud654\ub294 \ud65c\uc131\ud654 \ud568\uc218\uac00 ReLU\uc77c\ub54c \uc801\ub2f9\ud569\ub2c8\ub2e4.<\/p>\n
4. \ud559\uc2b5\ub960 \uc2a4\ucf00\uc974\ub7ec(Learning Rate Scheduler)<\/h3>\n
\ud559\uc2b5\uc774 \uc9c4\ud589\ub418\uba74\uc11c \ud559\uc2b5\ub960\uc744 \uadf8 \uc0c1\ud669\uc5d0 \ub9de\uac8c \uac00\ubcc0\uc801\uc73c\ub85c \uc801\ub2f9\ud558\uac8c \ubcc0\uacbd\ub420 \uc218 \uc788\ub2e4\uba74 \ub354 \ub0ae\uc740 \uc190\uc2e4\uac12\uc744 \uc5bb\uc744 \uc218 \uc788\uc2b5\ub2c8\ub2e4. \uc774\ub97c \uc704\ud574 \ud559\uc2b5\ub960\uc744 \uc2a4\ucf00\uc974\uc774 \ud544\uc694\ud569\ub2c8\ub2e4. \uc774\uc640 \uad00\ub828\ub41c \ucf54\ub4dc\ub294 \ub2e4\uc74c\uacfc \uac19\uc2b5\ub2c8\ub2e4.<\/p>\n
\r\n# \ud559\uc2b5\ub960 \uc2a4\ucf00\uc904\ub7ec\ub294 \uc635\ud2f0\ub9c8\uc774\uc838\ub97c \ud1b5\ud574 \uc801\uc6a9\ub41c\ub2e4.\r\noptimizer = torch.optim.SGD(model.parameters(), lr=0.01)\r\n\r\n# \uc9c0\uc815\ud55c \uc2a4\ud15d \ub2e8\uc704\ub85c \ud559\uc2b5\ub960\uc5d0 \uac10\ub9c8\ub97c \uacf1\ud574 \ud559\uc2b5\ub960\uc744 \uac10\uc18c\uc2dc\ud0a4\ub294 \ubc29\uc2dd\r\nscheduler = lr_scheduler.StepLR(optimizer, step_size=1, gamma= 0.99) \r\n\r\n# \uc9c0\uc815\ud55c \uc2a4\ud15d \uc9c0\uc810(\uc608\uc2dc\uc5d0\uc11c\ub294 10,20,40)\ub9c8\ub2e4 \ud559\uc2b5\ub960\uc5d0 \uac10\ub9c8\ub97c \uacf1\ud574\uc918\uc11c \uac10\uc18c\uc2dc\ud0a4\ub294 \ubc29\uc2dd\r\nscheduler = lr_scheduler.MultiStepLR(optimizer, milestones=[10,20,40], gamma= 0.1) \r\n\r\n# \ub9e4 epoch\ub9c8\ub2e4 \ud559\uc2b5\ub960\uc5d0 \uac10\ub9c8\ub97c \uacf1\ud574\uc90c\uc73c\ub85c\uc368 \uac10\uc18c\uc2dc\ud0a4\ub294 \ubc29\uc2dd\r\nscheduler = lr_scheduler.ExponentialLR(optimizer, gamma= 0.99) \r\n\r\n# \uc6d0\ud558\ub294 \uc5d0\ud3ed\ub9c8\ub2e4, \uc774\uc804 \ud559\uc2b5\ub960 \ub300\ube44 \ubcc0\uacbd\ud3ed\uc5d0 \ub530\ub77c \ud559\uc2b5\ub960\uc744 \uac10\uc18c\uc2dc\ucf1c\uc8fc\ub294 \ubc29\uc2dd\r\nscheduler = lr_scheduler.ReduceLROnPlateau(optimizer,threshold=1,patience=1,mode='min')\r\n<\/pre>\n\uc704\uc758 \uc2a4\ucf00\uc904\ub7ec\ub97c \ud559\uc2b5\uc5d0 \uc801\uc6a9\ud560\ub54c \uc0ac\uc6a9\ubc29\ubc95\uc740 \ub098\ub258\ub294\ub370, StepLR, MultiStepLR, ExponentialLR\uc758 \uacbd\uc6b0\ub294 \uc544\ub798 \ucf54\ub4dc\ub97c \ucc38\uace0\ud558\uae30 \ubc14\ub78d\ub2c8\ub2e4.<\/p>\n
\r\nfor i in range(num_epoch):\r\n scheduler.step() # !!\r\n\r\n optimizer.zero_grad()\r\n output = model.forward( .. )\r\n loss = loss( .. )\r\n loss.backward()\r\n optimizer.step()\r\n<\/pre>\nReduceLRONPlateau\uc758 \uacbd\uc6b0\ub294 \uc544\ub798 \ucf54\ub4dc\ub97c \ucc38\uace0\ud558\uae30 \ubc14\ub78d\ub2c8\ub2e4.<\/p>\n
\r\nfor i in range(num_epoch):\r\n optimizer.zero_grad()\r\n output = model.forward( .. )\r\n loss = loss( .. )\r\n loss.backward()\r\n optimizer.step()\r\n \r\n scheduler.step(loss) # !! \r\n<\/pre>\n5. \ud559\uc2b5 \ub370\uc774\ud130\uc758 \uc815\uaddc\ud654(Data Normalization)<\/h3>\n
\uc785\ub825\ub418\ub294 \ub370\uc774\ud130\uc5d0 \ub300\ud574\uc11c \uacf5\uac04\uc0c1 \ubd84\ud3ec\ub97c \uc815\uaddc\ud654\uc2dc\ucf1c\uc8fc\uac8c \ub418\uba74, \ub354 \ub192\uc740 \uc815\ud655\ub3c4\ub97c \uc5bb\uc744 \uc218 \uc788\uac8c \ub429\ub2c8\ub2e4. \uc815\uaddc\ud654\uc758 \ubc29\ubc95\uc740 \uc804\uccb4 \ub370\uc774\ud130\uc5d0 \ub300\ud55c \ud3c9\uade0\uacfc \ud45c\uc900\ud3b8\ucc28\ub97c \uc774\uc6a9\ud558\ub294\ub370, \ub370\uc774\ud130\uc5d0 \ub300\ud574\uc11c \ud3c9\uade0\uc744 \ube7c\uace0 \ud45c\uc900\ud3b8\ucc28\ub85c \ub098\ub220\uc90d\ub2c8\ub2e4. \uc774\ub97c \uc704\ud55c \ucf54\ub4dc\uc758 \uc608\ub294 \uc544\ub798\uc640 \uac19\uc2b5\ub2c8\ub2e4.<\/p>\n
\r\nmnist_train = dset.MNIST(\r\n \".\/\", \r\n train=True, \r\n transform=transforms.Compose([\r\n transforms.ToTensor(),\r\n transforms.Normalize(mean=(0.1307,), std=(0.3081,))\r\n ]),\r\n download=True\r\n)\r\n\r\nmnist_test = dset.MNIST(\r\n \".\/\", \r\n train=False, \r\n transform=transforms.Compose([\r\n transforms.ToTensor(),\r\n transforms.Normalize(mean=(0.1307,), std=(0.3081,))\r\n ]),\r\n download=True)\r\n<\/pre>\n\ub370\uc774\ud130\uc5d0 \ub300\ud55c \ud3c9\uade0 \ubc0f \ud45c\uc900\ud3b8\ucc28\ub97c \uc5bb\ub294 \uac83\uc740 \uc544\ub798\uc758 \uae00\uc744 \ucc38\uace0\ud558\uae30 \ubc14\ub78d\ub2c8\ub2e4.<\/p>\n