Você não pode selecionar mais de 25 tópicos
Os tópicos devem começar com uma letra ou um número, podem incluir traços ('-') e podem ter até 35 caracteres.
1 linhas
16 KiB
1 linhas
16 KiB
{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"learning.ipynb","provenance":[],"authorship_tag":"ABX9TyOH0O9m/wI7scpeov7qGokX"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"}},"cells":[{"cell_type":"code","execution_count":43,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"cMHw9PfA4PdG","executionInfo":{"status":"ok","timestamp":1651650933526,"user_tz":-480,"elapsed":3253,"user":{"displayName":"ming li","userId":"14148720490428311514"}},"outputId":"c546da43-6e70-4e0e-f90d-c167e3bc5cfc"},"outputs":[{"output_type":"stream","name":"stdout","text":["Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"]}],"source":["from google.colab import drive\n","drive.mount('/content/drive')\n","import os\n","import torch\n","import random"]},{"cell_type":"code","source":["path = '/content/drive/My Drive/social_computing/data/count.txt'"],"metadata":{"id":"in4a4Lmr4R5h","executionInfo":{"status":"ok","timestamp":1651650136175,"user_tz":-480,"elapsed":2,"user":{"displayName":"ming li","userId":"14148720490428311514"}}},"execution_count":30,"outputs":[]},{"cell_type":"code","source":["f = open(path, encoding = \"utf-8\")\n","file_data = f.readlines()"],"metadata":{"id":"YynmL5fP4bwT","executionInfo":{"status":"ok","timestamp":1651650136175,"user_tz":-480,"elapsed":2,"user":{"displayName":"ming li","userId":"14148720490428311514"}}},"execution_count":31,"outputs":[]},{"cell_type":"code","source":["from typing import Dict\n","id_list = []\n","for line in file_data:\n"," id_list.append(line.strip('\\n').split(' ')[3][:-1])\n","\n","# data_list['id'] = [PullRequest_count, Fork_count, Watch_count, IssueComment_count, class]\n","map = dict()\n","map['PullRequestEvent'] = 0\n","map['ForkEvent'] = 1\n","map['WatchEvent'] = 2\n","map['IssueCommentEvent'] = 3\n","data_list = dict()"],"metadata":{"id":"Uz02esuk6bRl","executionInfo":{"status":"ok","timestamp":1651650137149,"user_tz":-480,"elapsed":976,"user":{"displayName":"ming li","userId":"14148720490428311514"}}},"execution_count":32,"outputs":[]},{"cell_type":"code","source":["# Initialize data_list\n","for id in id_list:\n"," data_list[id] = [0,0,0,0,0]"],"metadata":{"id":"JuyvyC-l9gRU","executionInfo":{"status":"ok","timestamp":1651650139027,"user_tz":-480,"elapsed":1885,"user":{"displayName":"ming li","userId":"14148720490428311514"}}},"execution_count":33,"outputs":[]},{"cell_type":"code","source":["for line in file_data: \n"," raw = line.strip('\\n').split(' ') \n"," data_list[raw[3][:-1]][map[raw[1][:-1]]] = int(raw[-1])"],"metadata":{"id":"72-EFogQ82I2","executionInfo":{"status":"ok","timestamp":1651650140722,"user_tz":-480,"elapsed":1697,"user":{"displayName":"ming li","userId":"14148720490428311514"}}},"execution_count":34,"outputs":[]},{"cell_type":"code","source":["p_list = []\n","final_data = []\n","for id in id_list:\n"," p_list.append(data_list[id][0])\n","p_list.sort(reverse=True)\n","threshold = p_list[int(len(p_list)*0.1)]\n","for id in id_list:\n"," if data_list[id][0]>=threshold:\n"," data_list[id][4] = 1\n"," final_data.append(data_list[id][1:])"],"metadata":{"id":"OiHpD_rX4nFS","executionInfo":{"status":"ok","timestamp":1651650364944,"user_tz":-480,"elapsed":3201,"user":{"displayName":"ming li","userId":"14148720490428311514"}}},"execution_count":38,"outputs":[]},{"cell_type":"code","source":["final_data = torch.tensor(final_data)"],"metadata":{"id":"W7d1zzzaCSVS","executionInfo":{"status":"ok","timestamp":1651650424843,"user_tz":-480,"elapsed":473,"user":{"displayName":"ming li","userId":"14148720490428311514"}}},"execution_count":39,"outputs":[]},{"cell_type":"code","source":["print(final_data.shape)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"VAQlRswJCXEy","executionInfo":{"status":"ok","timestamp":1651650434749,"user_tz":-480,"elapsed":409,"user":{"displayName":"ming li","userId":"14148720490428311514"}},"outputId":"5e2bf89e-ec92-4149-aa8b-01b42f232332"},"execution_count":40,"outputs":[{"output_type":"stream","name":"stdout","text":["torch.Size([883436, 4])\n"]}]},{"cell_type":"code","source":["class Dataset(torch.utils.data.Dataset):\n"," def __init__(self, data):\n"," self.labels = data[:, -1]\n"," self.x = data[:, 0:-1]\n"," def __len__(self):\n"," return len(self.labels)\n","\n"," def __getitem__(self, idx):\n"," x = self.x[idx].float()\n"," y = self.labels[idx]\n"," return x, y"],"metadata":{"id":"gPygHvtPBDZO","executionInfo":{"status":"ok","timestamp":1651651824344,"user_tz":-480,"elapsed":490,"user":{"displayName":"ming li","userId":"14148720490428311514"}}},"execution_count":68,"outputs":[]},{"cell_type":"code","source":["# randomly sample 70% as training data, and remaining 30% data as testing data\n","sample = [i for i in range(len(final_data))]\n","sample = random.sample(sample, len(final_data))\n","train = Dataset(final_data[sample[:int(len(final_data)*0.7)]])\n","train_dataloader = torch.utils.data.DataLoader(train, batch_size=256, shuffle=True, drop_last=True)"],"metadata":{"id":"McymePR1CtOl","executionInfo":{"status":"ok","timestamp":1651651827418,"user_tz":-480,"elapsed":1657,"user":{"displayName":"ming li","userId":"14148720490428311514"}}},"execution_count":69,"outputs":[]},{"cell_type":"code","source":["from torch.optim import Adam\n","from tqdm import tqdm\n","from torch import nn\n","def train(model, trainloader, learning_rate, epochs):\n"," use_cuda = torch.cuda.is_available()\n"," device = torch.device(\"cuda\" if use_cuda else \"cpu\")\n"," criterion = nn.CrossEntropyLoss()\n"," optimizer = Adam(model.parameters(), lr= learning_rate)\n"," if use_cuda:\n"," model = model.cuda()\n"," criterion = criterion.cuda()\n"," for epoch_num in range(epochs):\n"," acc_train = 0\n"," loss_train = 0\n"," for train_input, train_label in tqdm(trainloader):\n"," model.zero_grad()\n"," train_input = train_input.to(device)\n"," train_label = train_label.to(device)\n"," output = model(train_input)\n"," batch_loss = criterion(output, train_label)\n"," loss_train += batch_loss.item()\n"," batch_loss.backward()\n"," optimizer.step()\n"," with torch.no_grad():\n"," label_new = output.argmax(dim=1)\n"," acc_train = acc_train + (train_label==label_new).float().mean()\n"," with torch.no_grad():\n"," loss_train = loss_train / len(trainloader)\n"," acc_train = acc_train / len(trainloader)\n"," print(f'training loss:{loss_train}, training accuracy:{acc_train}')"],"metadata":{"id":"piLHSHz2EQ4J","executionInfo":{"status":"ok","timestamp":1651651827418,"user_tz":-480,"elapsed":3,"user":{"displayName":"ming li","userId":"14148720490428311514"}}},"execution_count":70,"outputs":[]},{"cell_type":"code","source":["model = nn.Sequential(nn.Linear(3,2))\n","train(model, trainloader=train_dataloader, learning_rate=11, epochs=20)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":346},"id":"4SfwM6QhGggu","executionInfo":{"status":"error","timestamp":1651652161257,"user_tz":-480,"elapsed":17598,"user":{"displayName":"ming li","userId":"14148720490428311514"}},"outputId":"73e9fef0-0b46-4e36-8a2e-2e240723dfcf"},"execution_count":78,"outputs":[{"output_type":"stream","name":"stderr","text":["100%|██████████| 4831/4831 [00:10<00:00, 465.96it/s]\n"]},{"output_type":"stream","name":"stdout","text":["training loss:68.62970850901237, training accuracy:0.8554679155349731\n"]},{"output_type":"stream","name":"stderr","text":[" 64%|██████▍ | 3091/4831 [00:06<00:03, 442.70it/s]\n"]},{"output_type":"error","ename":"KeyboardInterrupt","evalue":"ignored","traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)","\u001b[0;32m<ipython-input-78-917947935479>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSequential\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mLinear\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrainloader\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtrain_dataloader\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlearning_rate\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m11\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m20\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m","\u001b[0;32m<ipython-input-70-c9ff92025fab>\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(model, trainloader, learning_rate, epochs)\u001b[0m\n\u001b[1;32m 13\u001b[0m \u001b[0macc_train\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 14\u001b[0m \u001b[0mloss_train\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m \u001b[0;32mfor\u001b[0m \u001b[0mtrain_input\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_label\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtqdm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrainloader\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 16\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzero_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 17\u001b[0m \u001b[0mtrain_input\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain_input\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tqdm/std.py\u001b[0m in \u001b[0;36m__iter__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 1193\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1194\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1195\u001b[0;31m \u001b[0;32mfor\u001b[0m \u001b[0mobj\u001b[0m \u001b[0;32min\u001b[0m \u001b[0miterable\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1196\u001b[0m \u001b[0;32myield\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1197\u001b[0m \u001b[0;31m# Update and possibly print the progressbar.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m__next__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 528\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sampler_iter\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 529\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 530\u001b[0;31m \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_next_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 531\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_num_yielded\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 532\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dataset_kind\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0m_DatasetKind\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mIterable\u001b[0m \u001b[0;32mand\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m_next_data\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 568\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_next_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 569\u001b[0m \u001b[0mindex\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_next_index\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# may raise StopIteration\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 570\u001b[0;31m \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dataset_fetcher\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfetch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# may raise StopIteration\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 571\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_pin_memory\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 572\u001b[0m \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_utils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpin_memory\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpin_memory\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/utils/data/_utils/fetch.py\u001b[0m in \u001b[0;36mfetch\u001b[0;34m(self, possibly_batched_index)\u001b[0m\n\u001b[1;32m 50\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 51\u001b[0m \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 52\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcollate_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m","\u001b[0;31mKeyboardInterrupt\u001b[0m: "]}]},{"cell_type":"code","source":["test_data = final_data[sample[int(len(final_data)*0.7):]][:, :-1].float()\n","test_label = final_data[sample[int(len(final_data)*0.7):]][:, -1].long()\n","(model(test_data).argmax(dim=1)==test_label).float().mean()"],"metadata":{"id":"P6D3S8uaIZch","executionInfo":{"status":"ok","timestamp":1651652046350,"user_tz":-480,"elapsed":406,"user":{"displayName":"ming li","userId":"14148720490428311514"}}},"execution_count":76,"outputs":[]},{"cell_type":"code","source":[""],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"2xjuln6OGqoM","executionInfo":{"status":"ok","timestamp":1651652080979,"user_tz":-480,"elapsed":391,"user":{"displayName":"ming li","userId":"14148720490428311514"}},"outputId":"af0457f4-ec9f-4c63-9f16-55a421cb3a08"},"execution_count":77,"outputs":[{"output_type":"execute_result","data":{"text/plain":["tensor(0.9002)"]},"metadata":{},"execution_count":77}]},{"cell_type":"code","source":[""],"metadata":{"id":"8Mj19ZN-IrH2"},"execution_count":null,"outputs":[]}]}
|