You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

439 regels
12 KiB

1 jaar geleden
  1. #!/usr/bin/perl
  2. #######################################################################
  3. # driver.pl - CS:APP Data Lab driver
  4. #
  5. # Copyright (c) 2004-2011, R. Bryant and D. O'Hallaron, All rights
  6. # reserved. May not be used, modified, or copied without permission.
  7. #
  8. # Note: The driver can use either btest or the BDD checker to check
  9. # puzzles for correctness. This version of the lab uses btest, which
  10. # has been extended to do better testing of both integer and
  11. # floating-point puzzles.
  12. #
  13. #######################################################################
  14. use strict 'vars';
  15. use Getopt::Std;
  16. use lib ".";
  17. use Driverlib;
  18. # Set to 1 to use btest, 0 to use the BDD checker.
  19. my $USE_BTEST = 1;
  20. # Generic settings
  21. $| = 1; # Flush stdout each time
  22. umask(0077); # Files created by the user in tmp readable only by that user
  23. $ENV{PATH} = "/usr/local/bin:/usr/bin:/bin";
  24. #
  25. # usage - print help message and terminate
  26. #
  27. sub usage {
  28. printf STDERR "$_[0]\n";
  29. printf STDERR "Usage: $0 [-h] [-u \"nickname\"]\n";
  30. printf STDERR "Options:\n";
  31. printf STDERR " -h Print this message.\n";
  32. printf STDERR " -u \"nickname\" Send autoresult to server, using nickname on scoreboard)\n";
  33. die "\n";
  34. }
  35. ##############
  36. # Main routine
  37. ##############
  38. my $login = getlogin() || (getpwuid($<))[0] || "unknown";
  39. my $tmpdir = "/var/tmp/datalab.$login.$$";
  40. my $diemsg = "The files are in $tmpdir.";
  41. my $driverfiles;
  42. my $infile;
  43. my $autograded;
  44. my $status;
  45. my $inpuzzles;
  46. my $puzzlecnt;
  47. my $line;
  48. my $blank;
  49. my $name;
  50. my $c_points;
  51. my $c_rating;
  52. my $c_errors;
  53. my $p_points;
  54. my $p_rating;
  55. my $p_errors;
  56. my $total_c_points;
  57. my $total_c_rating;
  58. my $total_p_points;
  59. my $total_p_rating;
  60. my $tops;
  61. my $tpoints;
  62. my $trating;
  63. my $foo;
  64. my $name;
  65. my $msg;
  66. my $nickname;
  67. my $autoresult;
  68. my %puzzle_c_points;
  69. my %puzzle_c_rating;
  70. my %puzzle_c_errors;
  71. my %puzzle_p_points;
  72. my %puzzle_p_ops;
  73. my %puzzle_p_maxops;
  74. my %puzzle_number;
  75. # Parse the command line arguments
  76. no strict;
  77. getopts('hu:f:A');
  78. if ($opt_h) {
  79. usage();
  80. }
  81. # The default input file is bits.c (change with -f)
  82. $infile = "bits.c";
  83. $nickname = "";
  84. #####
  85. # These are command line args that every driver must support
  86. #
  87. # Causes the driver to send an autoresult to the server on behalf of user
  88. if ($opt_u) {
  89. $nickname = $opt_u;
  90. check_nickname($nickname);
  91. }
  92. # Hidden flag that indicates that the driver was invoked by an autograder
  93. if ($opt_A) {
  94. $autograded = $opt_A;
  95. }
  96. #####
  97. # Drivers can also define an arbitary number of other command line args
  98. #
  99. # Optional hidden flag used by the autograder
  100. if ($opt_f) {
  101. $infile = $opt_f;
  102. }
  103. use strict 'vars';
  104. ################################################
  105. # Compute the correctness and performance scores
  106. ################################################
  107. # Make sure that an executable dlc (data lab compiler) exists
  108. (-e "./dlc" and -x "./dlc")
  109. or die "$0: ERROR: No executable dlc binary.\n";
  110. # If using the bdd checker, then make sure it exists
  111. if (!$USE_BTEST) {
  112. (-e "./bddcheck/cbit/cbit" and -x "./bddcheck/cbit/cbit")
  113. or die "$0: ERROR: No executable cbit binary.\n";
  114. }
  115. #
  116. # Set up the contents of the scratch directory
  117. #
  118. system("mkdir $tmpdir") == 0
  119. or die "$0: Could not make scratch directory $tmpdir.\n";
  120. # Copy the student's work to the scratch directory
  121. unless (system("cp $infile $tmpdir/bits.c") == 0) {
  122. clean($tmpdir);
  123. die "$0: Could not copy file $infile to scratch directory $tmpdir.\n";
  124. }
  125. # Copy the various autograding files to the scratch directory
  126. if ($USE_BTEST) {
  127. $driverfiles = "Makefile dlc btest.c decl.c tests.c btest.h bits.h";
  128. unless (system("cp -r $driverfiles $tmpdir") == 0) {
  129. clean($tmpdir);
  130. die "$0: Could not copy autogradingfiles to $tmpdir.\n";
  131. }
  132. }
  133. else {
  134. $driverfiles = "dlc tests.c bddcheck";
  135. unless (system("cp -r $driverfiles $tmpdir") == 0) {
  136. clean($tmpdir);
  137. die "$0: Could not copy support files to $tmpdir.\n";
  138. }
  139. }
  140. # Change the current working directory to the scratch directory
  141. unless (chdir($tmpdir)) {
  142. clean($tmpdir);
  143. die "$0: Could not change directory to $tmpdir.\n";
  144. }
  145. #
  146. # Generate a zapped (for coding rules) version of bits.c. In this
  147. # zapped version of bits.c, any functions with illegal operators are
  148. # transformed to have empty function bodies.
  149. #
  150. print "1. Running './dlc -z' to identify coding rules violations.\n";
  151. system("cp bits.c save-bits.c") == 0
  152. or die "$0: ERROR: Could not create backup copy of bits.c. $diemsg\n";
  153. system("./dlc -z -o zap-bits.c bits.c") == 0
  154. or die "$0: ERROR: zapped bits.c did not compile. $diemsg\n";
  155. #
  156. # Run btest or BDD checker to determine correctness score
  157. #
  158. if ($USE_BTEST) {
  159. print "\n2. Compiling and running './btest -g' to determine correctness score.\n";
  160. system("cp zap-bits.c bits.c");
  161. # Compile btest
  162. system("make btestexplicit") == 0
  163. or die "$0: Could not make btest in $tmpdir. $diemsg\n";
  164. # Run btest
  165. $status = system("./btest -g > btest-zapped.out 2>&1");
  166. if ($status != 0) {
  167. die "$0: ERROR: btest check failed. $diemsg\n";
  168. }
  169. }
  170. else {
  171. print "\n2. Running './bddcheck/check.pl -g' to determine correctness score.\n";
  172. system("cp zap-bits.c bits.c");
  173. $status = system("./bddcheck/check.pl -g > btest-zapped.out 2>&1");
  174. if ($status != 0) {
  175. die "$0: ERROR: BDD check failed. $diemsg\n";
  176. }
  177. }
  178. #
  179. # Run dlc to identify operator count violations.
  180. #
  181. print "\n3. Running './dlc -Z' to identify operator count violations.\n";
  182. system("./dlc -Z -o Zap-bits.c save-bits.c") == 0
  183. or die "$0: ERROR: dlc unable to generated Zapped bits.c file.\n";
  184. #
  185. # Run btest or the bdd checker to compute performance score
  186. #
  187. if ($USE_BTEST) {
  188. print "\n4. Compiling and running './btest -g -r 2' to determine performance score.\n";
  189. system("cp Zap-bits.c bits.c");
  190. # Compile btest
  191. system("make btestexplicit") == 0
  192. or die "$0: Could not make btest in $tmpdir. $diemsg\n";
  193. print "\n";
  194. # Run btest
  195. $status = system("./btest -g -r 2 > btest-Zapped.out 2>&1");
  196. if ($status != 0) {
  197. die "$0: ERROR: Zapped btest failed. $diemsg\n";
  198. }
  199. }
  200. else {
  201. print "\n4. Running './bddcheck/check.pl -g -r 2' to determine performance score.\n";
  202. system("cp Zap-bits.c bits.c");
  203. $status = system("./bddcheck/check.pl -g -r 2 > btest-Zapped.out 2>&1");
  204. if ($status != 0) {
  205. die "$0: ERROR: Zapped bdd checker failed. $diemsg\n";
  206. }
  207. }
  208. #
  209. # Run dlc to get the operator counts on the zapped input file
  210. #
  211. print "\n5. Running './dlc -e' to get operator count of each function.\n";
  212. $status = system("./dlc -W1 -e zap-bits.c > dlc-opcount.out 2>&1");
  213. if ($status != 0) {
  214. die "$0: ERROR: bits.c did not compile. $diemsg\n";
  215. }
  216. #################################################################
  217. # Collect the correctness and performance results for each puzzle
  218. #################################################################
  219. #
  220. # Collect the correctness results
  221. #
  222. %puzzle_c_points = (); # Correctness score computed by btest
  223. %puzzle_c_errors = (); # Correctness error discovered by btest
  224. %puzzle_c_rating = (); # Correctness puzzle rating (max points)
  225. $inpuzzles = 0; # Becomes true when we start reading puzzle results
  226. $puzzlecnt = 0; # Each puzzle gets a unique number
  227. $total_c_points = 0;
  228. $total_c_rating = 0;
  229. open(INFILE, "$tmpdir/btest-zapped.out")
  230. or die "$0: ERROR: could not open input file $tmpdir/btest-zapped.out\n";
  231. while ($line = <INFILE>) {
  232. chomp($line);
  233. # Notice that we're ready to read the puzzle scores
  234. if ($line =~ /^Score/) {
  235. $inpuzzles = 1;
  236. next;
  237. }
  238. # Notice that we're through reading the puzzle scores
  239. if ($line =~ /^Total/) {
  240. $inpuzzles = 0;
  241. next;
  242. }
  243. # Read and record a puzzle's name and score
  244. if ($inpuzzles) {
  245. ($blank, $c_points, $c_rating, $c_errors, $name) = split(/\s+/, $line);
  246. $puzzle_c_points{$name} = $c_points;
  247. $puzzle_c_errors{$name} = $c_errors;
  248. $puzzle_c_rating{$name} = $c_rating;
  249. $puzzle_number{$name} = $puzzlecnt++;
  250. $total_c_points += $c_points;
  251. $total_c_rating += $c_rating;
  252. }
  253. }
  254. close(INFILE);
  255. #
  256. # Collect the performance results
  257. #
  258. %puzzle_p_points = (); # Performance points
  259. $inpuzzles = 0; # Becomes true when we start reading puzzle results
  260. $total_p_points = 0;
  261. $total_p_rating = 0;
  262. open(INFILE, "$tmpdir/btest-Zapped.out")
  263. or die "$0: ERROR: could not open input file $tmpdir/btest-Zapped.out\n";
  264. while ($line = <INFILE>) {
  265. chomp($line);
  266. # Notice that we're ready to read the puzzle scores
  267. if ($line =~ /^Score/) {
  268. $inpuzzles = 1;
  269. next;
  270. }
  271. # Notice that we're through reading the puzzle scores
  272. if ($line =~ /^Total/) {
  273. $inpuzzles = 0;
  274. next;
  275. }
  276. # Read and record a puzzle's name and score
  277. if ($inpuzzles) {
  278. ($blank, $p_points, $p_rating, $p_errors, $name) = split(/\s+/, $line);
  279. $puzzle_p_points{$name} = $p_points;
  280. $total_p_points += $p_points;
  281. $total_p_rating += $p_rating;
  282. }
  283. }
  284. close(INFILE);
  285. #
  286. # Collect the operator counts generated by dlc
  287. #
  288. open(INFILE, "$tmpdir/dlc-opcount.out")
  289. or die "$0: ERROR: could not open input file $tmpdir/dlc-opcount.out\n";
  290. $tops = 0;
  291. while ($line = <INFILE>) {
  292. chomp($line);
  293. if ($line =~ /(\d+) operators/) {
  294. ($foo, $foo, $foo, $name, $msg) = split(/:/, $line);
  295. $puzzle_p_ops{$name} = $1;
  296. $tops += $1;
  297. }
  298. }
  299. close(INFILE);
  300. #
  301. # Print a table of results sorted by puzzle number
  302. #
  303. print "\n";
  304. printf("%s\t%s\n", "Correctness Results", "Perf Results");
  305. printf("%s\t%s\t%s\t%s\t%s\t%s\n", "Points", "Rating", "Errors",
  306. "Points", "Ops", "Puzzle");
  307. foreach $name (sort {$puzzle_number{$a} <=> $puzzle_number{$b}}
  308. keys %puzzle_number) {
  309. printf("%d\t%d\t%d\t%d\t%d\t\%s\n",
  310. $puzzle_c_points{$name},
  311. $puzzle_c_rating{$name},
  312. $puzzle_c_errors{$name},
  313. $puzzle_p_points{$name},
  314. $puzzle_p_ops{$name},
  315. $name);
  316. }
  317. $tpoints = $total_c_points + $total_p_points;
  318. $trating = $total_c_rating + $total_p_rating;
  319. print "\nScore = $tpoints/$trating [$total_c_points/$total_c_rating Corr + $total_p_points/$total_p_rating Perf] ($tops total operators)\n";
  320. #
  321. # Optionally send the autoresult to the contest server if the driver
  322. # was called with the -u command line flag.
  323. #
  324. if ($nickname) {
  325. # Generate the autoresult
  326. $autoresult = "$tpoints|$total_c_points|$total_p_points|$tops";
  327. foreach $name (sort {$puzzle_number{$a} <=> $puzzle_number{$b}}
  328. keys %puzzle_number) {
  329. $autoresult .= " |$name:$puzzle_c_points{$name}:$puzzle_c_rating{$name}:$puzzle_p_points{$name}:$puzzle_p_ops{$name}";
  330. }
  331. # Post the autoresult to the server. The Linux login id is
  332. # concatenated with the user-supplied nickname for some (very) loose
  333. # authentication of submissions.
  334. &Driverlib::driver_post("$login:$nickname", $autoresult, $autograded);
  335. }
  336. # Clean up and exit
  337. clean ($tmpdir);
  338. exit;
  339. ##################
  340. # Helper functions
  341. #
  342. #
  343. # check_nickname - Check a nickname for legality
  344. #
  345. sub check_nickname {
  346. my $nickname = shift;
  347. # Nicknames can't be empty
  348. if (length($nickname) < 1) {
  349. die "$0: Error: Empty nickname.\n";
  350. }
  351. # Nicknames can't be too long
  352. if (length($nickname) > 35) {
  353. die "$0: Error: Nickname exceeds 35 characters.\n";
  354. }
  355. # Nicknames can have restricted set of metacharacters (e.g., no #
  356. # HTML tags)
  357. if (!($nickname =~ /^[_-\w.,'@ ]+$/)) {
  358. die "$0: Error: Illegal character in nickname. Only alphanumerics, apostrophes, commas, periods, dashes, underscores, and ampersands are allowed.\n";
  359. }
  360. # Nicknames can't be all whitespace
  361. if ($nickname =~ /^\s*$/) {
  362. die "$0: Error: Nickname is all whitespace.\n";
  363. }
  364. }
  365. #
  366. # clean - remove the scratch directory
  367. #
  368. sub clean {
  369. my $tmpdir = shift;
  370. system("rm -rf $tmpdir");
  371. }