# 9031. Online Judge - Judging Systemspawn

Introduce how to build Judging System for this online judge app.

## 1. Structure

### 1.1 Supported Languages

For each question, we have multiple language specified solution. Currently, this online judge application supports three languages: Java, Javascript and Python.

### 1.2 Test Case

For each question, there is one test case file shared by all solutions. For example, below is the test case for ‘Two Sum’ questions. There are 5 test cases in the test file. Each test case consists of three lines.

# ./server/solution/two-sum/testcase.txt
null
0
[0,0]
[]
0
[0,0]
[1,0,-1]
-1
[1,2]
[2,7,11,15]
9
[0,1]
[1,2,3,4,5,6,7,8,9,10]
19
[8,9]
[230,863,916,585,981,404,316,785,88,12,70,435,384,778,887,755,740,337,86,92,325,422,815,650,920,125,277,336,221,847,168,23,677,61,400,136,874,363,394,199,863,997,794,587,124,321,212,957,764,173,314,422,927,783,930,282,306,506,44,926,691,568,68,730,933,737,531,180,414,751,28,546,60,371,493,370,527,387,43,541,13,457,328,227,652,365,430,803,59,858,538,427,583,368,375,173,809,896,370,789]
542
[28,45]


Example of two sum:

Given nums = [2, 7, 11, 15], target = 9,

Because nums[0] + nums[1] = 2 + 7 = 9,
return [0, 1].


Corresponding test case. The first line is the ‘nums’ array, the second line is the ‘target’ value and the third line is the expected result.

[2,7,11,15]
9
[0,1]


## 2. Solution Verifier

For each language, there are at least two files.

• Solution.java(js/py) - The solution for the question. Its content will be replaced by user’s submitted solution.
• SolutionTester.java(js/py) - The test class to verify solution. It reads the test cases from ‘testcase.txt’ file and iterate them one by one, verify the input parameters against the expected result.

The following codes are the example for ‘Two Sum’ question. Other questions have the same structure.

### 2.1 Java

Solution.java

// ./server/solution/two-sum/java/Solution.java
import java.util.HashMap;

public class Solution {
public int[] twoSum(int[] nums, int target) {
int[] res = new int[]{0,0};
if (nums == null || nums.length < 2) {
return res;
}

HashMap<Integer, Integer> map = new HashMap<Integer, Integer>();

for (int i = 0; i < nums.length; i++) {
if(map.containsKey(nums[i])) {
res[0] = map.get(nums[i]);
res[1] = i;
return res;
} else {
map.put(target - nums[i], i);
}
}
return res;
}
}


SolutionTester.java

// ./server/solution/two-sum/java/SolutionTester.java
import java.io.BufferedWriter;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;

public class SolutionTester {
private static final String TESTCASE_FILE = "testcase.txt";
private static final String TESTRESULT_FILE = "testresult.txt";

public static void main(String[] args) {
Solution solution = new Solution();

//System.out.println("testTwoSum");

boolean testResult = true;
try {
try {
String line;
int count = 0;
while ((line = br.readLine()) != null) {
int[] nums = ParserUtil.stringToIntegerArray(line);
int target = Integer.parseInt(line);
int[] expected = ParserUtil.stringToIntegerArray(line);
// create an test execution
int[] ret = solution.twoSum(nums, target);

testResult = Arrays.equals(expected, ret);
count++;

if (!testResult) {
String content = "[Fail]Failed at: Input: " + Arrays.toString(nums) + ", " + target + "; Your answer:" + Arrays.toString(ret) + "; Expected answer: " + Arrays.toString(expected);
System.out.println("[Fail]" + Arrays.toString(nums) + ", " + target + ";" + Arrays.toString(ret) + ";" + Arrays.toString(expected));
saveTestResult(content);
break;
}
}
if (testResult) {
System.out.println("[Success]Your solution passed all " + count +" test cases!");
saveTestResult("[Success]Your solution passed all " + count +" test cases!");
}
}
catch (Exception io) {
System.out.println(io.getMessage());
printToFile(io);
}
finally {
br.close();
}
}
catch (IOException ioe) {
System.out.println(ioe.getMessage());
printToFile(ioe);
} finally {
}
}

public static void saveTestResult(String content) {
BufferedWriter bw = null;
try {
bw = new BufferedWriter(new FileWriter(TESTRESULT_FILE));
bw.write(content);
} catch (IOException ioe) {
ioe.printStackTrace();
} finally {
try {
if (bw != null) {
bw.close();
}
} catch (IOException ex) {
ex.printStackTrace();
}
}
}

public static void printToFile(Exception ex) {
PrintWriter pw = null;
try {
pw = new PrintWriter(TESTRESULT_FILE);
ex.printStackTrace(pw);
} catch (IOException ioe) {
ioe.printStackTrace();
} finally {
try {
if (pw != null) {
pw.close();
}
} catch (Exception ex2) {
ex2.printStackTrace();
}
}
}
}


### 2.2 Javascript

Solution.js

// ./server/solution/two-sum/javascript/Solution.js
var twoSum = function(nums, target) {
if (nums == null || nums.length < 2) {
return [0, 0];
}
var ret = [];
var exist = {};
for (var i = 0; i < nums.length; i++) {
if (typeof exist[target - nums[i]] !== "undefined") {
ret.push(exist[target - nums[i]]);
ret.push(i);
}

exist[nums[i]] = i;
}

return ret;
};

module.exports = twoSum;


SolutionTester.js

// ./server/solution/two-sum/javascript/SolutionTester.js
const path = require("path");
const fs = require("fs");
const twoSum = require("./Solution.js");

// read test case from file
const file = path.resolve(__dirname, "testcase.txt");
//console.log(data.toString());
const lines = data.toString().split("\n");
//console.log(lines);
var testcases = [];
for (let i = 0; i < lines.length; i = i + 3) {
// convert string to number with map(Number)
let nums = null;
if (lines[i] !== "null") {
nums = lines[i]
.substring(1, lines[i].length - 1)
.split(",")
.map(Number);
}
let target = parseInt(lines[i + 1], 10);
let expected = lines[i + 2]
.substring(1, lines[i + 2].length - 1)
.split(",")
.map(Number);

testcases.push({ nums, target, expected });
//console.log(testcase);
}

let testresult = true;
for (let i = 0; i < testcases.length; i++) {
const testcase = testcases[i];
//console.log(testcase);
var result = twoSum(testcase.nums, testcase.target);
//console.log("result:", result);
//console.log("testcase.expected:", testcase.expected);
if (!isEqual(testcase.expected, result)) {
const message =
"[Fail][" +
testcase.nums +
"]," +
testcase.target +
";[" +
result +
"];" +
testcase.expected;
testresult = false;
console.log(message);
break;
}
}

if (testresult) {
const message =
"[Success]Your solution passed all " + testcases.length + " test cases!";
console.log(message);
}

function isEqual(value, other) {
// Get the value type
var type = Object.prototype.toString.call(value);

// If the two objects are not the same type, return false
if (type !== Object.prototype.toString.call(other)) return false;

// If items are not an object or array, return false
if (["[object Array]", "[object Object]"].indexOf(type) < 0) return false;

// Compare the length of the length of the two items
var valueLen =
type === "[object Array]" ? value.length : Object.keys(value).length;
var otherLen =
type === "[object Array]" ? other.length : Object.keys(other).length;
if (valueLen !== otherLen) return false;

// Compare two items
var compare = function(item1, item2) {
// Get the object type
var itemType = Object.prototype.toString.call(item1);

// If an object or array, compare recursively
if (["[object Array]", "[object Object]"].indexOf(itemType) >= 0) {
if (!isEqual(item1, item2)) return false;
} else {
// Otherwise, do a simple comparison
// If the two items are not the same type, return false
if (itemType !== Object.prototype.toString.call(item2)) return false;

// Else if it's a function, convert to a string and compare
// Otherwise, just compare
if (itemType === "[object Function]") {
if (item1.toString() !== item2.toString()) return false;
} else {
if (item1 !== item2) return false;
}
}
};

// Compare properties
if (type === "[object Array]") {
for (var i = 0; i < valueLen; i++) {
if (compare(value[i], other[i]) === false) return false;
}
} else {
for (var key in value) {
if (value.hasOwnProperty(key)) {
if (compare(value[key], other[key]) === false) return false;
}
}
}

// If nothing failed, return true
return true;
}


### 2.2 Python

Solution.py

// ./server/solution/two-sum/python/Solution.py
class Solution(object):
def twoSum(self, nums, target):
if nums is None :
return [0,0]
if len(nums) <= 1:
return [0,0]
buff_dict = {}
for i in range(len(nums)):
if nums[i] in buff_dict:
return [buff_dict[nums[i]], i]
else:
buff_dict[target - nums[i]] = i


SolutionTester.py

// ./server/solution/two-sum/python/SolutionTester.py
import json
import Solution

def stringToIntegerList(input):

def stringToInt(input):
return int(input)

def integerListToString(nums, len_of_list=None):
if not len_of_list:
len_of_list = len(nums)
return json.dumps(nums[:len_of_list])

def main():
with open('testcase.txt', "r") as f:
i = 0
passall = True
while i < len(lines) :
line = lines[i]
nums = stringToIntegerList(line)
if (nums == "null") :
nums = None
#print nums
line = lines[i+1]
#print line
target = stringToInt(line)
line = lines[i+2]
#print line
expected = stringToIntegerList(line)

ret = Solution.Solution().twoSum(nums, target)

if (expected != ret) :
if (nums is None) :
strnums = 'null'
else:
strnums = integerListToString(nums)
print "[Fail]" + strnums + ", " + str(target) + ";" + integerListToString(ret) + ";" + integerListToString(expected)
passall = False
break

i = i + 3
#print out

if passall == True :
print "[Success]Your solution passed all " + str(len(lines)/3) + " test cases!"

if __name__ == '__main__':
main()


## 3. Judging Engine

We defined several runners for different languages. Use ‘RunnerManager’ to manage this runners.

### 3.1 Base Class

Runner.js

// ./server/judgingengine/Runner.js
class Runner {
constructor() {}

run(file, directory, filename, extension, callback) {
console.log("run");
}
}

module.exports = Runner;


### 3.2 Java Runner

// ./server/judgingengine/JavaRunner.js
const { spawn } = require("child_process");
const Runner = require("./Runner");
const exec = require("child_process").exec;

class JavaRunner extends Runner {
sourceFile() {
return this.sourcefile;
}
testFile() {
return this.testfile;
}

constructor() {
super();
this.sourcefile = "Solution.java";
this.testfile = "SolutionTester.java";
}

run(file, directory, filename, extension, callback) {
if (extension.toLowerCase() !== ".java") {
console.log(${file} is not a java file.); } this.compile(file, directory, filename, callback); } // compile java source file runtest(file, directory, filename, callback) { const argsCompile = []; argsCompile[0] = file; const runner = spawn("sh", argsCompile, { shell: true }); runner.stdout.on("data", data => { console.log(stdout:${data});
callback("ok", String(data)); // 0, no error
});
runner.stderr.on("data", data => {
console.log(compile-stderr: ${String(data)}); callback("err", String(data)); // 1, error }); runner.on("close", data => { this.log(close:${data});
});
}

// compile java source file
compile(file, directory, filename, callback) {
// set working directory for child_process
const options = { cwd: directory };
const options2 = { classpath: directory };
// var compiler = spawn('javac', ['CodeJava.java']);
const argsCompile = [];
argsCompile[0] = directory + "/ParserUtil.java";
argsCompile[1] = directory + "/Solution.java";
argsCompile[2] = file;
console.log(argsCompile);
const compiler = spawn("javac", argsCompile, options2);
compiler.stdout.on("data", data => {
console.log(stdout: ${data}); }); compiler.stderr.on("data", data => { console.log(compile-stderr:${String(data)});
callback("err_cmp", String(data)); // 1, compile error
});
compiler.on("close", data => {
if (data === 0) {
this.execute(filename, options, callback);
}
});
}

// execute the compiled class file
execute(filename, options, callback) {
const argsRun = [];
argsRun[0] = filename;
const executor = spawn("java", argsRun, options);
executor.stdout.on("data", output => {
const out = String(output);
//console.log(javaRunner->execute(): stdout:);
console.log(output);
if (out.startsWith("[Success]") || out.startsWith("[Fail]")) {
callback("ok", String(output)); // 0, no error
}
});
executor.stderr.on("data", output => {
//console.log(javaRunner->execute(): stderr: ${String(output)}); callback("err_exe", String(output)); // 2, execution failure }); executor.on("close", output => { this.log(javaRunner->execute(): close:${output});
});
}

log(message) {
console.log(message);
}
}

module.exports = JavaRunner;


### 3.3 Javascript Runner

// ./server/judgingengine/JavaScriptRunner.js
const { spawn } = require("child_process");
const Runner = require("./Runner");

class JavaScriptRunner extends Runner {
sourceFile() {
return this.sourcefile;
}
testFile() {
return this.testfile;
}

constructor() {
super();
this.sourcefile = "Solution.js";
this.testfile = "SolutionTester.js";
}

run(file, directory, filename, extension, callback) {
if (extension.toLowerCase() !== ".js") {
console.log(${file} is not a javascript file.); } this.execute(file, directory, callback); } execute(file, directory, callback) { // set working directory for child_process const options = { cwd: directory }; const argsRun = []; argsRun[0] = file; console.log(options:${options});
console.log(argsRun: ${argsRun}); // node SolutionTester.js 1 0 -1 -1 const executor = spawn("node", argsRun, options); executor.stdout.on("data", output => { const out = String(output); //console.log(javaRunner->execute(): stdout:); console.log(output); if (out.startsWith("[Success]") || out.startsWith("[Fail]")) { callback("ok", String(output)); // ok, no error } }); executor.stderr.on("data", output => { console.log(stderr:${String(output)});
callback("err_exe", String(output)); // err, execution failure
});
executor.on("close", output => {
console.log(stdout: ${output}); }); } } module.exports = JavaScriptRunner;  ### 3.4 Python Runner // ./server/judgingengine/PythonRunner.js const { spawn } = require("child_process"); const Runner = require("./Runner"); class PythonRunner extends Runner { sourceFile() { return this.sourcefile; } testFile() { return this.testfile; } constructor() { super(); this.sourcefile = "Solution.py"; this.testfile = "SolutionTester.py"; } run(file, directory, filename, extension, callback) { if (extension.toLowerCase() !== ".py") { console.log(${file} is not a python file.);
}
this.execute(file, directory, callback);
}

execute(file, directory, callback) {
// set working directory for child_process
const options = { cwd: directory };
const argsRun = [];
argsRun[0] = file;
console.log(options: ${options}); console.log(argsRun:${argsRun});
const executor = spawn("python", argsRun, options);
executor.stdout.on("data", output => {
const out = String(output);
console.log(pythonRunner->execute(): stdout:);
console.log(output);
if (out.startsWith("[Success]") || out.startsWith("[Fail]")) {
callback("ok", String(output)); // ok, no error
}
});
executor.stderr.on("data", output => {
console.log(stderr: ${String(output)}); callback("err_exe", String(output)); // err, execution failure }); executor.on("close", output => { console.log(stdout:${output});
});
}
}

module.exports = PythonRunner;


### 3.4 Runner Manager

Use ‘factory’ to create Runner. Save user’s solution to file first, then call specific runner to compile and run it.

// ./server/judgingengine/RunnerManager.js
const path = require("path");
const FileApi = require("../api/FileApi");
const CRunner = require("./CRunner");
const CppRunner = require("./CppRunner");
const JavaRunner = require("./JavaRunner");
const JavaScriptRunner = require("./JavaScriptRunner");
const PythonRunner = require("./PythonRunner");
const appRoot = require("app-root-path");
const moment = require("moment");
const os = require("os");

function Factory() {
this.createRunner = function createRunner(lang) {
let runner;

if (lang === "c") {
runner = new CRunner();
} else if (lang === "c++") {
runner = new CppRunner();
} else if (lang === "java") {
runner = new JavaRunner();
} else if (lang === "javascript") {
runner = new JavaScriptRunner();
} else if (lang === "python") {
runner = new PythonRunner();
}

return runner;
};
}

module.exports = {
run(question, lang, solution, callback) {
const factory = new Factory();
const runner = factory.createRunner(lang.toLowerCase());

// copy all files in the question folder from solution folder
const sourceDir = path.resolve(
${appRoot}, "server", "solution", question ); const targetDir = path.resolve( ${appRoot},
"server",
"judgingengine",
"temp",
question + "_" + lang + "_" + moment().toISOString() // 2013-02-04T22:44:30.652Z
);

// copy source code files
FileApi.copyDirectory(path.join(sourceDir, lang), targetDir, err => {
if (err) {
callback("99", String(err)); // 99, system error
}

const testcaseFile = path.join(targetDir, "testcase.txt");
// copy test case file
FileApi.copyFile(
path.join(sourceDir, "testcase.txt"),
testcaseFile,
err => {
if (err) {
callback("99", String(err)); // 99, system error
}
// save the solution to Solution.java
const sourceFile = path.resolve(targetDir, runner.sourceFile());
//console.log(source file: ${sourceFile}); const filename = path.parse(sourceFile).name; // main const extension = path.parse(sourceFile).ext; // .java //console.log(filename:${filename});
//console.log(extension: ${extension}); if (lang == "javascript") { // get method name and export it const method = solution .substring(solution.indexOf("var") + 4, solution.indexOf("=")) .trim(); solution = solution + " " + "module.exports = " + method + ";"; //solution = solution + os.EOL + " module.exports = reverseString;"; } FileApi.saveFile(sourceFile, solution, () => { const testFile = path.resolve(targetDir, runner.testFile()); const testFileName = path.parse(testFile).name; // main runner.run(testFile, targetDir, testFileName, extension, function( status, message ) { if (status == "ok") { // no error console.log("message"); console.log(message); if (message.startsWith("[Success]")) { callback("pass", message.slice(9)); // ok, pass } else { callback("fail", message.slice(6)); // ok, fail } } else { callback(status, message); } }); }); } ); }); } };  ## 4. Usage ### 4.1 Route // ./server/routes/submission.js var express = require("express"); var router = express.Router(); var submission_controller = require("../controllers/submission"); ... router.post("/run", submission_controller.submission_run); module.exports = router;  ### 4.2 Controller Controller method to save the submission and call the ‘run’ method. // ./server/controllers/submission.js exports.submission_run = function(req, res, next) { SleepUtil.sleep(); //sleep.sleep(3); //sleep for 3 seconds var newsubmit = new Submission({ username: req.body.username, questionname: req.body.questionname, language: req.body.language, solution: req.body.solution, status: "initial", // not submitted -> just created timeupdated: moment(new Date(Date.now())), timesubmitted: moment(new Date(Date.now())), runtime: 0 }); //console.log(newsubmit); Submission.findOne( { username: newsubmit.username, questionname: newsubmit.questionname, language: newsubmit.language, status: "initial" }, function(err, submission) { if (err) { return next(err); } if (!submission) { // 1. Save the submission first newsubmit.save({ new: true }, function(err, newsubmit) { if (err) { return next(err); } // 3. Run run(req, res, next, newsubmit); }); } else { // Update solution submission.solution = newsubmit.solution; submission.timeupdated = moment(new Date(Date.now())); Submission.findByIdAndUpdate( submission._id, {$set: submission },
{ new: true },
function(err, submission) {
if (err) return next(err);
// 3. Run
run(req, res, next, submission);
}
);
}
}
);
};


Call judging engine to verify the solution. Calculate the time consumption, save all the results and send them back to client.

// ./server/controllers/submission.js
function run(req, res, next, submission) {
var start = moment(new Date(Date.now()));

// 2. Then, run the solution to get the test result
RunnerManager.run(
submission.questionname,
submission.language,
submission.solution,
function(status, message) {
const result = {
status,
message
};
console.log(status);
if (status == "pass" || status == "fail") {
var end = moment(new Date(Date.now()));

var ms = moment(end, "DD/MM/YYYY HH:mm:ss").diff(
moment(start, "DD/MM/YYYY HH:mm:ss")
);

// 3. Find the submission
Submission.findById(submission.id, function(err, submission) {
// update status
submission.status = status;
submission.runtime = ms;
submission.timesubmitted = moment(new Date(Date.now()));

//console.log(submission);
// 4. Update the submission
submission.save(function(err) {
if (err) return next(err);
res.end(JSON.stringify(result));
});
});
} else {
res.end(JSON.stringify(result));
}
}
);
}


## 5. Client(Angular)

### 5.1 Submit Event

Submit the solution and show the result to UI.

// ./src/app/components/questions/algorithm-question.component.ts

...

onSubmitSolution() {
this.testResult = -1;
if (!this.validate2()) {
return;
}

//Form is valid, now perform create or update
let question = this.baseForm.value;
this.printLog(question);
let id = "";
let solution = "";
if (this.selectedLang == "java") {
id = this.submitId1;
solution = this.code1;
} else if (this.selectedLang == "javascript") {
id = this.submitId2;
solution = this.code2;
} else if (this.selectedLang == "python") {
id = this.submitId3;
solution = this.code3;
}
let submission = new Submission(
id,
this.uniquename,
question.language,
solution,
"initial",
new Date(),
null,
0
);
this.printLog(submission);

// Submit solution
this.submissionService.submitSolution(submission).subscribe(
response => {
this.printLog(response);
if (response.status === "pass") {
this.handleSuccess2(response.message);
this.testResult = 10;
this.resultMessage = response.message;
} else {
this.handleError2(response.message);
this.testResult = 20;
this.resultMessage = response.message;
}
// reset id to null to avoid update
if (this.selectedLang == "java") {
this.submitId1 = "";
} else if (this.selectedLang == "javascript") {
this.submitId2 = "";
} else if (this.selectedLang == "python") {
this.submitId3 = "";
}
this.refresh();
},
error => {
this.handleError2(error);
}
);
}


### 5.2 Service

// ./src/app/services/submission.service.ts
import { Injectable } from "@angular/core";
import { Observable } from "rxjs/Observable";
import { HttpClient } from "@angular/common/http";
import { environment } from "../../environments/environment";

import { QuestionExt, Submission } from "./../models";

@Injectable()
export class SubmissionService {
baseUrl = environment.apiUrl;
apiUrl = this.baseUrl + "api/submission";

...

//Submit solution
submitSolution(submission: Submission): Observable<any> {
return this.http.post(this.apiUrl + "/run", submission);
}
}


## 6. Demo

Select one question to submit solution, two results. Either the solution passes all the test cases, or it fails for one of the test cases.