diff --git a/README.md b/README.md index 17cae8c..61973ff 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,27 @@ # EvaluationEngine -Sources of the Java Evaluation Engine +Sources of the Java Evaluation Engine. The evaluation engine performs backend tasks for OpenML: + +- `process_datasets`: download a dataset, describe information per feature (e.g., the datatype and number of values per feature) and upload the results. +- `evaluate_run`: download a dataset and its splits, run the run's predictor on this dataset, and upload the evaluation (e.g., mean absolute error). +- `extract_features_all`: download a dataset, describe some features such as maximum number of distinct values of nominal features, and upload the results. +- `extract_features_simple`: same as `extract_features_all`, but with a subset of features. ## Usage -Please note that this library relies on the OpenML API Connector (maven id: org.openml.apiconnector) and often requires a snapshot version that is not released on Maven yet. Usually, this can be achieved by adding the latest version to your java classpath. -For more information on Snapshot dependencies, please check: -https://www.mojohaus.org/versions-maven-plugin/examples/unlock-snapshots.html + +See [docker](./docker/README.md) directory for how to run this using docker. + +### Prerequisites + +- Java. It works with 11 LTS, compatibility with other versions is unknown. +- Maven + +### Compile / package + + mvn clean compile # only compile + mvn clean package -DskipTests # package (generate jar) + +### Run +Generate the jar (see "Compile / package"). Then + + java -jar target/EvaluationEngine-*-SNAPSHOT-jar-with-dependencies.jar -f process_dataset -id 1 + java -jar target/EvaluationEngine-*-SNAPSHOT-jar-with-dependencies.jar -f evaluate_run -id 1 \ No newline at end of file diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000..82c2576 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,12 @@ +FROM maven:3.9.7-eclipse-temurin-11-alpine AS build +COPY lib /app/lib +COPY src /app/src +COPY pom.xml /app +RUN mvn -f /app/pom.xml clean package -DskipTests + +FROM openjdk:11-slim-bullseye +RUN useradd --create-home unprivileged-user +USER unprivileged-user + +COPY --from=build /app/target/EvaluationEngine-*-jar-with-dependencies.jar /usr/local/lib/evaluation-engine.jar +ENTRYPOINT ["java", "-jar", "/usr/local/lib/evaluation-engine.jar"] \ No newline at end of file diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 0000000..b10fc72 --- /dev/null +++ b/docker/README.md @@ -0,0 +1,20 @@ +# Evaluation Engine + +This docker container contains the OpenML Evaluation Engine. +Instructions only tested on a Linux host machine. + +First pull the docker image: + + docker pull openml/evaluation-engine + + +## Usage + + docker run -it openml/evaluation-engine + + +## Building + + docker build -f docker/Dockerfile --tag openml/evaluation-engine:0.0.1 . + + docker --push openml/evaluation-engine:0.0.1 \ No newline at end of file diff --git a/lib/README.md b/lib/README.md new file mode 100644 index 0000000..c3571c5 --- /dev/null +++ b/lib/README.md @@ -0,0 +1,5 @@ +# Libs + +Libraries that are not published to maven. + +We're having some problems publishing the Openml Apiconnector and OpenmlWeka to Maven Central. See build_instructions.txt in the directory of the ApiConnector (openml-java). I'm not able to make it work (Jos, Sept 2024). oss.sonatype.org throws a "Failed to deploy artifacts: Could not transfer artifact org.openml:apiconnector:kar:1.0.26 from/to sonatype-nexus-staging (https://oss.sonatype.org/service/local/staging/deploy/maven2/): Transfer failed for https://oss.sonatype.org/service/local/staging/deploy/maven2/org/openml/apiconnector/1.0.26/apiconnector-1.0.26.jar 401 Content access is protected by token." I assume that my token (created on https://central.sonatype.com/) is incorrect. I cannot log into https://oss.sonatype.org/. After a while, I decided to give up and to add the libraries here. Ugly, but it works... diff --git a/lib/apiconnector-1.0.11-SNAPSHOT.jar b/lib/apiconnector-1.0.11-SNAPSHOT.jar deleted file mode 100644 index 82971b9..0000000 Binary files a/lib/apiconnector-1.0.11-SNAPSHOT.jar and /dev/null differ diff --git a/lib/apiconnector-1.0.26-SNAPSHOT.jar b/lib/apiconnector-1.0.26-SNAPSHOT.jar new file mode 100644 index 0000000..e365fb9 Binary files /dev/null and b/lib/apiconnector-1.0.26-SNAPSHOT.jar differ diff --git a/lib/apiconnector/META-INF/MANIFEST.MF b/lib/apiconnector/META-INF/MANIFEST.MF new file mode 100644 index 0000000..ac4ddf8 --- /dev/null +++ b/lib/apiconnector/META-INF/MANIFEST.MF @@ -0,0 +1,4 @@ +Manifest-Version: 1.0 +Created-By: Maven Jar Plugin 3.2.0 +Build-Jdk-Spec: 11 + diff --git a/lib/apiconnector/META-INF/maven/org.openml/apiconnector/pom.properties b/lib/apiconnector/META-INF/maven/org.openml/apiconnector/pom.properties new file mode 100644 index 0000000..008d08e --- /dev/null +++ b/lib/apiconnector/META-INF/maven/org.openml/apiconnector/pom.properties @@ -0,0 +1,3 @@ +artifactId=apiconnector +groupId=org.openml +version=1.0.26-SNAPSHOT diff --git a/lib/apiconnector/META-INF/maven/org.openml/apiconnector/pom.xml b/lib/apiconnector/META-INF/maven/org.openml/apiconnector/pom.xml new file mode 100644 index 0000000..8b4e598 --- /dev/null +++ b/lib/apiconnector/META-INF/maven/org.openml/apiconnector/pom.xml @@ -0,0 +1,279 @@ + + + + 4.0.0 + + org.openml + apiconnector + 1.0.26-SNAPSHOT + jar + + APIConnector + Ancestor for API connector libraries for specific frameworks. + https://www.openml.org/ + + Leiden University, NL + https://liacs.leidenuniv.nl/ + + + + BSD-3-Clause + https://opensource.org/licenses/BSD-3-Clause + repo + + + + + + janvanrijn + Jan van Rijn + j.n.van.rijn@liacs.leidenuniv.nl + http://www.janvanrijn.eu + Leiden University, NL + http://liacs.leidenuniv.nl/ + + Postdoctoral Reaseacher + + + + + + + org.sonatype.oss + oss-parent + 7 + + + + scm:git:https://github.com/openml/java + scm:git:https://github.com/openml/java + https://github.com/openml/java + apiconnector-1.0.23 + + + + + release-sign-artifacts + + + performRelease + true + + + + + + org.apache.maven.plugins + maven-gpg-plugin + 1.4 + + + sign-artifacts + verify + + sign + + + + + + + + + + + no-tests + + true + + + + + + + junit + junit + 4.13 + test + + + + org.apache.commons + commons-io + 1.3.2 + + + + org.apache.commons + commons-lang3 + 3.12.0 + + + + org.apache.httpcomponents + httpclient + 4.5.13 + + + + org.apache.commons + commons-csv + 1.8 + + + + org.apache.httpcomponents + httpmime + 4.5.13 + + + + com.thoughtworks.xstream + xstream + 1.4.20 + + + + gov.nist.math + scimark + 2.0 + + + + org.json + json + 20210307 + + + + + UTF-8 + UTF-8 + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.1 + + 1.10 + 1.10 + + + + + org.apache.maven.plugins + maven-surefire-plugin + 3.0.0-M5 + + + **/Test*.java + + true + true + + + + + org.apache.maven.plugins + maven-release-plugin + 3.0.0-M1 + + false + true + true + true + + + + + + + + org.apache.maven.plugins + maven-clean-plugin + 3.1.0 + + + + . + + **/*~ + **/.attach_pid* + **/hs_err_pid* + + false + + + + + + + org.apache.maven.plugins + maven-jar-plugin + 3.2.0 + + + + test-jar + + + + + + + org.apache.maven.plugins + maven-source-plugin + 3.2.1 + + + attach-sources + + jar + + + true + + + + attach-test-sources + + test-jar + + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + 3.2.0 + + + attach-javadocs + + jar + + + + + protected + true + + ${project.name} + + + + org.eluder.coveralls + coveralls-maven-plugin + 4.3.0 + + + + + diff --git a/lib/commons-cli-1.2.jar b/lib/commons-cli-1.2.jar deleted file mode 100644 index ce4b9ff..0000000 Binary files a/lib/commons-cli-1.2.jar and /dev/null differ diff --git a/lib/commons-codec-1.9.jar b/lib/commons-codec-1.9.jar deleted file mode 100644 index ef35f1c..0000000 Binary files a/lib/commons-codec-1.9.jar and /dev/null differ diff --git a/lib/commons-io-2.4.jar b/lib/commons-io-2.4.jar deleted file mode 100644 index 90035a4..0000000 Binary files a/lib/commons-io-2.4.jar and /dev/null differ diff --git a/lib/commons-lang3-3.1.jar b/lib/commons-lang3-3.1.jar deleted file mode 100644 index a85e539..0000000 Binary files a/lib/commons-lang3-3.1.jar and /dev/null differ diff --git a/lib/commons-logging-1.2.jar b/lib/commons-logging-1.2.jar deleted file mode 100644 index 93a3b9f..0000000 Binary files a/lib/commons-logging-1.2.jar and /dev/null differ diff --git a/lib/commons-math3-3.4.jar b/lib/commons-math3-3.4.jar deleted file mode 100644 index f3eb86e..0000000 Binary files a/lib/commons-math3-3.4.jar and /dev/null differ diff --git a/lib/cortana.3103.jar b/lib/cortana.3103.jar deleted file mode 100644 index bae8854..0000000 Binary files a/lib/cortana.3103.jar and /dev/null differ diff --git a/lib/httpclient-4.5.2.jar b/lib/httpclient-4.5.2.jar deleted file mode 100644 index 701609f..0000000 Binary files a/lib/httpclient-4.5.2.jar and /dev/null differ diff --git a/lib/httpcore-4.4.4.jar b/lib/httpcore-4.4.4.jar deleted file mode 100644 index ac4a877..0000000 Binary files a/lib/httpcore-4.4.4.jar and /dev/null differ diff --git a/lib/httpmime-4.5.2.jar b/lib/httpmime-4.5.2.jar deleted file mode 100644 index 474670a..0000000 Binary files a/lib/httpmime-4.5.2.jar and /dev/null differ diff --git a/lib/openmlcortana.jar b/lib/openmlcortana.jar deleted file mode 100644 index f977ef3..0000000 Binary files a/lib/openmlcortana.jar and /dev/null differ diff --git a/lib/openmlweka-0.9.10-SNAPSHOT.jar b/lib/openmlweka-0.9.10-SNAPSHOT.jar new file mode 100644 index 0000000..a7e1beb Binary files /dev/null and b/lib/openmlweka-0.9.10-SNAPSHOT.jar differ diff --git a/lib/openmlweka/META-INF/MANIFEST.MF b/lib/openmlweka/META-INF/MANIFEST.MF new file mode 100644 index 0000000..78d1ef2 --- /dev/null +++ b/lib/openmlweka/META-INF/MANIFEST.MF @@ -0,0 +1,6 @@ +Manifest-Version: 1.0 +Archiver-Version: Plexus Archiver +Created-By: Apache Maven +Built-By: josvdvelde +Build-Jdk: 11.0.23 + diff --git a/lib/openmlweka/META-INF/maven/org.openml/openmlweka/pom.properties b/lib/openmlweka/META-INF/maven/org.openml/openmlweka/pom.properties new file mode 100644 index 0000000..421fd5f --- /dev/null +++ b/lib/openmlweka/META-INF/maven/org.openml/openmlweka/pom.properties @@ -0,0 +1,5 @@ +#Generated by Maven +#Fri Jul 19 11:28:28 CEST 2024 +groupId=org.openml +artifactId=openmlweka +version=0.9.10-SNAPSHOT diff --git a/lib/openmlweka/META-INF/maven/org.openml/openmlweka/pom.xml b/lib/openmlweka/META-INF/maven/org.openml/openmlweka/pom.xml new file mode 100644 index 0000000..3b2a42a --- /dev/null +++ b/lib/openmlweka/META-INF/maven/org.openml/openmlweka/pom.xml @@ -0,0 +1,264 @@ + + + + 4.0.0 + + org.openml + openmlweka + 0.9.10-SNAPSHOT + jar + + OpenmlWeka + Library for connecting Weka algorithms to OpenML. + https://openml.org/ + + Leiden University, NL + https://liacs.leidenuniv.nl/ + + + + GNU General Public License 3 + http://www.gnu.org/licenses/gpl-3.0.txt + repo + + + + + + janvanrijn + Jan van Rijn + j.n.van.rijn@liacs.leidenuniv.nl + http://www.janvanrijn.eu + University of Freiburg, DE + https://www.uni-freiburg.de/ + + Postdoctoral Reaseacher + + + + + + org.sonatype.oss + oss-parent + 7 + + + + scm:git:https://github.com/openml/openml-weka + scm:git:https://github.com/openml/openml-weka + https://github.com/openml/openml-weka + openmlweka-0.9.0 + + + + + release-sign-artifacts + + + performRelease + true + + + + + + org.apache.maven.plugins + maven-gpg-plugin + 1.1 + + + sign-artifacts + verify + + sign + + + + + + + + + + + no-tests + + true + + + + + + + junit + junit + 4.11 + test + + + + org.openml + apiconnector + 1.0.23 + + + + org.json + json + 20140107 + + + + org.apache.commons + commons-lang3 + 3.1 + + + + org.apache.commons + commons-math3 + 3.6.1 + + + + nz.ac.waikato.cms.weka + weka-dev + [3.9.0,) + + + + com.github.fracpete + multisearch-weka-package + [2018.9.2,) + + + + + UTF-8 + UTF-8 + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 2.0.2 + + 1.6 + 1.6 + + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.7.2 + + + **/Test*.java + + true + true + + + + + org.apache.maven.plugins + maven-release-plugin + 2.5.3 + + false + true + true + true + + + + + + + + org.apache.maven.plugins + maven-clean-plugin + 2.4.1 + + + + . + + **/*~ + **/.attach_pid* + **/hs_err_pid* + + false + + + + + + + org.apache.maven.plugins + maven-jar-plugin + 2.3.2 + + + + test-jar + + + + + + + org.apache.maven.plugins + maven-source-plugin + 2.1.2 + + + attach-sources + + jar + + + true + + + + attach-test-sources + + test-jar + + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + 3.1.0 + + + attach-javadocs + + jar + + + + + 8 + protected + true + + ${project.name} + + + + org.eluder.coveralls + coveralls-maven-plugin + 4.3.0 + + + + diff --git a/lib/org.json-20120521.jar b/lib/org.json-20120521.jar deleted file mode 100644 index f11fcd0..0000000 Binary files a/lib/org.json-20120521.jar and /dev/null differ diff --git a/lib/weka.jar b/lib/weka.jar deleted file mode 100644 index 35b31cd..0000000 Binary files a/lib/weka.jar and /dev/null differ diff --git a/lib/xstream-1.4.7.jar b/lib/xstream-1.4.7.jar deleted file mode 100644 index ea4b6a2..0000000 Binary files a/lib/xstream-1.4.7.jar and /dev/null differ diff --git a/pom.xml b/pom.xml index a1bfb2c..5cbca79 100644 --- a/pom.xml +++ b/pom.xml @@ -28,32 +28,90 @@ Assistant Professor - - + + + + 11 + 11 + 11 + + junit junit - 4.11 + 4.13.2 test - + org.openml openmlweka - 0.9.9-SNAPSHOT - + 0.9.10-SNAPSHOT + system + ${project.basedir}/lib/openmlweka-0.9.10-SNAPSHOT.jar - + + + + org.apache.commons + commons-io + 1.3.2 + + + + org.apache.commons + commons-lang3 + 3.12.0 + + + + org.apache.httpcomponents + httpclient + 4.5.13 + + + + org.apache.commons + commons-csv + 1.8 + + + + org.apache.httpcomponents + httpmime + 4.5.13 + + + + com.thoughtworks.xstream + xstream + 1.4.20 + + + + gov.nist.math + scimark + 2.0 + + + + org.json + json + 20210307 + + + org.openml apiconnector - 1.0.25-SNAPSHOT + 1.0.26 + system + ${project.basedir}/lib/apiconnector-1.0.26-SNAPSHOT.jar - - - + + org.mod4j.org.apache.commons cli @@ -71,19 +129,62 @@ weka-stable 3.8.2 - + + + com.github.fracpete + multisearch-weka-package + 2021.2.17 + + - + maven-compiler-plugin - 3.7.0 + 3.13.0 + + + maven-assembly-plugin + 3.7.1 1.10 1.10 + + + org.openml.webapplication.Main + + + + jar-with-dependencies + + + + make-assembly + package + + single + + + + + + + lib/apiconnector + + **/* + + + + lib/openmlweka + + **/* + + + +