Skip to content

Commit

Permalink
added size limit to path map to prevent out-of-memory
Browse files Browse the repository at this point in the history
  • Loading branch information
thomasWeise committed Mar 4, 2020
1 parent 7e9fc88 commit bf7cef4
Show file tree
Hide file tree
Showing 4 changed files with 83 additions and 14 deletions.
4 changes: 2 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -44,15 +44,15 @@ First, you need to add the following repository, which is a repository that can
```

Than you can add the dependency on our `aitoa-code` repository into your `dependencies` section.
Here, `0.8.36` is the current version of `aitoa-code`.
Here, `0.8.37` is the current version of `aitoa-code`.
Notice that you may have more dependencies in your `dependencies` section, say on `junit`, but here I just put the one for `aitoa-code` as example.

```xml
<dependencies>
<dependency>
<groupId>com.github.thomasWeise</groupId>
<artifactId>aitoa-code</artifactId>
<version>0.8.36</version>
<version>0.8.37</version>
</dependency>
</dependencies>
```
Expand Down
2 changes: 1 addition & 1 deletion pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@

<groupId>aitoa</groupId>
<artifactId>aitoa-code</artifactId>
<version>0.8.36</version>
<version>0.8.37</version>
<packaging>jar</packaging>
<name>aitoa-code</name>
<description>Example Source Codes from the Book "Introduction to Optimization Algorithms"</description>
Expand Down
4 changes: 2 additions & 2 deletions src/main/java/aitoa/algorithms/EA1p1WithFitness.java
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
* the solution space
*/
// start relevant
public class EA1p1WithFitness<X, Y>
public final class EA1p1WithFitness<X, Y>
implements IMetaheuristic<X, Y> {
/** the fitness assignment process */
public final FitnessAssignmentProcess<? super X> fitness;
Expand Down Expand Up @@ -92,7 +92,7 @@ public final void solve(final IBlackBoxProcess<X, Y> process) {

/** {@inheritDoc} */
@Override
public String toString() {
public final String toString() {
return "(1+1)-EA_" + //$NON-NLS-1$
this.fitness.toString();
}
Expand Down
87 changes: 78 additions & 9 deletions src/main/java/aitoa/utils/Experiment.java
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,9 @@
import java.nio.file.FileAlreadyExistsException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.ThreadLocalRandom;
import java.util.function.BiFunction;
Expand Down Expand Up @@ -406,7 +407,7 @@ public static final Path logFile(final Path root,
*/
private static final Path __logFile(final Path root,
final String algorithm, final String instance,
final long randSeed, final HashSet<Path> done,
final long randSeed, final __FileSet done,
final boolean onlyComputePath) throws IOException {

synchronized (IOUtils._IO_SYNCH) {
Expand Down Expand Up @@ -958,7 +959,7 @@ public static final <X, Y> void executeExperiment(
final boolean waitAfterIOError) {
Experiment._executeExperiment(stages, outputDir,
nameFunction, writeLogInfos, waitAfterSkippedRuns,
waitAfterWorkWasDone, waitAfterIOError, new HashSet<>());
waitAfterWorkWasDone, waitAfterIOError, new __FileSet());
}

/** perform garbage collection */
Expand Down Expand Up @@ -1095,7 +1096,7 @@ static final <X, Y> void _executeExperiment(
final boolean writeLogInfos,
final boolean waitAfterSkippedRuns,
final boolean waitAfterWorkWasDone,
final boolean waitAfterIOError, final HashSet<Path> done) {
final boolean waitAfterIOError, final __FileSet done) {

final BiFunction<IMetaheuristic<X, Y>,
BlackBoxProcessBuilder<X, Y>,
Expand Down Expand Up @@ -1298,11 +1299,18 @@ static final <X, Y> void _executeExperiment(
.printSetup((BufferedWriter) bw));
} catch (final IOException
| OutOfMemoryError error) {
Experiment.__doGc();

synchronized (done) {
done.remove(logFile);
if (error instanceof OutOfMemoryError) {
// The reason for the out-of-memory situation might have been
// that "done" grew too big. Maybe if we just clear it, we
// can prevent the error from occurring again.
done.clear();
} else {
done.remove(logFile);
}
}
Experiment.__doGc();

if (writeLogInfos) {
if (error instanceof OutOfMemoryError) {
ConsoleIO.stderr(
Expand Down Expand Up @@ -1500,7 +1508,20 @@ public static final void executeExperimentInParallel(
Supplier<IExperimentStage<?, ?, ?, ?>>> stages,
final Path outputDir) {
Experiment.executeExperimentInParallel(stages, outputDir,
Runtime.getRuntime().availableProcessors());
Experiment.defaultThreadCount());
}

/**
* Get the default thread count
*
* @return the default thread count
*/
public static final int defaultThreadCount() {
final int i = Runtime.getRuntime().availableProcessors();
if (i < 1) {
throw new IllegalStateException("No processors?"); //$NON-NLS-1$
}
return i;
}

/**
Expand Down Expand Up @@ -1578,7 +1599,7 @@ public static final <X, Y> void executeExperimentInParallel(
+ cores + " worker threads.");//$NON-NLS-1$
}

final HashSet<Path> done = new HashSet<>();
final __FileSet done = new __FileSet();

for (int i = threads.length; (--i) >= 0;) {
final Thread t = threads[i] = new Thread(
Expand Down Expand Up @@ -1619,4 +1640,52 @@ public static final <X, Y> void executeExperimentInParallel(
+ " worker threads, the experiment is complete.");//$NON-NLS-1$
}
}

/** the set of paths */
private static final class __FileSet
extends LinkedHashMap<Path, Object> {
/** the serial version uid */
private static final long serialVersionUID = 1L;
/** the key */
private static final Object KEY = new Object();
/** the maximum map size */
private static final int MAX_SIZE = 1024 * 1024;

/** create */
__FileSet() {
super();
}

/** {@inheritDoc} */
@Override
@SuppressWarnings("rawtypes")
protected final boolean
removeEldestEntry(final Map.Entry eldest) {
return this.size() > __FileSet.MAX_SIZE;
}

/**
* Add an element
*
* @param path
* the path
* @return {@code true} if the path was new, {@code false}
* otherwise
*/
public final boolean add(final Path path) {
return (this.put(path, __FileSet.KEY) == null);
}

/**
* Check if the element {@code path} is present
*
* @param path
* the path
* @return {@code true} if the element {@code path} is
* present, {@code false} if not
*/
public final boolean contains(final Path path) {
return this.get(path) != null;
}
}
}

0 comments on commit bf7cef4

Please sign in to comment.