build.xml

<?xml version="1.0"?>
<project name="pegasus" default="dist" xmlns:if="ant:if" xmlns:unless="ant:unless">
  <property environment="env"/>
  <property file="build.properties"/>
  <property file="build.git.properties"/>
  <property name="build.source" value="1.8"/>
  <property name="build.target" value="1.8"/>
  <property name="make" value="make"/>
  <property name="bash" value="bash"/>
  <property name="compile.lint" value="-Xlint:none"/>
  <property name="build.src" location="build/main/classes"/>
  <property name="test.src"  location="build/tests/classes"/>
  <property name="junitreport.dir"  value="${test.src}/junitreport"/>
  <property name="dist.src.dir" location="dist/pegasus-source-${pegasus.version}"/>
  <property name="dist.dir" location="dist/pegasus-${pegasus.version}"/>
  <property name="dist.bin" location="${dist.dir}/bin"/>
  <property name="dist.etc" location="${dist.dir}/etc"/>
  <property name="dist.data" location="${dist.dir}/share"/>
  <property name="dist.share" location="${dist.data}/pegasus"/>
  <property name="dist.jars" location="${dist.share}/java"/>
  <property name="dist.aws.jars" location="${dist.share}/java/aws"/>
  <property name="dist.schema" location="${dist.share}/schema"/>
  <property name="dist.doc" location="${dist.dir}/share/doc/pegasus"/>

  <!-- Set property to identify if running in CI mode. -->
  <property name="CI" value="true" if:set="env.bamboo_buildKey" />
  <property name="CI" value="false" unless:set="env.bamboo_buildKey" />

  <path id="build.classpath">
    <fileset dir="share/pegasus/java" includes="*.jar"/>
  </path>

  <!-- Dependencies needed for Java Unit Testing -->
  <path id="java.test.classpath">
    <fileset dir="release-tools/jars/target/dependency" includes="*.jar"/>
  </path>

  <path id="build.aws.classpath">
    <fileset dir="share/pegasus/java/aws" includes="*.jar"/>
  </path>

  <echo>Pegasus Version: ${pegasus.version}</echo>
  <echo>JAVA TARGET is ${build.target}</echo>
  <echo>JAVA SOURCE is ${build.source}</echo>
  <echo>Running in CI ${CI}</echo>

  <tstamp>
    <format property="ISODATE.full" pattern="yyyy-MM-dd'T'HH:mm:ssZZZZZ"/>
    <format property="ISODATE.short" pattern="yyyyMMdd"/>
    <format property="ISODATE.utc" pattern="yyyyMMddHHmmss'Z'" timezone="UTC"/>
    <format property="ISODATE.rpm" pattern="EEE MMM dd yyyy"/>
    <format property="ISODATE.deb" pattern="EEE, d MMM yyyy HH:mm:ss Z"/>
  </tstamp>
  <echo>Build timestamp: ${ISODATE.utc}</echo>

  <!-- Determine the system architecture and platform -->
  <exec executable="${basedir}/release-tools/getsystem"
        outputproperty="dist.arch"
        failonerror="true"/>
  <echo>Architecture: ${dist.arch}</echo>

  <!-- OSID is used for Debian package builds -->
  <exec executable="${basedir}/release-tools/getosid"
        outputproperty="dist.osid"
        failonerror="true"/>
  <echo>OSID: ${dist.osid}</echo>

  <!-- Determine the system library location (lib64 or lib) -->
  <exec executable="${basedir}/release-tools/determine-lib-dir"
        outputproperty="dist.lib.subdir"
        failonerror="true"/>
  <echo>Library subdir: ${dist.lib.subdir}</echo>
  <property name="dist.lib" location="${dist.dir}/${dist.lib.subdir}"/>

  <!-- Determine the system Python -->
  <exec executable="${basedir}/release-tools/get-system-python"
        outputproperty="dist.python"
        failonerror="true">
  </exec>
  <echo>Python: ${dist.python}</echo>

  <!-- Show pip3 version -->
  <exec executable="${dist.python}"
        outputproperty="dist.pip3"
        failonerror="true">
        <arg line="-m pip -V" />
  </exec>
  <echo>pip3: ${dist.pip3}</echo>

  <!-- Determine the system Python module location -->
  <exec executable="${basedir}/release-tools/python-modules-dir"
        outputproperty="dist.python-lib.subdir"
        failonerror="true">
    <arg value="${dist.lib.subdir}"/>
  </exec>
  <echo>Python library subdir: ${dist.python-lib.subdir}</echo>
  <property name="dist.python-lib" location="${dist.dir}/${dist.python-lib.subdir}"/>

  <macrodef name="compile-python">
    <attribute name="name"/>
    <sequential>
      <mkdir dir="${dist.python-lib}"/>
      <exec executable="mktemp" failonerror="true" outputproperty="target.dir">
        <arg line="-d"/>
      </exec>
      <exec executable="${dist.python}" failonerror="true">
        <arg line="-m pip install --target ${target.dir} --no-deps packages/@{name}"/>
      </exec>
      <copy preservelastmodified="true" todir="${dist.python-lib}">
        <fileset dir="${target.dir}"/>
      </copy>
      <delete dir="${target.dir}"/>
      <mkdir dir="${dist.lib}/pegasus"/>
      <symlink link="${dist.lib}/pegasus/python" resource="../../${dist.python-lib.subdir}" overwrite="true"/>
    </sequential>
  </macrodef>

  <macrodef name="make-c-tool">
    <attribute name="name"/>
    <sequential>
      <exec executable="${make}" dir="packages/@{name}" failonerror="true">
        <env key="prefix" value="${dist.dir}"/>
        <env key="datadir" value="${dist.data}"/>
        <arg line="prefix=${dist.dir}"/>
        <arg line="datadir=${dist.data}"/>
        <arg line="install"/>
      </exec>
    </sequential>
  </macrodef>

  <macrodef name="package-binary">
    <attribute name="name"/>
    <sequential>
      <!-- creates the binary tarball -->
      <exec executable="sh" dir="dist" failonerror="true">
        <arg value="-c"/>
        <arg value="tar --exclude=\.git --exclude=\.gitignore --exclude=pegasus-python-wrapper -cf @{name}.tar pegasus-${pegasus.version}"/>
      </exec>
      <gzip src="dist/@{name}.tar" destfile="dist/@{name}.tar.gz"/>
      <delete file="dist/@{name}.tar"/>
    </sequential>
  </macrodef>

  <target name="setup" depends="set-git-hash">
    <echo>Git hash: ${pegasus.build.git.hash}</echo>

    <mkdir dir="dist"/>
    <mkdir dir="${build.src}"/>
    <mkdir dir="${test.src}"/>
    <mkdir dir="${dist.dir}"/>

    <!-- Not sure why these are needed -->
    <echo file="${dist.dir}/stamp">${ISODATE.utc} ${dist.arch}${line.separator}</echo>
    <copy preservelastmodified="true" todir="${build.src}" file="LICENSE"/>

    <!-- Generate build properties file -->
    <echo file="${build.src}/pegasus.build.properties">
pegasus.build.version=${pegasus.version}
pegasus.build.platform=${dist.arch}
pegasus.build.timestamp=${ISODATE.utc}
pegasus.build.libdir=${dist.lib.subdir}
pegasus.build.pydir=${dist.python-lib.subdir}
pegasus.build.git.hash=${pegasus.build.git.hash}
    </echo>

    <!-- RPM spec file with updated version number -->
    <copy preservelastmodified="true" overwrite="true" file="pegasus.spec.in" tofile="pegasus.spec">
      <filterset>
        <filter token="PEGASUS_VERSION" value="${pegasus.version}"/>
        <filter token="DATE" value="${ISODATE.rpm}"/>
      </filterset>
    </copy>

    <!-- Create pegasus-config -->
    <mkdir dir="${dist.bin}"/>
    <copy preservelastmodified="true" todir="${dist.bin}">
      <fileset dir="bin">
          <include name="pegasus-python-wrapper"/>
          <include name="pegasus-db-admin"/>
      </fileset>
      <filterset>
        <filter token="PEGASUS_VERSION" value="${pegasus.version}"/>
        <filter token="LIBDIR" value="${dist.lib.subdir}"/>
        <filter token="PYTHON_LIBDIR" value="${dist.lib.subdir}/pegasus/python"/>
      </filterset>
    </copy>
  </target>

  <target name="set-git-hash" unless="pegasus.build.git.hash">
    <!-- Get Git Hash if it isn't already set -->
    <echo>Setting pegasus.build.git.hash</echo>
    <exec executable="/bin/bash" outputproperty="pegasus.build.git.hash" failonerror="true">
        <arg value="-c"/>
        <arg value="git rev-parse HEAD"/>
    </exec>
  </target>

  <!-- Python code format -->
  <target name="code-format-python" description="Python code format">
    <exec executable="tox" dir="packages/pegasus-common" resultproperty="python-common.format.error">
      <env key="CI" value="${CI}"/>
      <arg line="--recreate"/>
      <arg line="-e lint"/>
    </exec>
    <exec executable="tox" dir="packages/pegasus-api" resultproperty="python-api.format.error">
      <env key="CI" value="${CI}"/>
      <arg line="--recreate"/>
      <arg line="-e lint"/>
    </exec>
    <exec executable="tox" dir="packages/pegasus-worker" resultproperty="python-worker.format.error">
      <env key="CI" value="${CI}"/>
      <arg line="--recreate"/>
      <arg line="-e lint"/>
    </exec>
    <exec executable="tox" dir="packages/pegasus-python" resultproperty="python.format.error">
      <env key="CI" value="${CI}"/>
      <arg line="--recreate"/>
      <arg line="-e lint"/>
    </exec>

    <fail>
      Lint Error
      # -- NOTE -- #
      Invalid Python code format, run `ant code-format-python`.
      # -- NOTE -- #
      <condition>
        <or>
          <not>
            <equals arg1="${python-common.format.error}" arg2="0"></equals>
          </not>
          <not>
            <equals arg1="${python-api.format.error}" arg2="0"></equals>
          </not>
          <not>
            <equals arg1="${python-worker.format.error}" arg2="0"></equals>
          </not>
          <not>
            <equals arg1="${python.format.error}" arg2="0"></equals>
          </not>
        </or>
      </condition>
    </fail>
  </target>

  <!-- Java code format -->
  <target name="code-format-java" description="Java code format">
    <fileset id="java.source" dir=".">
      <include name="src/**/*.java"/>
      <exclude name="src/**/aws/**/CloudWatchLog.java"/>
      <include name="test/junit/**/*.java"/>
    </fileset>

    <pathconvert refid="java.source" property="java.source" pathsep=" " />

    <java jar="${basedir}/release-tools/jars/google-java-format-1.7-all-deps.jar" fork="true" maxmemory="128m" resultproperty="java.format.error">
      <arg value="--aosp"/>
      <arg value="--replace" unless:true="${CI}" />
      <arg line="--set-exit-if-changed --dry-run" if:true="${CI}" />
      <arg line="${java.source}"/>
    </java>

    <fail>
      Lint Error
      # -- NOTE -- #
      Invalid Java code format, run `ant code-format-java` and commit the changes to fix the issue.
      # -- NOTE -- #
      <condition>
        <not>
          <equals arg1="${java.format.error}" arg2="0"></equals>
        </not>
      </condition>
    </fail>
  </target>

  <target name="compile-pegasus-common" depends="setup">
    <compile-python name="pegasus-common"/>
  </target>

  <target name="compile-pegasus-api" depends="setup">
    <compile-python name="pegasus-api"/>
  </target>

  <target name="compile-pegasus-worker" depends="setup">
    <compile-python name="pegasus-worker"/>
  </target>

  <target name="compile-pegasus-python" depends="setup">
    <compile-python name="pegasus-python"/>
  </target>

  <target name="compile-python" depends="compile-pegasus-common,compile-pegasus-api,compile-pegasus-worker,compile-pegasus-python">
    <!-- Update pegasus-config -->
    <replace file="${dist.dir}/${dist.python-lib.subdir}/Pegasus/cli/pegasus-config.py" token="@PEGASUS_VERSION@" value="${pegasus.version}" />
    <replace file="${dist.dir}/${dist.python-lib.subdir}/Pegasus/cli/pegasus-config.py" token="@LIBDIR@" value="${dist.lib.subdir}" />
    <replace file="${dist.dir}/${dist.python-lib.subdir}/Pegasus/cli/pegasus-config.py" token="@PYTHON_LIBDIR@" value="${dist.lib.subdir}/pegasus/python" />
    <!-- replace sometimes changes permissions - make sure pegasus-config.py permissions are correct -->
    <chmod perm="0644" type="file">
      <fileset dir="${dist.python-lib}/Pegasus/cli">
        <include name="pegasus-config.py"/>
      </fileset>
    </chmod>

    <chmod perm="0755" type="file">
      <fileset dir="${dist.python-lib}">
        <include name="Pegasus/tools/kickstart_parser.py"/>
      </fileset>
    </chmod>
  </target>

  <target name="compile-externals" depends="setup" description="Compile external dependencies">
    <exec executable="${dist.python}" dir="src" failonerror="true">
      <arg line="-m pip install -t ../build/externals --no-deps -r requirements.txt"/>
    </exec>
  </target>

  <target name="compile-common" depends="setup" description="Compile common java code">
    <javac destdir="${build.src}" srcdir="src"
           encoding="UTF-8" deprecation="true" debug="true"
           target="${build.target}" source="${build.source}"
           includes="edu/isi/pegasus/common/**/*.java"
           excludes="edu/isi/pegasus/common/**/*Test.java,edu/isi/pegasus/planner/**/*.java"
           classpathref="build.classpath"
           includeantruntime="false">
      <compilerarg value="${compile.lint}"/>
    </javac>
  </target>

  <target name="compile-vdl" depends="setup" description="Compile java VDL code">
    <javac destdir="${build.src}" srcdir="src"
           target="${build.target}" source="${build.source}"
           encoding="UTF-8" debug="true"
           includes="org/griphyn/vdl/*.java,org/griphyn/vdl/**/*.java"
           excludes="org/griphyn/vdl/*Test.java,org/griphyn/vdl/**/*Test.java"
           classpathref="build.classpath"
           includeantruntime="false">
      <compilerarg value="${compile.lint}"/>
    </javac>
  </target>

  <target name="compile-planner" depends="setup" description="Compile java planner code">
    <javac destdir="${build.src}" srcdir="src"
           target="${build.target}" source="${build.source}"
           encoding="UTF-8" debug="true"
           includes="edu/isi/pegasus/planner/**/*.java,edu/isi/ikcap/**/*.java"
           excludes="edu/isi/pegasus/planner/**/*Test.java,org/griphyn/**/*.java"
           classpathref="build.classpath"
           includeantruntime="false">
      <compilerarg value="${compile.lint}"/>
    </javac>
  </target>

  <target name="compile-aws-batch" depends="compile-planner" description="Compile Pegasus AWS Batch Code">
    <echo>${toString:build.aws.classpath}</echo>
    <echo>${toString:build.classpath}</echo>
    <javac destdir="${build.src}" srcdir="src"
           target="${build.target}" source="${build.source}"
           encoding="UTF-8" debug="true"
           includes="edu/isi/pegasus/aws/**/*.java"
           excludes="edu/isi/pegasus/aws/**/*Test.java,org/griphyn/**/*.java"
           classpathref="build.aws.classpath"
           includeantruntime="false">
      <compilerarg value="${compile.lint}"/>
      <classpath refid="build.classpath" />
    </javac>
  </target>

  <target name="compile-junit" depends="setup,compile-common,compile-planner" description="Compile java unit tests">
    <javac destdir="${test.src}" srcdir="test/junit"
           target="${build.target}" source="${build.source}"
           encoding="UTF-8" debug="true"
           includes="edu/isi/pegasus/**/*.java"
           includeantruntime="false">
      <classpath>
        <path refid="build.classpath"/>
        <path refid="java.test.classpath"/>
        <path location="${build.src}"/>
      </classpath>
      <compilerarg value="${compile.lint}"/>
    </javac>
  </target>

  <target name="compile-pegasus-cluster" depends="setup" description="Compile Pegasus Cluster">
    <make-c-tool name="pegasus-cluster"/>
  </target>

  <target name="compile-pegasus-keg" depends="setup" description="Compile Pegasus Keg">
    <make-c-tool name="pegasus-keg"/>
  </target>

  <target name="compile-pegasus-kickstart" depends="setup" description="Compile Pegasus Kickstart">
    <make-c-tool name="pegasus-kickstart"/>
  </target>

  <target name="compile-pegasus-mpi-cluster" depends="setup" description="Compile Pegasus MPI Cluster">
    <make-c-tool name="pegasus-mpi-cluster"/>
  </target>

  <target name="compile-c" depends="setup,compile-pegasus-cluster,compile-pegasus-keg,compile-pegasus-kickstart" description="Compile C tools"/>

  <target name="compile-java" depends="compile-vdl,compile-planner,compile-common,compile-junit" description="Copile all java code"/>

  <target name="compile" depends="compile-java,compile-c,compile-externals" description="Compile all code"/>

  <target name="jar-planner" depends="compile-common,compile-planner" description="Generate planner jar">
    <mkdir dir="${dist.jars}"/>
    <jar destfile="${dist.jars}/pegasus.jar" basedir="${build.src}"
         includes="*.properties,edu/isi/pegasus/**/*.class,edu/isi/ikcap/**/*.class"
         excludes="*.class,edu/isi/pegasus/aws/**/*.class,edu/isi/pegasus/planner/test/**/*.class,edu/isi/pegasus/**/*Test.class,org/griphyn/**/*Test.class,org/griphyn/**/*.class"/>
    <chmod perm="0644" file="${dist.jars}/pegasus.jar"/>
  </target>

  <target name="jar-aws-batch" depends="compile-aws-batch" description="Generate jar for Pegauss AWS Batch Code">
    <mkdir dir="${dist.jars}"/>
    <jar destfile="${dist.jars}/pegasus-aws-batch.jar" basedir="${build.src}"
         includes="*.properties,edu/isi/pegasus/aws/**/*.class"
         excludes="*.class,edu/isi/pegasus/aws/**/*Test.class"/>
    <chmod perm="0644" file="${dist.jars}/pegasus-aws-batch.jar"/>
  </target>

  <target name="jar-junit" depends="compile-junit" description="Generate unit test jar">
    <mkdir dir="${dist.jars}"/>
    <jar destfile="${dist.jars}/pegasus-test.jar" basedir="${test.src}"
         includes="edu/isi/pegasus/**/*.class"/>
    <chmod perm="0644" file="${dist.jars}/pegasus-test.jar"/>
  </target>

  <target name="jar-vdl" depends="compile-vdl" description="Generate VDL jar">
    <mkdir dir="${dist.jars}"/>
    <jar destfile="${dist.jars}/vdl.jar" basedir="${build.src}"
         includes="org/griphyn/vdl/**/*.class,org/griphyn/vdl/*.class"
         excludes="*.class,org/griphyn/vdl/**/*Test.class,org/griphyn/vdl/*Test.class,edu/isi/pegasus/**/*.jar"/>
    <chmod perm="0644" file="${dist.jars}/vdl.jar"/>
  </target>

  <target name="jar" depends="jar-planner,jar-vdl" description="Generate all jar files"/>

  <target name="doc-user-guide" depends="setup" description="Generate user guide from sphinx source">
    <mkdir dir="${dist.doc}"/>

    <replaceregexp file="doc/sphinx/conf.py" match="^version.*$" replace="version = &quot;${pegasus.version}&quot;" byline="true" />
    <replaceregexp file="doc/sphinx/conf.py" match="^release.*$" replace="release = &quot;${pegasus.version}&quot;" byline="true" />
    <exec executable="tox" dir="packages/pegasus-python" failonerror="true">
      <arg line="--recreate"/>
      <arg line="-e docs"/>
    </exec>
    <replaceregexp file="doc/sphinx/conf.py" match="^version.*$" replace="version = &quot;5.0.0dev&quot;" byline="true" />
    <replaceregexp file="doc/sphinx/conf.py" match="^release.*$" replace="release = &quot;5.0.0dev&quot;" byline="true" />

    <!-- Copy HTML -->
    <copy preservelastmodified="true" todir="${dist.doc}/wordpress">
      <fileset dir="doc/sphinx/_build/html" includes="**/*"/>
    </copy>

    <!-- Copy PDF -->
    <copy preservelastmodified="true" todir="${dist.doc}/wordpress">
      <fileset dir="doc/sphinx/_build/latex" includes="**/*.pdf"/>
    </copy>

    <!-- Copy manpages -->
    <copy preservelastmodified="true" todir="${dist.data}/man/man1">
      <fileset dir="doc/sphinx/_build/man" includes="*"/>
    </copy>

    <!-- ensure the python docs where created -->
    <fail message="Python documentation was not generated correctly!">
        <condition>
            <not>
                <available file="doc/sphinx/_build/html/python/Pegasus.api.html" />
            </not>
        </condition>
    </fail>

  </target>

  <target name="doc-java" depends="setup" description="Generate java documentation">
    <delete dir="${dist.doc}/wordpress/javadoc"/>
    <javadoc destdir="${dist.doc}/wordpress/javadoc"
             Author="true"
             Use="true"
             Version="true"
             Private="true"
             Windowtitle="PEGASUS"
	     Encoding="UTF-8"
             classpathref="build.classpath">
      <packageset dir="src">
        <include name="edu/isi/pegasus/planner/dax/**"/>
	  <include name="edu/isi/pegasus/planner/selector/**"/>
      </packageset>
      <fileset dir="src" includes="*.java"/>
      <doctitle><![CDATA[PEGASUS ${pegasus.version}]]></doctitle>
      <group title="Pegasus" packages="edu.isi.pegasus.*"/>
      <link href="https://docs.oracle.com/javase/8/docs/api/"/>
    </javadoc>
  </target>

  <target name="doc-schemas" depends="setup" description="Generate schemas">
    <!-- doc schemas -->
    <mkdir dir="${dist.doc}/schemas"/>
    <copy preservelastmodified="true" todir="${dist.doc}/schemas">
      <fileset dir="doc/schemas" includes="**/*"/>
    </copy>
    <fixcrlf srcdir="${dist.doc}/schemas" includes="**/*.xsd" eol="lf"/>
    <fixcrlf srcdir="${dist.doc}/schemas" includes="**/*.xml" eol="lf"/>
    <fixcrlf srcdir="${dist.doc}/schemas" includes="**/*.yml" eol="lf"/>

    <mkdir dir="${dist.doc}/wordpress/schemas"/>
    <copy preservelastmodified="true" todir="${dist.doc}/wordpress/schemas">
      <fileset dir="doc/schemas" includes="**/*"/>
    </copy>
    <fixcrlf srcdir="${dist.doc}/wordpress/schemas" includes="**/*.xsd" eol="lf"/>
    <fixcrlf srcdir="${dist.doc}/wordpress/schemas" includes="**/*.xml" eol="lf"/>
    <fixcrlf srcdir="${dist.doc}/wordpress/schemas" includes="**/*.yml" eol="lf"/>
  </target>

  <target name="doc" depends="setup,doc-user-guide,doc-java,doc-schemas" description="Generate documentation"/>

  <target name="dist-doc" depends="doc" description="Build documentation tarball">
    <tar destfile="dist/pegasus-doc-${pegasus.version}.tar.gz"
         basedir="dist" longfile="gnu" compression="gzip"
         includes="pegasus-${pegasus.version}/share/man/**,pegasus-${pegasus.version}/share/doc/**"/>
  </target>

  <target name="dist-worker" depends="setup,compile-c,compile-python,compile-externals" description="Build worker tarball">
    <!-- bin -->
    <copy preservelastmodified="true" todir="${dist.bin}">
      <fileset dir="bin">
          <exclude name=".gitignore"/>
          <exclude name="pegasus-python-wrapper"/>
      </fileset>
    </copy>
    <chmod perm="0755" type="file">
      <fileset dir="${dist.bin}">
        <include name="*"/>
      </fileset>
    </chmod>
    <exec executable="release-tools/update-python-tools" failonerror="true">
      <arg value="${dist.bin}"/>
    </exec>
    <copy preservelastmodified="true" todir="${dist.lib}/pegasus/externals/python">
      <fileset dir="build/externals">
        <include name="boto3*/**/*"/>
        <include name="botocore*/**/*"/>
        <include name="jmespath*/**/*"/>
        <include name="dateutil*/**/*"/>
        <include name="s3transfer*/**/*"/>
        <include name="urllib3*/**/*"/>
        <include name="globus_sdk*/**/*"/>
        <include name="six"/>
        <include name="six.py"/>
      </fileset>
    </copy>
    <copy preservelastmodified="true" todir="${dist.dir}" file="LICENSE"/>
    <package-binary name="pegasus-worker-${pegasus.version}-${dist.arch}"/>
  </target>

  <target name="dist-common" depends="setup,jar,jar-aws-batch,compile-c,compile-python,compile-externals">
    <!-- bin -->
    <copy preservelastmodified="true" todir="${dist.bin}">
      <fileset dir="bin">
        <exclude name=".gitignore"/>
        <exclude name="pegasus-python-wrapper"/>
      </fileset>
    </copy>
    <chmod perm="0755" type="file">
      <fileset dir="${dist.bin}">
        <include name="*"/>
      </fileset>
    </chmod>
    <exec executable="release-tools/update-python-tools" failonerror="true">
      <arg value="${dist.bin}"/>
    </exec>

    <!-- etc -->
    <mkdir dir="${dist.etc}"/>
    <copy preservelastmodified="true" todir="${dist.etc}">
      <fileset dir="etc" includes="**/*"/>
    </copy>

    <!-- lib -->
    <copy preservelastmodified="true" todir="${dist.lib}/pegasus/externals/python">
      <fileset dir="build/externals">
        <exclude name="bin/**"/>
      </fileset>
    </copy>

    <!-- share -->
    <mkdir dir="${dist.share}"/>
    <copy preservelastmodified="true" todir="${dist.share}">
      <fileset dir="share/pegasus" includes="**/*">
        <exclude name=".gitignore"/>
      </fileset>
    </copy>
    <chmod perm="0755" type="file">
      <fileset dir="${dist.share}" includes="**/*.sh"/>
      <fileset dir="${dist.share}" includes="**/*.py"/>
    </chmod>
    <chmod perm="0755" type="file">
      <fileset dir="${dist.share}/notification" includes="*"/>
    </chmod>
    <chmod perm="0644" type="file">
      <fileset dir="${dist.jars}" includes="*.jar"/>
      <fileset dir="${dist.aws.jars}" includes="*.jar"/>
    </chmod>
    <!-- include a copy of the worker package - this is so that the planner can include it
         quickly in workflows running on the same system as the planner -->
    <mkdir dir="${dist.share}/worker-packages"/>
    <copy preservelastmodified="true" todir="${dist.share}/worker-packages">
        <fileset dir="dist" includes="pegasus-worker-${pegasus.version}-${dist.arch}.tar.gz" />
    </copy>

    <copy preservelastmodified="true" todir="${dist.dir}">
      <fileset dir="${basedir}">
        <include name="README.md"/>
        <include name="LICENSE"/>
        <include name="RELEASE_NOTES"/>
      </fileset>
    </copy>
  </target>

  <target name="dist-release" depends="dist-worker,dist-common" description="Build binary release tarball">
    <package-binary name="pegasus-binary-${pegasus.version}-${dist.arch}"/>
  </target>

  <target name="dist" depends="dist-worker,dist-common" description="Build binary release tarball without documentation">
    <package-binary name="pegasus-nodocs-${pegasus.version}-${dist.arch}"/>
  </target>

  <target name="dist-source" description="Build source tarball">
    <exec executable="release-tools/create-source-tar" failonerror="true">
      <arg line="${pegasus.version}"/>
    </exec>
  </target>

  <target name="dist-python-source" description="Build source tarball for Pegasus Python Modules">
    <replace file="packages/pegasus-common/setup.py" token="version=read_version()" value="version=&quot;${pegasus.version}&quot;" />
    <exec executable="${dist.python}" dir="packages/pegasus-common" failonerror="true">
      <arg line="setup.py sdist --dist-dir ../../dist"/>
    </exec>
    <replace file="packages/pegasus-common/setup.py" value="version=read_version()" token="version=&quot;${pegasus.version}&quot;"/>

    <replace file="packages/pegasus-api/setup.py" token="version=read_version()" value="version=&quot;${pegasus.version}&quot;"/>
    <exec executable="${dist.python}" dir="packages/pegasus-api" failonerror="true">
      <arg line="setup.py sdist --dist-dir ../../dist"/>
    </exec>
    <replace file="packages/pegasus-api/setup.py" value="version=read_version()" token="version=&quot;${pegasus.version}&quot;"/>

    <replace file="packages/pegasus-python/setup.py" token="version=read_version()" value="version=&quot;${pegasus.version}&quot;"/>
    <exec executable="${dist.python}" dir="packages/pegasus-python" failonerror="true">
      <arg line="setup.py sdist --dist-dir ../../dist"/>
    </exec>
    <replace file="packages/pegasus-python/setup.py" value="version=read_version()" token="version=&quot;${pegasus.version}&quot;"/>

    <replace file="packages/pegasus-worker/setup.py" token="version=read_version()" value="version=&quot;${pegasus.version}&quot;"/>
    <exec executable="${dist.python}" dir="packages/pegasus-worker" failonerror="true">
      <arg line="setup.py sdist --dist-dir ../../dist"/>
    </exec>
    <replace file="packages/pegasus-worker/setup.py" value="version=read_version()" token="version=&quot;${pegasus.version}&quot;"/>
  </target>

  <target name="dist-rpm" depends="dist-source,setup" description="Build RPM package">
    <exec executable="release-tools/build-rpms" failonerror="true">
      <arg line="${pegasus.version}"/>
    </exec>
  </target>

  <target name="dist-deb" depends="clean-scratch,dist-source,setup" description="Build Debian package">
    <exec executable="release-tools/build-deb" failonerror="true">
      <arg line="${pegasus.version} ${dist.osid}"/>
    </exec>
  </target>

  <target name="test-python" description="Execute python unit tests">
    <exec executable="tox" dir="packages/pegasus-common" failonerror="true">
      <arg line="--recreate"/>
    </exec>
    <exec executable="tox" dir="packages/pegasus-api" failonerror="true">
      <arg line="--recreate"/>
    </exec>
    <exec executable="tox" dir="packages/pegasus-worker" failonerror="true">
      <arg line="--recreate"/>
    </exec>
    <exec executable="tox" dir="packages/pegasus-python" failonerror="true">
      <arg line="--recreate"/>
    </exec>
  </target>

  <target name="test-java" depends="jar,jar-junit,compile-pegasus-common,compile-pegasus-python,compile-externals" description="Run java unit tests">
    <!-- Copy required YAML schemas -->
    <!-- Copied from dist-common, can be replaced by a new, shared target -->
    <mkdir dir="${dist.share}"/>
    <copy preservelastmodified="true" todir="${dist.share}">
      <fileset dir="share/pegasus" includes="**/*">
        <exclude name=".gitignore"/>
      </fileset>
    </copy>

    <!-- etc -->
    <mkdir dir="${dist.etc}"/>
    <copy preservelastmodified="true" todir="${dist.etc}">
      <fileset dir="etc" includes="**/*"/>
    </copy>

    <copy file="bin/pegasus-python-wrapper" tofile="${dist.bin}/pegasus-db-admin" preservelastmodified="true"/>
    <chmod perm="0755" type="file" file="${dist.bin}/pegasus-db-admin"/>
    <exec executable="release-tools/update-python-tools" failonerror="true">
      <arg value="${dist.bin}"/>
    </exec>

    <copy preservelastmodified="true" todir="${dist.lib}/pegasus/externals/python">
      <fileset dir="build/externals">
        <exclude name="bin/**"/>
      </fileset>
    </copy>

    <mkdir dir="${junitreport.dir}"/>
    <java classname="org.junit.platform.console.ConsoleLauncher" fork="true" failonerror="true">
      <sysproperty key="externals.python.path" value="${dist.lib}/pegasus/externals/python"/>
      <sysproperty key="pegasus.home.schemadir" value="${dist.schema}"/>
      <sysproperty key="pegasus.home.bindir" value="${dist.bin}"/>
      <sysproperty key="pegasus.home.sysconfdir" value="${dist.etc}"/>
      <sysproperty key="pegasus.home.sharedstatedir" value="${dist.share}"/>
      <classpath>
        <path refid="build.classpath"/>
        <path refid="build.aws.classpath"/>
        <path refid="java.test.classpath"/>
        <path location="${dist.jars}/pegasus.jar"/>
        <path location="${dist.jars}/pegasus-test.jar"/>
        <path location="${dist.jars}/vdl.jar"/>
      </classpath>
      <arg value="execute"/>
      <arg value="--fail-if-no-tests"/>
      <arg value="--scan-classpath=build/tests/classes"/>
      <arg line="--reports-dir ${junitreport.dir}"/>
    </java>
    <junitreport todir="${junitreport.dir}">
      <fileset dir="${junitreport.dir}">
        <include name="TEST-*.xml"/>
      </fileset>
      <report format="frames" todir="${junitreport.dir}/html"/>
    </junitreport>
  </target>

  <target name="test-kickstart" depends="compile-pegasus-kickstart,compile-pegasus-worker" description="Run kickstart unit tests">
    <copy file="bin/pegasus-python-wrapper" tofile="${dist.bin}/pegasus-integrity" preservelastmodified="true"/>
    <chmod perm="0755" type="file" file="${dist.bin}/pegasus-integrity"/>
    <exec executable="release-tools/update-python-tools" failonerror="true">
      <arg value="${dist.bin}"/>
    </exec>
    <exec executable="${make}" dir="packages/pegasus-kickstart" failonerror="true">
      <arg line="test"/>
      <env key="PEGASUS_BIN_DIR" value="${dist.bin}"/>
    </exec>
  </target>

  <target name="test-pmc" depends="compile-pegasus-mpi-cluster" description="Run PMC unit tests">
    <exec executable="${make}" dir="packages/pegasus-mpi-cluster" failonerror="true">
        <arg line="test"/>
    </exec>
  </target>

  <target name="test-c" depends="test-kickstart,test-pmc" description="Run C unit tests"/>

  <target name="test-transfer" depends="compile-pegasus-kickstart,compile-pegasus-worker" description="Run transfer unit tests">
    <copy file="bin/pegasus-python-wrapper" tofile="${dist.bin}/pegasus-transfer" preservelastmodified="true"/>
    <chmod perm="0755" type="file" file="${dist.bin}/pegasus-transfer"/>
    <exec executable="release-tools/update-python-tools" failonerror="true">
      <arg value="${dist.bin}"/>
    </exec>
    <exec executable="./test.sh" dir="test/unit/transfer" failonerror="true">
      <env key="PEGASUS_BIN_DIR" value="${dist.bin}"/>
    </exec>
  </target>

  <target name="test" depends="dist,test-python,test-java,test-c,test-transfer" description="Run all unit tests"/>

  <target name="tutorial-image-build" depends="setup" description="Builds the Docker based tutorial container image">
      <exec executable="docker" dir="tutorial/docker" failonerror="true">
          <arg line="build --no-cache --build-arg CACHEBUST=${ISODATE.utc} --build-arg PEGASUS_VERSION=${pegasus.version} -t pegasus/tutorial:${pegasus.version} ."/>
    </exec>
  </target>

  <!-- This target should clean everything _except_ the dist directory
       as we need it in multistep builds in bamboo -->
  <target name="dist-clean">
    <delete dir="${dist.dir}"/>
    <delete dir="build"/>
    <exec executable="${make}" dir="packages/pegasus-cluster" failonerror="false">
      <arg line="distclean"/>
    </exec>
    <exec executable="${make}" dir="packages/pegasus-keg" failonerror="false">
      <arg line="distclean"/>
    </exec>
    <exec executable="${make}" dir="packages/pegasus-kickstart" failonerror="false">
      <arg line="distclean"/>
    </exec>
    <exec executable="${make}" dir="packages/pegasus-mpi-cluster" failonerror="false">
      <arg line="distclean"/>
    </exec>
    <exec executable="${make}" dir="doc/sphinx" failonerror="false">
      <arg line="clean"/>
    </exec>
    <exec executable="${dist.python}" dir="packages/pegasus-common" failonerror="false">
      <arg line="setup.py clean --all"/>
    </exec>
    <exec executable="${dist.python}" dir="packages/pegasus-api" failonerror="false">
      <arg line="setup.py clean --all"/>
    </exec>
    <exec executable="${dist.python}" dir="packages/pegasus-python" failonerror="false">
      <arg line="setup.py clean --all"/>
    </exec>
    <exec executable="${dist.python}" dir="packages/pegasus-worker" failonerror="false">
      <arg line="setup.py clean --all"/>
    </exec>

    <delete dir="packages/pegasus-common/.tox" followsymlinks="false" removenotfollowedsymlinks="true"/>
    <delete dir="packages/pegasus-common/.cache"/>
    <delete dir="packages/pegasus-common/.coverage"/>
    <delete dir="packages/pegasus-common/.pytest_cache"/>
    <delete dir="packages/pegasus-common/test-reports"/>
    <delete dir="packages/pegasus-common/htmlcov"/>
    <delete>
      <fileset dir="packages/pegasus-common" includes="**/*.pyc"/>
    </delete>
    <delete>
      <dirset dir="packages/pegasus-common" includes="**/__pycache__"/>
    </delete>

    <delete dir="packages/pegasus-api/.tox" followsymlinks="false" removenotfollowedsymlinks="true"/>
    <delete dir="packages/pegasus-api/.cache"/>
    <delete dir="packages/pegasus-api/.coverage"/>
    <delete dir="packages/pegasus-api/.pytest_cache"/>
    <delete dir="packages/pegasus-api/test-reports"/>
    <delete dir="packages/pegasus-api/htmlcov"/>
    <delete>
      <fileset dir="packages/pegasus-api" includes="**/*.pyc"/>
    </delete>
    <delete>
      <dirset dir="packages/pegasus-api" includes="**/__pycache__"/>
    </delete>

    <delete dir="packages/pegasus-worker/.tox" followsymlinks="false" removenotfollowedsymlinks="true"/>
    <delete dir="packages/pegasus-worker/.cache"/>
    <delete dir="packages/pegasus-worker/.coverage"/>
    <delete dir="packages/pegasus-worker/.pytest_cache"/>
    <delete dir="packages/pegasus-worker/test-reports"/>
    <delete dir="packages/pegasus-worker/htmlcov"/>
    <delete>
      <fileset dir="packages/pegasus-worker" includes="**/*.pyc"/>
    </delete>
    <delete>
      <dirset dir="packages/pegasus-worker" includes="**/__pycache__"/>
    </delete>

    <delete dir="packages/pegasus-python/.tox" followsymlinks="false" removenotfollowedsymlinks="true"/>
    <delete dir="packages/pegasus-python/.cache"/>
    <delete dir="packages/pegasus-python/.coverage"/>
    <delete dir="packages/pegasus-python/.pytest_cache"/>
    <delete dir="packages/pegasus-python/test-reports"/>
    <delete dir="packages/pegasus-python/htmlcov"/>
    <delete>
      <fileset dir="packages/pegasus-python" includes="**/*.pyc"/>
    </delete>
    <delete>
      <dirset dir="packages/pegasus-python" includes="**/__pycache__"/>
    </delete>

    <!-- Clean auto-generated doc -->
    <delete dir="doc/sphinx/python/"/>
    <delete dir="doc/sphinx/java/"/>
    <delete dir="doc/sphinx/_build/"/>
  </target>

  <target name="clean-scratch">
    <delete dir="${dist.dir}"/>
  </target>

  <!-- really clean everything -->
  <target name="clean" depends="dist-clean" description="Delete all build artifacts">
    <delete dir="dist"/>
  </target>
</project>