build_categories.php

#!/usr/bin/php
<?php

// this is meant to be used from the command line, not a webpage, but
//  it's written in PHP so we can reuse config.php and stuff.

chdir(dirname(__FILE__));  // just in case.
require_once('config.php');

// !!! FIXME: put this in a common.php file or something.
$supported_formats = [
    'md' => 'gfm',
    'mediawiki' => 'mediawiki'
];

$editwarning = '<!-- DO NOT HAND-EDIT CATEGORY LISTS, THEY ARE AUTOGENERATED AND WILL BE OVERWRITTEN, BASED ON TAGS IN INDIVIDUAL PAGE FOOTERS. EDIT THOSE INSTEAD. -->';

$escrawdata = escapeshellarg($raw_data);

putenv("GIT_SSH_COMMAND=ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -q -i $ssh_key_fname");
putenv("GIT_COMMITTER_NAME=$git_committer_name");
putenv("GIT_COMMITTER_EMAIL=$git_committer_email");
putenv("GIT_AUTHOR_NAME=$git_committer_name");
putenv("GIT_AUTHOR_EMAIL=$git_committer_email");


function file_contents_match($path1, $path2)
{
    if ($path1 == $path2) {
        return true;
    } else if (filesize($path1) != filesize($path2)) {
        return false;
    }

    $fh1 = fopen($path1, 'rb');
    if ($fh1 === false) {
        return false;
    }

    $fh2 = fopen($path2, 'rb');
    if ($fh2 === false) {
        fclose($fh1);
        return false;
    }

    $size = 1024 * 32;  // compare 32k at a time.
    $retval = true;
    while (!feof($fh1)) {
        if (fread($fh1, $size) != fread($fh2, $size)) {
            $retval = false;
            break;
        }
    }

    fclose($fh1);
    fclose($fh2);

    return $retval;
}

// overwrite an existing file with a temp file only if they've changed, so the timestamp doesn't update if we just made the same file.
// this prevents the page recooker from doing unnecessary heavy lifting.
// This will delete the temp file if it doesn't rename it, even in failure.
function replace_with_tempfile($tmppath, $origpath)
{
    $retval = true;
    if (file_contents_match($tmppath, $origpath)) {
        unlink($tmppath);  // wasn't needed!
    } else {
        $retval = rename($tmppath, $origpath);
        if (!$retval) {
            unlink($tmppath);  // oh well.
        }
    }
    return $retval;
}

$categories = array();
function build_category_lists($srcdir)
{
    global $supported_formats, $categories;

    $dirp = opendir($srcdir);
    if ($dirp === false) {
        print("Failed to opendir '$srcdir'\n");
        exit(1);
    }

    while (($dent = readdir($dirp)) !== false) {
        if (substr($dent, 0, 1) == '.') { continue; }  // skip ".", "..", and metadata.
        $src = "$srcdir/$dent";
        if (is_dir($src)) {
            // categories are per-subdir, don't walk the tree.
            continue;  //build_category_lists($src);
        } else {
            $ext = strrchr($dent, '.');

            if ($ext === false) {
                continue;
            }

            $ext = substr($ext, 1);
            //print("cookext: '$ext'\n");

            if (!isset($supported_formats[$ext])) {
                continue;
            }

            $from_format = $supported_formats[$ext];
            $page = preg_replace('/^.*\//', '', $dent);
            $page = preg_replace('/\..*$/', '', $page);
            $fp = fopen($src, 'r');
            if (!$fp) {
                print("Failed to open '$src'!\n");
                exit(1);
            }

            $pending_categories = NULL;

            while (($line = fgets($fp)) !== false) {
                // The categories come right after a '----' line and must be the final nonblank line of the file.

                // Strictly speaking, it's legal in Markdown to do:
                //
                // My sub-header
                // -------------
                //
                // (which is equivalent to "# My sub-header" on a single line.)
                // ...so this can catch nonsense if the sub-header is four chars long,
                // and at the end of the file, but we attempt to mitigate.

                $line = trim($line);
                if ($line == '----') {  # !!! FIXME: strictly speaking, this can fail if you have two `----` lines in a row at the end of the file...but, like...don't do that.
                    if (($line = fgets($fp)) !== false) {
                        $pending_categories = array();
                        $cats = explode(',', trim($line));
                        foreach ($cats as $c) {
                            $c = trim($c);
                            $count = 0;
                            if ($from_format == "mediawiki") {
                                $c = preg_replace('/^\[\[(.*?)\]\]$/', '$1', $c, 1, $count);
                            } else if ($from_format == "gfm") {
                                $c = preg_replace('/^\[(.*?)\]\(.*?\)$/', '$1', $c, 1, $count);
                            }
                            // currently we have pages that don't have these wikilinked, so don't check $count==1 here for now.
                            if (/*($count == 1) &&*/ ($c != "")) {
                                $pending_categories[] = $c;
                            }
                        }
                    }
                } else if ($line != '') {
                    $pending_categories = NULL;  // wasn't the end of the file, don't treat this as a category footer thing.
                }
            }
            fclose($fp);

            if ($pending_categories != NULL) {
                foreach ($pending_categories as $c) {
                    if (!isset($categories[$c])) {
                        $categories[$c] = array();
                    }
                    //print("Adding '$page' to '$c'\n");
                    $categories[$c][$page] = true;
                }
            }
        }
    }
    closedir($dirp);
}

function write_category_list($fp, $cats, $singletopic, $ismediawiki)
{
    global $editwarning, $categories;

    fputs($fp, "$editwarning\n");

    $pages = NULL;

    if ($singletopic) {
        fputs($fp, "<!-- BEGIN CATEGORY LIST -->\n");
        $pages = $categories[$cats];
    } else {
        fputs($fp, "<!-- BEGIN CATEGORY LIST: $cats -->\n");
        $pages = array();
        $splitcats = explode(',', $cats);
        if ((count($splitcats) > 0) && isset($categories[trim($splitcats[0])])) {
            $include_this = true;
            foreach ($categories[trim($splitcats[0])] as $page => $unused) {
                $include_this = true;
                $page = trim($page);
                foreach ($splitcats as $cat) {
                    $cat = trim($cat);
                    if (!isset($categories[$cat][$page])) {
                        $include_this = false;
                        break;
                    }
                }

                if ($include_this) {
                    $pages[$page] = true;
                }
            }
        }
    }

    ksort($pages, SORT_STRING|SORT_FLAG_CASE);

    $at_least_one = false;
    //$last_letter = '';
    foreach ($pages as $p => $unused) {
        // this isn't super-useful when everything starts with "SDL_"  :)
        /*
        $letter = strtoupper(substr($p, 0, 1));
        if ($letter != $last_letter) {
            $last_letter = $letter;
            fputs($fp, "* $letter\n");
        }
        */
        if ($ismediawiki) {
            fputs($fp, "* [[$p]]\n");
        } else {
            fputs($fp, "- [$p]($p)\n");
        }
        $at_least_one = true;
    }

    if (!$at_least_one) {
        if ($ismediawiki) {
            fputs($fp, "* (none.)\n");
        } else {
            fputs($fp, "- (none.)\n");
        }
    }

    fputs($fp, "<!-- END CATEGORY LIST -->\n");
}

function find_subdirs($base, $dname, &$output)
{
    $dirp = opendir($dname);
    if ($dirp === false) {
        return;  // oh well.
    }

    $sep = ($base == NULL) ? '' : '/';

    while (($dent = readdir($dirp)) !== false) {
        $path = "$dname/$dent";
        if (substr($dent, 0, 1) == '.') {
            continue;  // skip ".", "..", and metadata.
        } else if (is_dir($path)) {
            $thisbase = "$base$sep$dent";
            $output[] = $thisbase;
            find_subdirs($thisbase, $path, $output);
        }
    }

    closedir($dirp);

    return $output;
}

function handle_subdir($dname)
{
    global $categories;
    global $editwarning;

    $categories = array();
    build_category_lists($dname);

    foreach ($categories as $cat => $pages) {
        //print("DIR '$dname' CATEGORY '$cat':\n");
        //print_r($pages);

        // keep in MediaWiki format if it exists, start new pages in Markdown.
        $ismediawiki = true;
        $path = "$dname/$cat.mediawiki";
        if (!file_exists($path)) {
            $ismediawiki = false;
            $path = "$dname/$cat.md";
            if (!file_exists($path)) {
                file_put_contents($path, "# $cat\n\n$editwarning\n<!-- BEGIN CATEGORY LIST -->\n<!-- END CATEGORY LIST -->\n\n");
                //file_put_contents($path, "# $cat\n\n## Functions\n\n<!-- BEGIN CATEGORY LIST: $cat, CategoryAPIFunction -->\n<!-- END CATEGORY LIST -->\n\n## Datatypes\n\n<!-- BEGIN CATEGORY LIST: $cat, CategoryAPIDatatype -->\n<!-- END CATEGORY LIST -->\n\n## Structs\n\n<!-- BEGIN CATEGORY LIST: $cat, CategoryAPIStruct -->\n<!-- END CATEGORY LIST -->\n\n## Enums\n\n<!-- BEGIN CATEGORY LIST: $cat, CategoryAPIEnum -->\n<!-- END CATEGORY LIST -->\n\n## Macros\n\n<!-- BEGIN CATEGORY LIST: $cat, CategoryAPIMacro -->\n<!-- END CATEGORY LIST -->\n\n");
            }
        }

        $tmppath = "$path.tmp";
        $contents = '';

        $in = fopen($path, "r");
        if ($in === false) {
            print("Failed to open '$path' for reading\n");
            system("cd $escrawdata && git clean -dfq && git checkout -- .");
            exit(1);
        }

        $out = fopen($tmppath, "w");
        if ($out === false) {
            print("Failed to open '$tmppath' for writing\n");
            system("cd $escrawdata && git clean -dfq && git checkout -- .");
            exit(1);
        }

        $wrote_list = false;
        while (($line = fgets($in)) !== false) {
            //print("LINE: [" . trim($line) . "]\n");
            $trimmed = trim($line);
            if ($trimmed == $editwarning) {
                // skip these lines, we'll write them out when we output the actual list.
            } else if ($trimmed == '----') {  // the footer? Just stuff the list before it, oh well.
                if (!$wrote_list) {
                    write_category_list($out, $cat, true, $ismediawiki);
                    $wrote_list = true;
                }
                fputs($out, "----\n");
            } else if ($trimmed == '<!-- BEGIN CATEGORY LIST -->') {  // doesn't list a specific category, do the one the page is named for.
                if (!$wrote_list) {
                    write_category_list($out, $cat, true, $ismediawiki);
                    $wrote_list = true;
                }
                while (($line = fgets($in)) !== false) {
                    if (trim($line) == '<!-- END CATEGORY LIST -->') {
                        break;
                    }
                }
            } else if (preg_match('/^\<\!\-\- BEGIN CATEGORY LIST: (.*?) \-\-\>$/', $trimmed, $cats) === 1) {  // lists a specific set of subcategories
                write_category_list($out, $cats[1], false, $ismediawiki);
                $wrote_list = true;  // if this is here, they probably don't want a global list.
                while (($line = fgets($in)) !== false) {
                    if (trim($line) == '<!-- END CATEGORY LIST -->') {
                        break;
                    }
                }
            } else {
                fputs($out, $line);
            }
        }

        fclose($in);

        if (!$wrote_list) {
            write_category_list($out, $cat, true, $ismediawiki);
        }

        fclose($out);

        if (!replace_with_tempfile($tmppath, $path)) {
            print("Failed to rename '$tmppath' to '$path'!\n");
            system("cd $escrawdata && git clean -dfq && git checkout -- .");
            exit(1);
        }
    }
}


// Mainline!

$git_repo_lock_fp = fopen($repo_lock_fname, 'c+');
if ($git_repo_lock_fp === false) {
    print("Failed to obtain Git repo lock. Please try again later.\n");
    exit(1);
} else if (flock($git_repo_lock_fp, LOCK_EX) === false) {
    print("Exclusive flock of Git repo lock failed. Please try again later.");  // uh...?
    exit(1);
}

handle_subdir($raw_data);  // get the root directory.

$subdirs = array();
find_subdirs(NULL, $raw_data, $subdirs);
foreach ($subdirs as $d) {
    handle_subdir("$raw_data/$d");
}

unset($output);
$failed = ((exec("cd $escrawdata && git status -s |wc -l", $output, $rc) === false) || ($rc != 0));
if ($failed) {
    print("Failed to run 'git status'!\n");
    system("cd $escrawdata && git clean -dfq && git checkout -- .");
    exit(1);
}

$changes = (isset($output[0]) && $output[0] != '0');
if ($changes) {
    $cmd = "( cd $escrawdata && git add -A && git commit -m 'Sync category pages' && git push ) 2>&1";
    unset($output);
    $failed = (exec($cmd, $output, $result) === false) || ($result != 0);
    if ($failed) {
        print("FAILED GIT RUN!\n\n    cmd:\n'$cmd'\n\noutput:\n");
        foreach ($output as $l) {
            print("    $l\n");
        }
        exit(1);
    }
}

flock($git_repo_lock_fp, LOCK_UN);
fclose($git_repo_lock_fp);
unset($git_repo_lock_fp);

exit(0);
?>