update.py

#! python

import re
import urllib2
import argparse
import os
import sys
import shutil
import json
import time
import getpass
import platform
from sys import stdin
import ssl

ADDR = "https://bb.video-stitch.com:8010"
DIRECTORY = os.path.dirname(os.path.realpath(__file__))
DEPS_FOLDER = os.path.join(DIRECTORY, "external_deps")
LIB_FOLDER = os.path.join(DEPS_FOLDER, "lib")
LIB64_FOLDER = os.path.join(DEPS_FOLDER, "lib64")
PASS_PATH = os.path.abspath(os.path.join(DIRECTORY, '..', 'pass.py'))
def os_name():
    if platform.system() == "Windows" or\
            "cygwin" in platform.system().lower():
        return "windows"
    elif platform.system() == "Darwin":
        return "mac"
    else:
        return "linux"

def target_name():
    if os_name() != "linux":
        return os_name()
    else:
        return ARGS.target

def stdin_readline_without_cr():
    return sys.stdin.readline()[:-len(os.linesep)]

def ask_for_credentials():
    global LOGIN
    global PASS
    print("Enter your buildbot login : ")
    LOGIN = stdin_readline_without_cr()
    PASS = getpass.getpass()

def ask_boolean(prompt):
    boolstr = ""
    while(boolstr not in ["yes","no"]):
        print(prompt + " (yes or no)")
        boolstr = stdin_readline_without_cr()
        if (boolstr == ""):
            boolstr = "no"
    return boolstr == "yes"

REPOS = ["deps", "visualizer-deps", "visualizer-data"]

PARSER = argparse.ArgumentParser(
    description="Retrieve deps for the application you want to build")
PARSER.add_argument(
    "repo",
    help="Repo you want to pull (" + ",".join(REPOS + ["all)"]),
    )
PARSER.add_argument(
    "-v",
    "--verbose",
    action="store_true",
    help="Verbose mode",
    )

TARGETS = ["linux", "armhf", "aarch64", "android-arm"]
PARSER.add_argument(
    "--target", default="linux",
    help="target platform in case of cross compilation on linux (" + ",".join(TARGETS) + ")",
    )

ARGS = PARSER.parse_args()

def get_credentials():
    if not "LOGIN" in globals() or not "PASS" in globals():
        global LOGIN
        global PASS
        try:
            with open(PASS_PATH,"r") as f:
                line=f.readline()
                while(line):
                    for var in ["LOGIN", "PASS", "PROMPT_LOGIN"]:
                        match=re.match(var + "=\"(.*)\"", line)
                        if match:
                            globals()[var]=match.group(1)
                    line=f.readline()
        except IOError:
            print("File pass.py not found.")
            print("The pass.py file is an helper file where your "
                  "buildbot credentials are stored")
            print("NB: for now those informations are stored clear, "
                  "so anyone can have your password from it")
            print("This script can create it for you right now, if "
                  "you don't want to you will not be prompt anymore")
            print("You can still add it later manually")
            create_pass_py = ask_boolean("Do you want to create the file?")
            ask_for_credentials()
            with open(PASS_PATH,"w") as f:
                if create_pass_py:
                    f.write("LOGIN=\"" + LOGIN + "\"\n")
                    f.write("PASS=\"" + PASS + "\"")
                else:
                    f.write("PROMPT_LOGIN=\"True\"")
        try:
            if (PROMPT_LOGIN):
                if stdin.isatty():
                    ask_for_credentials()
        except Exception:
            pass

BB_DL_URL = "/".join([ADDR, "downloads", target_name()])

def get_default_url(repo_name):
    return "/".join([BB_DL_URL, repo_name])

class DownloadableItem(object):
    def __init__(self, typeh, name, modiftime):
        self.type = typeh
        self.name = name
        self.modiftime = modiftime

def make_auth(url):
    get_credentials()
    passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
    passman.add_password(None, ADDR + "/downloads", LOGIN, PASS)
    authhandler = urllib2.HTTPBasicAuthHandler(passman)
    opener = urllib2.build_opener(authhandler)
    urllib2.install_opener(opener)

def open_url(url):
    try:
        make_auth(url)
        if hasattr(ssl, '_create_unverified_context'):
            ssl._create_default_https_context = ssl._create_unverified_context
        urlret = urllib2.urlopen(url, timeout=4)
    except urllib2.HTTPError as err:
        if err.code == 401:
            raise Exception("The login and password you provided "
                            "are not correct")
        else:
            raise
    except urllib2.URLError:
        try:
            urlret = urllib2.urlopen(url)
        except urllib2.URLError:
            raise Exception("An error occured : looks like the "
                            "buildbot is unreachable.")
    return urlret.read()

def download_file(url, file_name="", verbose=False, **kwargs):
    if file_name == "":
        file_name = url.split("/")[-1]
    if verbose:
        print("Starting download of " + file_name)
    file_content = open_url(url)
    if os.path.dirname(file_name) and\
            not os.path.exists(os.path.dirname(file_name)):
        os.makedirs(os.path.dirname(file_name))
    with open(file_name,"wb") as f:
        f.write(file_content)
    os.chmod(file_name, 0o755)
    if verbose:
        print(file_name + " downloaded successfully")

def get_package_url(repo_name, package):
    return "/".join([get_default_url(repo_name), package["name"],
                     package["version"]])

def get_file_and_dir_list(html):
    dilist = []
    for line in html.split("\n"):
        # TODO: rewrite this shit
        #date_pattern = "[0-9]{2}-[A-Z][a-z]{2}-[0-9]{4}"
        date_pattern = "[0-9]{4}-[0-9]{2}-[0-9]{2}"
        match = re.match(".*<a href=\"(.*)\">\\1</a>\\s+(" + date_pattern +\
            " [0-9]{2}:[0-9]{2}).*", line)
        if match:
            item_name = match.group(1)
            #modiftime = time.strptime(match.group(2),"%d-%b-%Y %H:%M")
            modiftime = time.strptime(match.group(2),"%Y-%m-%d %H:%M")
            if (item_name[-1] == "/"):
                t = "dir"
                item_name = item_name[:-1]
            else:
                t = "file"
            dilist.append(DownloadableItem(t,item_name,modiftime))
    return dilist

def string_matches_array(string, arr):
    for s in arr:
        if s in string:
            return True
    return False

def download_dir(url, dest_folder, verbose):
    if dest_folder == "":
        dest_folder = url.split("/")[-1]
    try:
        os.makedirs(dest_folder)
    except Exception:
        pass
    if verbose:
        print("Downloading " + url + " to " + dest_folder)
    try:
        html = open_url(url)
    except TypeError:
        html = open_url(url)
    except Exception as e:
        print("A problem occured while opening " + url)
        print(e)
        html = ""
    dlist = get_file_and_dir_list(html)
    # Download file and dir list
    for ditem in dlist:
        dest = "/".join([dest_folder,ditem.name])
        if ditem.type == "file":
            # Ignore if file hasn't changed
            if os.path.exists(dest):
                if os.stat(dest).st_mtime > time.mktime(ditem.modiftime):
                    continue
        # Call the right download function (file or dir)
        globals()["_".join(["download", ditem.type])]("/".join(
            [url, ditem.name]), dest, verbose)

def download_package(repo_name, package, verbose):
    print("Downloading package " + package["name"] + " from repo " +\
            repo_name)
    url = get_package_url(repo_name, package)
    open_url(url)
    not_static = not "static" in package or not package["static"]
    # on linux put the .so in external_deps/lib, on windows/mac in the build
    # directory
    if os_name() is "linux":
        if not_static:
            download_dir("/".join([url, "bin"]), LIB_FOLDER, verbose)
        download_dir("/".join([url, "lib"]), LIB_FOLDER, verbose)
        if target_name() != "linux":
            download_dir("/".join([url, "lib64"]), LIB64_FOLDER, verbose)
    else:
        if not_static:
            download_dir("/".join([url, "bin"]), "", verbose)
        download_dir("/".join([url, "lib"]), os.path.join(
            DEPS_FOLDER, "lib", package["name"]), verbose)
    download_dir("/".join([url,"include"]), os.path.join(
        DEPS_FOLDER, "include", package["name"]), verbose)
    print("Download of " + package["name"] + " complete.\n")

def download_visualizer_package(repo_name, package, verbose):
    print("Downloading package " + package["name"] + " from repo " +\
            repo_name)
    url = get_package_url(repo_name, package)
    open_url(url)
    if not "static" in package or not package["static"]:
        # on linux put the .so in external_deps/lib, on windows/mac in the
        # build directory
        if os_name() is "linux":
            download_dir("/".join([url,"bin"]), LIB_FOLDER, verbose)
        else:
            download_dir("/".join([url,"bin"]), "", verbose)
    download_dir("/".join([url,"lib"]), os.path.join(
        DEPS_FOLDER, "lib", package["name"]), verbose)
    download_dir("/".join([url,"include"]), os.path.join(
        DEPS_FOLDER, "include", package["name"]), verbose)
    print("Download of " + package["name"] + " complete.\n")

def download_visualizer_data(repo_name, package, verbose):
    print("Downloading package " + package["name"] + " from repo " +\
            repo_name)
    url = get_package_url(repo_name, package)
    open_url(url)
    download_dir(url, os.path.join("apps", "src",
                                   "videostitch-visualizer-projects",
                                   package["name"]), verbose)
    print("Download of " + package["name"] + " complete.\n")

def update_visualizer_deps(verbose):
    with open(os.path.join(DIRECTORY, "visualizer-deps.json"),"r") as f:
        deps = json.loads(f.read())
    if (deps["repo"] != "deps"):
        print("Looks like virualizer-deps for the repo " + deps["repo"] +\
                " are not yet implemented.")
    for package in deps["packages"]:
        try:
            download_visualizer_package(deps["repo"], package, verbose)
        except Exception as e:
            print("A problem occured while downloading " + package["name"] +\
                    " version " + package["version"])
            print(e)

def update_visualizer_data(verbose):
    with open(os.path.join(DIRECTORY, "visualizer-data.json"),"r") as f:
        deps = json.loads(f.read())
    if deps["repo"] != "deps":
        print("Looks like virualizer-deps for the repo " + deps["repo"] +\
                " are not yet implemented.")
    for package in deps["packages"]:
        try:
            download_visualizer_data(deps["repo"], package, verbose)
        except Exception as e:
            print("A problem occured while downloading " + package["name"] +\
                    " version " + package["version"])
            print(e)

def update_deps(verbose):
    try:
        shutil.rmtree(DEPS_FOLDER)
    except OSError:
        pass
    with open(os.path.join(DIRECTORY, "deps.json"),"r") as f:
        deps = json.loads(f.read())
    if (deps["repo"] != "deps"):
        print("Looks like deps for the repo " + deps["repo"] +\
                " are not yet implemented.")
    for package in deps["packages"]:
        try:
            download_package(deps["repo"], package, verbose)
        except TypeError:
            download_package(deps["repo"], package, verbose)
        except Exception as e:
            print("A problem occured while downloading " + package["name"] +\
                    " version " + package["version"])
            print(e)

if ARGS.target not in TARGETS:
    print("--target " + ARGS.target + " is an invalid target option.")
    print("target should be one of : " + ", ".join(TARGETS) + ".")
    exit(1)

if ARGS.repo in REPOS:
    REPOS = [ARGS.repo]
elif ARGS.repo != "all":
    print("Repo " + ARGS.repo + " is an invalid repo option.")
    print("Repo should be one of : " + ", ".join(["all"] + REPOS) + ".")
    exit(1)

for repo in REPOS:
    if repo == "deps":
        update_deps(ARGS.verbose)
    elif repo == "visualizer-deps":
        update_visualizer_deps(ARGS.verbose)
    elif repo == "visualizer-data":
        update_visualizer_data(ARGS.verbose)
    else:
        raise Exception("unknown command")