sage-download-file

#!/usr/bin/env bash

# Usage: sage-download-file URL
# Download URL to standard output, using either wget, curl, Python's
# urllib or a custom program $URL_GRABBER.
#
# AUTHORS:
#
# - William Stein: original sage-download_package Python script
#
# - Leif Leonhardy, Jeroen Demeyer (Trac #13373): use wget, curl or Python
#
#*****************************************************************************
#  Distributed under the terms of the GNU General Public License (GPL)
#  as published by the Free Software Foundation; either version 2 of
#  the License, or (at your option) any later version.
#                  http://www.gnu.org/licenses/
#*****************************************************************************

if [ $# -ne 1 ]; then
cat >&2 <<EOF
Usage: $0 URL

Download URL to standard output, using either wget, curl, Python's
urllib or a custom program $URL_GRABBER.
EOF
exit 2
fi

if [ -z "$URL_GRABBER" ]; then
    # Prefer Sage's Python because that's the same on all systems.
    # It should always work.
    if [ -n "$SAGE_LOCAL" ] && [ -x "$SAGE_LOCAL/bin/python" ]; then
        URL_GRABBER="$SAGE_LOCAL/bin/python"
    elif command -v curl &>/dev/null; then
        URL_GRABBER="curl --fail"
    elif command -v wget &>/dev/null; then
        URL_GRABBER="wget -nv -O-"
    # Pick Python last because we don't know which version it is,
    # so it's a bit risky.
    elif command -v python &>/dev/null; then
        URL_GRABBER="python"
    else
        echo >&2 "Error: $0: No program to fetch files from the web found."
        echo >&2 "       Either install Wget, curl or Python, or set the environment"
        echo >&2 "       variable URL_GRABBER to use some alternate, installed program."
        echo >&2 "       In the latter case, add options to download the contents of the"
        echo >&2 "       specified URL to the standard output. URL_GRABBER can also be"
        echo >&2 "       a Python interpreter."
        exit 3
    fi
fi


# Is $URL_GRABBER a Python interpreter?
if ! ( $URL_GRABBER --version </dev/null 2>&1 | grep '^Python ' ) &>/dev/null ; then
    # Not Python, simply execute it
    exec $URL_GRABBER "$1"
fi


# Use Python
exec $URL_GRABBER <<EOF

##############################################
# Python script to download a file to stdout #
##############################################

import sys

try:
    # Python 3
    import urllib.request as urllib
except ImportError:
    import urllib

# Change the URLopener to raise an exception on HTTP errors
def http_error_default(url, fp, errcode, errmsg, headers):
    fp.close()
    raise IOError(errcode, errmsg, url)

opener = urllib.FancyURLopener()
opener.http_error_default = http_error_default

# This reporthook is used below to print ...'s as the file downloads.
global cur
cur = 0
def reporthook(block, size, total):
    global cur
    n = 60*block*size
    # n ranges from 0 to 60*total, so we'll print at most 60 dots
    if n >= cur + total:
        sys.stderr.write('.')
        sys.stderr.flush()
        cur += total
sys.stderr.write('[')
sys.stderr.flush()
filename, info = opener.retrieve('''$1''', '/dev/stdout', reporthook)
sys.stderr.write(']\n')
EOF