"""Command line option parsing"""
import argparse
import logging
import os.path
import sys
from . import job, util, version
class ConfigAction(argparse.Action):
"""Set argparse results as config values"""
def __call__(self, parser, namespace, values, option_string=None):
namespace.options.append(((), self.dest, values))
class ConfigConstAction(argparse.Action):
"""Set argparse const values as config values"""
def __call__(self, parser, namespace, values, option_string=None):
namespace.options.append(((), self.dest, self.const))
class AppendCommandAction(argparse.Action):
def __call__(self, parser, namespace, values, option_string=None):
items = getattr(namespace, self.dest, None) or []
val = self.const.copy()
val["command"] = values
items.append(val)
setattr(namespace, self.dest, items)
class DeprecatedConfigConstAction(argparse.Action):
"""Set argparse const values as config values + deprecation warning"""
def __call__(self, parser, namespace, values, option_string=None):
sys.stderr.write(
f"Warning: {'/'.join(self.option_strings)} is deprecated. "
f"Use {self.choices} instead.\n")
namespace.options.append(((), self.dest, self.const))
class ConfigParseAction(argparse.Action):
"""Parse KEY=VALUE config options"""
def __call__(self, parser, namespace, values, option_string=None):
key, value = _parse_option(values)
key = key.split(".")
namespace.options.append((key[:-1], key[-1], value))
class PPParseAction(argparse.Action):
"""Parse KEY=VALUE post processor options"""
def __call__(self, parser, namespace, values, option_string=None):
key, value = _parse_option(values)
namespace.options_pp[key] = value
class InputfileAction(argparse.Action):
"""Collect input files"""
def __call__(self, parser, namespace, value, option_string=None):
namespace.input_files.append((value, self.const))
class MtimeAction(argparse.Action):
"""Configure mtime post processors"""
def __call__(self, parser, namespace, value, option_string=None):
namespace.postprocessors.append({
"name": "mtime",
"value": f"{{{self.const or value}}}",
})
class RenameAction(argparse.Action):
"""Configure rename post processors"""
def __call__(self, parser, namespace, value, option_string=None):
if self.const:
namespace.postprocessors.append({
"name": "rename",
"to" : value,
})
else:
namespace.postprocessors.append({
"name": "rename",
"from": value,
})
class UgoiraAction(argparse.Action):
"""Configure ugoira post processors"""
def __call__(self, parser, namespace, value, option_string=None):
if self.const:
value = self.const
else:
value = value.strip().lower()
if value in ("webm", "vp9"):
pp = {
"extension" : "webm",
"ffmpeg-args" : ("-c:v", "libvpx-vp9",
"-crf", "12",
"-b:v", "0", "-an"),
}
elif value == "vp9-lossless":
pp = {
"extension" : "webm",
"ffmpeg-args" : ("-c:v", "libvpx-vp9",
"-lossless", "1",
"-pix_fmt", "yuv420p", "-an"),
}
elif value == "vp8":
pp = {
"extension" : "webm",
"ffmpeg-args" : ("-c:v", "libvpx",
"-crf", "4",
"-b:v", "5000k", "-an"),
}
elif value == "mp4":
pp = {
"extension" : "mp4",
"ffmpeg-args" : ("-c:v", "libx264", "-an", "-b:v", "5M"),
"libx264-prevent-odd": True,
}
elif value == "gif":
pp = {
"extension" : "gif",
"ffmpeg-args" : ("-filter_complex", "[0:v] split [a][b];"
"[a] palettegen [p];[b][p] paletteuse"),
"repeat-last-frame": False,
}
elif value == "mkv" or value == "copy":
pp = {
"extension" : "mkv",
"ffmpeg-args" : ("-c:v", "copy"),
"repeat-last-frame": False,
}
elif value == "zip" or value == "archive":
pp = {
"mode" : "archive",
}
namespace.options.append(((), "ugoira", "original"))
else:
parser.error(f"Unsupported Ugoira format '{value}'")
pp["name"] = "ugoira"
pp["whitelist"] = ("pixiv", "danbooru")
namespace.options.append((("extractor",), "ugoira", True))
namespace.postprocessors.append(pp)
class PrintAction(argparse.Action):
def __call__(self, parser, namespace, value, option_string=None):
if self.const:
if self.const == "-":
namespace.options.append(((), "skip", False))
namespace.options.append(((), "download", False))
namespace.options.append((("output",), "mode", False))
filename = "-"
base = None
mode = "w"
else:
if self.const is None:
namespace.options.append(((), "skip", False))
namespace.options.append(((), "download", False))
value, path = value
base, filename = os.path.split(path)
mode = "a"
event, sep, format_string = value.partition(":")
if not sep:
format_string = event
event = ("prepare",)
else:
event = event.strip().lower()
if event not in {"init", "file", "after", "skip", "error",
"prepare", "prepare-after", "post", "post-after",
"finalize", "finalize-success", "finalize-error"}:
format_string = value
event = ("prepare",)
if not format_string:
return
if format_string.startswith("\\f"):
format_string = "\f" + format_string[2:]
if format_string[0] == "\f":
if format_string[1] == "F" and format_string[-1] != "\n":
format_string += "\n"
elif "{" not in format_string and " " not in format_string:
format_string = f"{{{format_string}}}\n"
elif format_string[-1] != "\n":
format_string += "\n"
namespace.postprocessors.append({
"name" : "metadata",
"event" : event,
"filename" : filename,
"base-directory": base or ".",
"content-format": format_string,
"open" : mode,
})
class Formatter(argparse.HelpFormatter):
"""Custom HelpFormatter class to customize help output"""
def __init__(self, prog):
argparse.HelpFormatter.__init__(self, prog, max_help_position=30)
def _format_action_invocation(self, action):
opts = action.option_strings
if action.metavar:
opts = opts.copy()
opts[-1] = f"{opts[-1]} {action.metavar}"
return ", ".join(opts)
def _format_usage(self, usage, actions, groups, prefix):
return f"Usage: {self._prog} [OPTIONS] URL [URL...]\n"
def format_help(self):
return self._long_break_matcher.sub(
"\n\n", self._root_section.format_help())
def _parse_option(opt):
key, _, value = opt.partition("=")
try:
value = util.json_loads(value)
except ValueError:
pass
return key, value
def build_parser():
"""Build and configure an ArgumentParser object"""
parser = argparse.ArgumentParser(
formatter_class=Formatter,
add_help=False,
)
general = parser.add_argument_group("General Options")
general.add_argument(
"-h", "--help",
action="help",
help="Print this help message and exit",
)
general.add_argument(
"--version",
action="version", version=version.__version__,
help="Print program version and exit",
)
general.add_argument(
"-f", "--filename",
dest="filename", metavar="FORMAT",
help=("Filename format string for downloaded files "
"('/O' for \"original\" filenames)"),
)
general.add_argument(
"-d", "--destination",
dest="base-directory", metavar="PATH", action=ConfigAction,
help="Target location for file downloads",
)
general.add_argument(
"-D", "--directory",
dest="directory", metavar="PATH",
help="Exact location for file downloads",
)
general.add_argument(
"-X", "--extractors",
dest="extractor_sources", metavar="PATH", action="append",
help="Load external extractors from PATH",
)
general.add_argument(
"-a", "--user-agent",
dest="user-agent", metavar="UA", action=ConfigAction,
help="User-Agent request header",
)
general.add_argument(
"--clear-cache",
dest="clear_cache", metavar="MODULE",
help="Delete cached login sessions, cookies, etc. for MODULE "
"(ALL to delete everything)",
)
general.add_argument(
"--compat",
dest="category-map", nargs=0, action=ConfigConstAction, const="compat",
help="Restore legacy 'category' names",
)
update = parser.add_argument_group("Update Options")
if util.EXECUTABLE:
update.add_argument(
"-U", "--update",
dest="update", action="store_const", const="latest",
help="Update to the latest version",
)
update.add_argument(
"--update-to",
dest="update", metavar="CHANNEL[@TAG]",
help=("Switch to a dfferent release channel (stable or dev) "
"or upgrade/downgrade to a specific version"),
)
update.add_argument(
"--update-check",
dest="update", action="store_const", const="check",
help="Check if a newer version is available",
)
else:
update.add_argument(
"-U", "--update-check",
dest="update", action="store_const", const="check",
help="Check if a newer version is available",
)
input = parser.add_argument_group("Input Options")
input.add_argument(
"urls",
metavar="URL", nargs="*",
help=argparse.SUPPRESS,
)
input.add_argument(
"-i", "--input-file",
dest="input_files", metavar="FILE", action=InputfileAction, const=None,
default=[],
help=("Download URLs found in FILE ('-' for stdin). "
"More than one --input-file can be specified"),
)
input.add_argument(
"-I", "--input-file-comment",
dest="input_files", metavar="FILE", action=InputfileAction, const="c",
help=("Download URLs found in FILE. "
"Comment them out after they were downloaded successfully."),
)
input.add_argument(
"-x", "--input-file-delete",
dest="input_files", metavar="FILE", action=InputfileAction, const="d",
help=("Download URLs found in FILE. "
"Delete them after they were downloaded successfully."),
)
input.add_argument(
"--no-input",
dest="input", nargs=0, action=ConfigConstAction, const=False,
help="Do not prompt for passwords/tokens",
)
output = parser.add_argument_group("Output Options")
output.add_argument(
"-q", "--quiet",
dest="loglevel", default=logging.INFO,
action="store_const", const=logging.ERROR,
help="Activate quiet mode",
)
output.add_argument(
"-w", "--warning",
dest="loglevel",
action="store_const", const=logging.WARNING,
help="Print only warnings and errors",
)
output.add_argument(
"-v", "--verbose",
dest="loglevel",
action="store_const", const=logging.DEBUG,
help="Print various debugging information",
)
output.add_argument(
"-g", "--get-urls",
dest="list_urls", action="count",
help="Print URLs instead of downloading",
)
output.add_argument(
"-G", "--resolve-urls",
dest="list_urls", action="store_const", const=128,
help="Print URLs instead of downloading; resolve intermediary URLs",
)
output.add_argument(
"-j", "--dump-json",
dest="dump_json", action="count",
help="Print JSON information",
)
output.add_argument(
"-J", "--resolve-json",
dest="dump_json", action="store_const", const=128,
help="Print JSON information; resolve intermediary URLs",
)
output.add_argument(
"-s", "--simulate",
dest="jobtype", action="store_const", const=job.SimulationJob,
help="Simulate data extraction; do not download anything",
)
output.add_argument(
"-E", "--extractor-info",
dest="jobtype", action="store_const", const=job.InfoJob,
help="Print extractor defaults and settings",
)
output.add_argument(
"-K", "--list-keywords",
dest="jobtype", action="store_const", const=job.KeywordJob,
help=("Print a list of available keywords and example values "
"for the given URLs"),
)
output.add_argument(
"-e", "--error-file",
dest="errorfile", metavar="FILE", action=ConfigAction,
help="Add input URLs which returned an error to FILE",
)
output.add_argument(
"-N", "--print",
dest="postprocessors", metavar="[EVENT:]FORMAT",
action=PrintAction, const="-", default=[],
help=("Write FORMAT during EVENT (default 'prepare') to standard "
"output instead of downloading files. "
"Can be used multiple times. "
"Examples: 'id' or 'post:{md5[:8]}'"),
)
output.add_argument(
"--Print",
dest="postprocessors", metavar="[EVENT:]FORMAT",
action=PrintAction, const="+",
help="Like --print, but downloads files as well",
)
output.add_argument(
"--print-to-file",
dest="postprocessors", metavar="[EVENT:]FORMAT FILE",
action=PrintAction, const=None, nargs=2,
help=("Append FORMAT during EVENT to FILE instead of downloading "
"files. Can be used multiple times"),
)
output.add_argument(
"--Print-to-file",
dest="postprocessors", metavar="[EVENT:]FORMAT FILE",
action=PrintAction, const=False, nargs=2,
help="Like --print-to-file, but downloads files as well",
)
output.add_argument(
"--list-modules",
dest="list_modules", action="store_true",
help="Print a list of available extractor modules",
)
output.add_argument(
"--list-extractors",
dest="list_extractors", metavar="[CATEGORIES]", nargs="*",
help=("Print a list of extractor classes "
"with description, (sub)category and example URL"),
)
output.add_argument(
"--write-log",
dest="logfile", metavar="FILE", action=ConfigAction,
help="Write logging output to FILE",
)
output.add_argument(
"--write-unsupported",
dest="unsupportedfile", metavar="FILE", action=ConfigAction,
help=("Write URLs, which get emitted by other extractors but cannot "
"be handled, to FILE"),
)
output.add_argument(
"--write-pages",
dest="write-pages", nargs=0, action=ConfigConstAction, const=True,
help=("Write downloaded intermediary pages to files "
"in the current directory to debug problems"),
)
output.add_argument(
"--print-traffic",
dest="print_traffic", action="store_true",
help="Display sent and read HTTP traffic",
)
output.add_argument(
"--no-colors",
dest="colors", action="store_false",
help="Do not emit ANSI color codes in output",
)
networking = parser.add_argument_group("Networking Options")
networking.add_argument(
"-R", "--retries",
dest="retries", metavar="N", type=int, action=ConfigAction,
help=("Maximum number of retries for failed HTTP requests "
"or -1 for infinite retries (default: 4)"),
)
networking.add_argument(
"--http-timeout",
dest="timeout", metavar="SECONDS", type=float, action=ConfigAction,
help="Timeout for HTTP connections (default: 30.0)",
)
networking.add_argument(
"--proxy",
dest="proxy", metavar="URL", action=ConfigAction,
help="Use the specified proxy",
)
networking.add_argument(
"--source-address",
dest="source-address", metavar="IP", action=ConfigAction,
help="Client-side IP address to bind to",
)
networking.add_argument(
"-4", "--force-ipv4",
dest="source-address", nargs=0, action=ConfigConstAction,
const="0.0.0.0",
help="Make all connections via IPv4",
)
networking.add_argument(
"-6", "--force-ipv6",
dest="source-address", nargs=0, action=ConfigConstAction, const="::",
help="Make all connections via IPv6",
)
networking.add_argument(
"--no-check-certificate",
dest="verify", nargs=0, action=ConfigConstAction, const=False,
help="Disable HTTPS certificate validation",
)
downloader = parser.add_argument_group("Downloader Options")
downloader.add_argument(
"-r", "--limit-rate",
dest="rate", metavar="RATE", action=ConfigAction,
help="Maximum download rate (e.g. 500k, 2.5M, or 800k-2M)",
)
downloader.add_argument(
"--chunk-size",
dest="chunk-size", metavar="SIZE", action=ConfigAction,
help="Size of in-memory data chunks (default: 32k)",
)
downloader.add_argument(
"--sleep",
dest="sleep", metavar="SECONDS", action=ConfigAction,
help=("Number of seconds to wait before each download. "
"This can be either a constant value or a range "
"(e.g. 2.7 or 2.0-3.5)"),
)
downloader.add_argument(
"--sleep-request",
dest="sleep-request", metavar="SECONDS", action=ConfigAction,
help=("Number of seconds to wait between HTTP requests "
"during data extraction"),
)
downloader.add_argument(
"--sleep-429",
dest="sleep-429", metavar="SECONDS", action=ConfigAction,
help=("Number of seconds to wait when receiving a "
"'429 Too Many Requests' response"),
)
downloader.add_argument(
"--sleep-extractor",
dest="sleep-extractor", metavar="SECONDS", action=ConfigAction,
help=("Number of seconds to wait before starting data extraction "
"for an input URL"),
)
downloader.add_argument(
"--no-part",
dest="part", nargs=0, action=ConfigConstAction, const=False,
help="Do not use .part files",
)
downloader.add_argument(
"--no-skip",
dest="skip", nargs=0, action=ConfigConstAction, const=False,
help="Do not skip downloads; overwrite existing files",
)
downloader.add_argument(
"--no-mtime",
dest="mtime", nargs=0, action=ConfigConstAction, const=False,
help=("Do not set file modification times according to "
"Last-Modified HTTP response headers")
)
downloader.add_argument(
"--no-download",
dest="download", nargs=0, action=ConfigConstAction, const=False,
help=("Do not download any files")
)
configuration = parser.add_argument_group("Configuration Options")
configuration.add_argument(
"-o", "--option",
dest="options", metavar="KEY=VALUE",
action=ConfigParseAction, default=[],
help=("Additional options. "
"Example: -o browser=firefox") ,
)
configuration.add_argument(
"-c", "--config",
dest="configs_json", metavar="FILE", action="append",
help="Additional configuration files",
)
configuration.add_argument(
"--config-yaml",
dest="configs_yaml", metavar="FILE", action="append",
help="Additional configuration files in YAML format",
)
configuration.add_argument(
"--config-toml",
dest="configs_toml", metavar="FILE", action="append",
help="Additional configuration files in TOML format",
)
configuration.add_argument(
"--config-create",
dest="config", action="store_const", const="init",
help="Create a basic configuration file",
)
configuration.add_argument(
"--config-status",
dest="config", action="store_const", const="status",
help="Show configuration file status",
)
configuration.add_argument(
"--config-open",
dest="config", action="store_const", const="open",
help="Open configuration file in external application",
)
configuration.add_argument(
"--config-ignore",
dest="config_load", action="store_false",
help="Do not read default configuration files",
)
configuration.add_argument(
"--ignore-config",
dest="config_load", action="store_false",
help=argparse.SUPPRESS,
)
authentication = parser.add_argument_group("Authentication Options")
authentication.add_argument(
"-u", "--username",
dest="username", metavar="USER", action=ConfigAction,
help="Username to login with",
)
authentication.add_argument(
"-p", "--password",
dest="password", metavar="PASS", action=ConfigAction,
help="Password belonging to the given username",
)
authentication.add_argument(
"--netrc",
dest="netrc", nargs=0, action=ConfigConstAction, const=True,
help="Enable .netrc authentication data",
)
cookies = parser.add_argument_group("Cookie Options")
cookies.add_argument(
"-C", "--cookies",
dest="cookies", metavar="FILE", action=ConfigAction,
help="File to load additional cookies from",
)
cookies.add_argument(
"--cookies-export",
dest="cookies-update", metavar="FILE", action=ConfigAction,
help="Export session cookies to FILE",
)
cookies.add_argument(
"--cookies-from-browser",
dest="cookies_from_browser",
metavar="BROWSER[/DOMAIN][+KEYRING][:PROFILE][::CONTAINER]",
help=("Name of the browser to load cookies from, with optional "
"domain prefixed with '/', "
"keyring name prefixed with '+', "
"profile prefixed with ':', and "
"container prefixed with '::' "
"('none' for no container (default), 'all' for all containers)"),
)
selection = parser.add_argument_group("Selection Options")
selection.add_argument(
"-A", "--abort",
dest="abort", metavar="N[:TARGET]",
help=("Stop current extractor(s) "
"after N consecutive file downloads were skipped. "
"Specify a TARGET to set how many levels to ascend or "
"to which subcategory to jump to. "
"Examples: '-A 3', '-A 3:2', '-A 3:manga'"),
)
selection.add_argument(
"-T", "--terminate",
dest="terminate", metavar="N",
help=("Stop current & parent extractors "
"and proceed with the next input URL "
"after N consecutive file downloads were skipped"),
)
selection.add_argument(
"--filesize-min",
dest="filesize-min", metavar="SIZE", action=ConfigAction,
help="Do not download files smaller than SIZE (e.g. 500k or 2.5M)",
)
selection.add_argument(
"--filesize-max",
dest="filesize-max", metavar="SIZE", action=ConfigAction,
help="Do not download files larger than SIZE (e.g. 500k or 2.5M)",
)
selection.add_argument(
"--download-archive",
dest="archive", metavar="FILE", action=ConfigAction,
help=("Record successfully downloaded files in FILE and "
"skip downloading any file already in it"),
)
selection.add_argument(
"--range",
dest="image-range", metavar="RANGE", action=ConfigAction,
help=("Index range(s) specifying which files to download. "
"These can be either a constant value, range, or slice "
"(e.g. '5', '8-20', or '1:24:3')"),
)
selection.add_argument(
"--chapter-range",
dest="chapter-range", metavar="RANGE", action=ConfigAction,
help=("Like '--range', but applies to manga chapters "
"and other delegated URLs"),
)
selection.add_argument(
"--filter",
dest="image-filter", metavar="EXPR", action=ConfigAction,
help=("Python expression controlling which files to download. "
"Files for which the expression evaluates to False are ignored. "
"Available keys are the filename-specific ones listed by '-K'. "
"Example: --filter \"image_width >= 1000 and "
"rating in ('s', 'q')\""),
)
selection.add_argument(
"--chapter-filter",
dest="chapter-filter", metavar="EXPR", action=ConfigAction,
help=("Like '--filter', but applies to manga chapters "
"and other delegated URLs"),
)
infojson = {
"name" : "metadata",
"event" : "init",
"filename": "info.json",
}
postprocessor = parser.add_argument_group("Post-processing Options")
postprocessor.add_argument(
"-P", "--postprocessor",
dest="postprocessors", metavar="NAME", action="append",
help="Activate the specified post processor",
)
postprocessor.add_argument(
"--no-postprocessors",
dest="postprocess", nargs=0, action=ConfigConstAction, const=False,
help=("Do not run any post processors")
)
postprocessor.add_argument(
"-O", "--postprocessor-option",
dest="options_pp", metavar="KEY=VALUE",
action=PPParseAction, default={},
help="Additional post processor options",
)
postprocessor.add_argument(
"--write-metadata",
dest="postprocessors",
action="append_const", const="metadata",
help="Write metadata to separate JSON files",
)
postprocessor.add_argument(
"--write-info-json",
dest="postprocessors",
action="append_const", const=infojson,
help="Write gallery metadata to a info.json file",
)
postprocessor.add_argument(
"--write-infojson",
dest="postprocessors",
action="append_const", const=infojson,
help=argparse.SUPPRESS,
)
postprocessor.add_argument(
"--write-tags",
dest="postprocessors",
action="append_const", const={"name": "metadata", "mode": "tags"},
help="Write image tags to separate text files",
)
postprocessor.add_argument(
"--zip",
dest="postprocessors",
action="append_const", const="zip",
help="Store downloaded files in a ZIP archive",
)
postprocessor.add_argument(
"--cbz",
dest="postprocessors",
action="append_const", const={
"name" : "zip",
"extension": "cbz",
},
help="Store downloaded files in a CBZ archive",
)
postprocessor.add_argument(
"--mtime",
dest="postprocessors", metavar="NAME", action=MtimeAction,
help=("Set file modification times according to metadata "
"selected by NAME. Examples: 'date' or 'status[date]'"),
)
postprocessor.add_argument(
"--mtime-from-date",
dest="postprocessors", nargs=0, action=MtimeAction,
const="date|status[date]",
help=argparse.SUPPRESS,
)
postprocessor.add_argument(
"--rename",
dest="postprocessors", metavar="FORMAT", action=RenameAction, const=0,
help=("Rename previously downloaded files from FORMAT "
"to the current filename format"),
)
postprocessor.add_argument(
"--rename-to",
dest="postprocessors", metavar="FORMAT", action=RenameAction, const=1,
help=("Rename previously downloaded files from the current filename "
"format to FORMAT"),
)
postprocessor.add_argument(
"--ugoira",
dest="postprocessors", metavar="FMT", action=UgoiraAction,
help=("Convert Pixiv Ugoira to FMT using FFmpeg. "
"Supported formats are 'webm', 'mp4', 'gif', "
"'vp8', 'vp9', 'vp9-lossless', 'copy', 'zip'."),
)
postprocessor.add_argument(
"--ugoira-conv",
dest="postprocessors", nargs=0, action=UgoiraAction, const="vp8",
help=argparse.SUPPRESS,
)
postprocessor.add_argument(
"--ugoira-conv-lossless",
dest="postprocessors", nargs=0, action=UgoiraAction,
const="vp9-lossless",
help=argparse.SUPPRESS,
)
postprocessor.add_argument(
"--ugoira-conv-copy",
dest="postprocessors", nargs=0, action=UgoiraAction, const="copy",
help=argparse.SUPPRESS,
)
postprocessor.add_argument(
"--exec",
dest="postprocessors", metavar="CMD",
action=AppendCommandAction, const={"name": "exec"},
help=("Execute CMD for each downloaded file. "
"Supported replacement fields are "
"{} or {_path}, {_directory}, {_filename}. "
"Example: --exec \"convert {} {}.png && rm {}\""),
)
postprocessor.add_argument(
"--exec-after",
dest="postprocessors", metavar="CMD",
action=AppendCommandAction, const={
"name": "exec", "event": "finalize"},
help=("Execute CMD after all files were downloaded. "
"Example: --exec-after \"cd {_directory} "
"&& convert * ../doc.pdf\""),
)
try:
parser._has_negative_number_optionals.clear()
except Exception:
pass
return parser
