#!/usr/bin/env python3
"""
Given the output of doctest and a file, adjust the doctests so they won't fail.
Doctest failures due to exceptions are ignored.
AUTHORS::
- Nicolas M. Thiéry <nthiery at users dot sf dot net> Initial version (2008?)
- Andrew Mathas <andrew dot mathas at sydney dot edu dot au> 2013-02-14
Cleaned up the code and hacked it so that the script can now cope with the
situations when either the expected output or computed output are empty.
Added doctest to sage.tests.cmdline
"""
# ****************************************************************************
# Copyright (C) 2006 William Stein
# 2009 Nicolas M. Thiery
# 2013 Andrew Mathas
# 2014 Volker Braun
# 2020 Jonathan Kliem
# 2021 Frédéric Chapoton
# 2023 Matthias Koeppe
#
# Distributed under the terms of the GNU General Public License (GPL)
# as published by the Free Software Foundation; either version 2 of
# the License, or (at your option) any later version.
# https://www.gnu.org/licenses/
# ****************************************************************************
import itertools
import json
import os
import re
import shlex
import subprocess
import sys
from argparse import ArgumentParser
from collections import defaultdict
from pathlib import Path
from sage.doctest.control import DocTestDefaults, DocTestController
from sage.doctest.parsing import parse_file_optional_tags, parse_optional_tags, unparse_optional_tags, update_optional_tags
from sage.env import SAGE_ROOT
from sage.features import PythonModule
from sage.features.all import all_features, module_feature, name_feature
from sage.misc.cachefunc import cached_function
from sage.misc.temporary_file import tmp_filename
parser = ArgumentParser(description="Given an input file with doctests, this creates a modified file that passes the doctests (modulo any raised exceptions). By default, the input file is modified. You can also name an output file.")
parser.add_argument('-l', '--long', dest='long', action="store_true", default=False,
help="include tests tagged '# long time'")
parser.add_argument("--distribution", type=str, default=[], action='append',
help="distribution package to test, e.g., 'sagemath-graphs', 'sagemath-combinat[modules]'; sets defaults for --venv and --environment. This option can be repeated to test several distributions")
parser.add_argument("--fixed-point", default=False, action="store_true",
help="whether to repeat until stable")
parser.add_argument("--venv", type=str, default='',
help="directory name of a venv where 'sage -t' is to be run")
parser.add_argument("--environment", type=str, default='',
help="name of a module that provides the global environment for tests, e.g., 'sage.all__sagemath_modules'; implies --keep-both and --full-tracebacks")
parser.add_argument("--no-test", default=False, action="store_true",
help="do not run the doctester, only rewrite '# optional/needs' tags; implies --only-tags")
parser.add_argument("--full-tracebacks", default=False, action="store_true",
help="include full tracebacks rather than '...'")
parser.add_argument("--only-tags", default=False, action="store_true",
help="only add '# optional/needs' tags where needed, ignore other failures")
parser.add_argument("--probe", metavar="FEATURES", type=str, default='',
help="check whether '# optional/needs' tags are still needed, remove those not needed")
parser.add_argument("--keep-both", default=False, action="store_true",
help="do not replace test results; duplicate the test instead, showing both results, and mark both copies '# optional'")
parser.add_argument("--overwrite", default=False, action="store_true",
help="never interpret a second filename as OUTPUT; overwrite the source files")
parser.add_argument("--no-overwrite", default=False, action="store_true",
help="never interpret a second filename as OUTPUT; output goes to files named INPUT.fixed")
parser.add_argument("--update-known-test-failures", default=False, action="store_true",
help="update the file pkgs/DISTRIBUTION/known-test-failures.json")
parser.add_argument("--verbose", default=False, action="store_true",
help="show details of all changes; implies --no-diff")
parser.add_argument("--no-diff", default=False, action="store_true",
help="don't show the 'git diff' of the modified files")
parser.add_argument("filename", nargs='*', help="input filenames; or (deprecated) INPUT_FILENAME OUTPUT_FILENAME if exactly two filenames are given and neither --overwrite nor --no-overwrite is present",
type=str)
runtest_default_environment = "sage.repl.ipython_kernel.all_jupyter"
def plain_distribution_and_extras(distribution):
# shortcuts / variants
distribution = distribution.replace('_', '-')
if not (distribution.startswith('sagemath-')
or distribution.startswith('sage-')):
distribution = f'sagemath-{distribution}'
# extras
m = re.fullmatch(r'([^[]*)(\[([^]]*)\])?', distribution)
return m.group(1), m.group(3)
def default_venv_environment_from_distribution(distribution):
if distribution:
plain_distribution, extras = plain_distribution_and_extras(distribution)
tox_env_name = 'sagepython-sagewheels-nopypi-norequirements'
if extras:
tox_env_name += '-' + extras.replace(',', '-')
default_venv = os.path.join(SAGE_ROOT, 'pkgs', plain_distribution, '.tox', tox_env_name)
default_environment = 'sage.all__' + plain_distribution.replace('-', '_')
else:
default_venv = ''
default_environment = runtest_default_environment
return default_venv, default_environment
@cached_function
def venv_explainer(distribution, venv=None, environment=None):
venv_explainers = []
default_venv, default_environment = default_venv_environment_from_distribution(distribution)
if venv:
if m := re.search(f'pkgs/(sage[^/]*)/[.]tox/((sagepython|sagewheels|nopypi|norequirements)-*)*([^/]*)$',
venv):
distribution, extras = m.group(1), m.group(4)
if extras:
distribution += '[' + extras.replace('-', ',') + ']'
default_venv_given_distribution, default_environment_given_distribution = default_venv_environment_from_distribution(distribution)
if (Path(venv).resolve() == Path(default_venv_given_distribution).resolve()
or not environment or environment == default_environment_given_distribution):
venv_explainers.append(f'--distribution {shlex.quote(distribution)}')
default_venv, default_environment = default_venv_given_distribution, default_environment_given_distribution
if venv and Path(venv).resolve() != Path(default_venv).resolve():
venv_explainers.append(f'--venv {shlex.quote(venv)}')
if environment and environment != default_environment:
venv_explainers.append(f'--environment {environment}')
if venv_explainers:
return ' (with ' + ' '.join(venv_explainers) + ')'
return ''
sep = "**********************************************************************\n"
def process_block(block, src_in_lines, file_optional_tags, venv_explainer=''):
if args.verbose:
print(sep + block.rstrip())
# Extract the line, what was expected, and was got.
if not (m := re.match('File "([^"]*)", line ([0-9]+), in ', block)):
return
def print_line(num):
if args.verbose and (src := src_in_lines[num]):
if src:
for line in src.split('\n'):
line = line.strip()
if line.startswith("sage: ") or line.startswith("....: "):
line = line[6:]
print(f" {line}") # indent to match the displayed "Example" in the sage-runtest message
def update_line(num, new, message=None):
src_in_lines[num] = new
if args.verbose and message:
print(f"sage-fixdoctests: {message}")
print_line(num)
def append_to_line(num, new, message=None):
update_line(num, src_in_lines[num] + new, message=message)
def prepend_to_line(num, new, message=None):
update_line(num, new + src_in_lines[num], message=message)
def update_line_optional_tags(num, *args, message=None, **kwds):
update_line(num,
update_optional_tags(src_in_lines[num], *args, **kwds),
message=message)
filename = m.group(1)
first_line_num = line_num = int(m.group(2)) # 1-based line number of the first line of the example
if m := re.search(r"using.*block-scoped tag.*'(sage: .*)'.*to avoid repeating the tag", block):
indent = (len(src_in_lines[first_line_num - 1]) - len(src_in_lines[first_line_num - 1].lstrip()))
append_to_line(line_num - 2, '\n' + ' ' * indent + m.group(1),
message="Adding this block-scoped tag")
print_line(first_line_num - 1)
if m := re.search(r"updating.*block-scoped tag.*'sage: (.*)'.*to avoid repeating the tag", block):
update_line_optional_tags(first_line_num - 1, tags=parse_optional_tags('# ' + m.group(1)),
message="Adding this tag to the existing block-scoped tag")
if m := re.search(r"referenced here was set only in doctest marked '# (optional|needs)[-: ]*([^;']*)", block):
optional = m.group(2).split()
if src_in_lines[first_line_num - 1].strip() in ['"""', "'''"]:
# This happens due to a virtual doctest in src/sage/repl/user_globals.py
return
optional = set(optional) - set(file_optional_tags)
update_line_optional_tags(first_line_num - 1, add_tags=optional,
message=f"Adding the tag(s) {optional}")
if m := re.search(r"tag '# (optional|needs)[-: ]([^;']*)' may no longer be needed", block):
optional = m.group(2).split()
update_line_optional_tags(first_line_num - 1, remove_tags=optional,
message=f"Removing the tag(s) {optional}")
if m2 := re.search('(Expected:|Expected nothing|Exception raised:)\n', block):
m1 = re.search('Failed example:\n', block)
line_num += block[m1.end() : m2.start()].count('\n') - 1
# Now line_num is the 1-based line number of the last line of the example
if m2.group(1) == 'Expected nothing':
expected = ''
block = '\n' + block[m2.end():] # so that split('\nGot:\n') does not fail below
elif m2.group(1) == 'Exception raised:':
# In this case, the doctester does not show the expected output,
# so we do not know how many lines it spans; so we check for the next prompt or
# docstring end.
expected = []
indentation = ' ' * (len(src_in_lines[line_num - 1]) - len(src_in_lines[line_num - 1].lstrip()))
i = line_num
while ((not src_in_lines[i].rstrip() or src_in_lines[i].startswith(indentation))
and not re.match(' *(sage:|""")', src_in_lines[i])):
expected.append(src_in_lines[i])
i += 1
block = '\n'.join(expected) + '\nGot:\n' + block[m2.end():]
else:
block = block[m2.end():]
else:
return
# Error testing.
if m := re.search(r"(?:ModuleNotFoundError: No module named|ImportError: cannot import name '(.*?)' from) '(.*?)'|AttributeError: module '(.*)?' has no attribute '(.*?)'", block):
if m.group(1):
# "ImportError: cannot import name 'function_field_polymod' from 'sage.rings.function_field' (unknown location)"
module = m.group(2) + '.' + m.group(1)
elif m.group(2):
# "ModuleNotFoundError: No module named ..."
module = m.group(2)
else:
# AttributeError: module 'sage.rings' has no attribute 'qqbar'
module = m.group(3) + '.' + m.group(4)
asked_why = re.search('#.*(why|explain)', src_in_lines[first_line_num - 1])
optional = module_feature(module)
if optional and optional.name not in file_optional_tags:
update_line_optional_tags(first_line_num - 1, add_tags=[optional.name],
message=f"Module '{module}' may be provided by feature '{optional.name}'; adding this tag")
if not asked_why:
# When no explanation has been demanded,
# we just mark the doctest with the feature
return
# Otherwise, continue and show the backtrace as 'GOT'
if 'Traceback (most recent call last):' in block:
expected, got = block.split('\nGot:\n')
if args.full_tracebacks:
if re.fullmatch(' *\n', got):
got = got[re.end(0):]
# don't show doctester internals (anything before first "<doctest...>" frame
if m := re.search('( *Traceback.*\n *)(?s:.*?)(^ *File "<doctest)( [^>]*)>', got, re.MULTILINE):
got = m.group(1) + '...\n' + m.group(2) + '...' + got[m.end(3):]
while m := re.search(' *File "<doctest( [^>]*)>', got):
got = got[:m.start(1)] + '...' + got[m.end(1):]
# simplify filenames shown in backtrace
while m := re.search('"([-a-zA-Z0-9._/]*/site-packages)/sage/', got):
got = got[:m.start(1)] + '...' + got[m.end(1):]
last_frame = got.rfind('File "')
if (last_frame >= 0
and (index_NameError := got.rfind("NameError:")) >= 0
and got[last_frame:].startswith('File "<doctest')):
if args.verbose:
print("sage-fixdoctests: This is a NameError from the top level of the doctest") # so we keep it brief
if m := re.match("NameError: name '(.*)'", got[index_NameError:]):
name = m.group(1)
if name in ['I', 'i']:
add_tags = ['sage.symbolic'] # This is how we mark it currently (2023-08)
elif len(name) >= 2 and (feature := name_feature(name)) and feature.name != 'sage.all':
# Don't mark use of 'x' '# needs sage.symbolic'; that's almost always wrong
# Likewise for variables like 'R', 'r'
add_tags = [feature.name] # FIXME: This feature may actually already be present in line, block, or file. Move this lookup code into the doctester and issue more specific instructions
elif args.only_tags:
if args.verbose:
print("sage-fixdoctests: No feature providing this global is known; no action because of --only-tags")
return
else:
add_tags = [f"NameError ('{name}', {venv_explainer.lstrip().lstrip('(')}"]
else:
if args.only_tags:
if args.verbose:
print("sage-fixdoctests: No feature providing this global is known; no action because of --only-tags")
return
add_tags = [f"NameError{venv_explainer}"]
update_line_optional_tags(first_line_num - 1, add_tags=add_tags,
message=f"Adding tag {add_tags}")
return
got = got.splitlines()
else:
got = got.splitlines()
got = ['Traceback (most recent call last):', '...', got[-1].lstrip()]
elif block[-21:] == 'Got:\n <BLANKLINE>\n':
expected = block[:-22]
got = ['']
else:
expected, got = block.split('\nGot:\n')
got = re.sub(r'(doctest:warning).*^( *DeprecationWarning:)',
r'\1...\n\2',
got, 1, re.DOTALL | re.MULTILINE)
got = got.splitlines() # got can't be the empty string
if args.only_tags:
if args.verbose:
print("sage-fixdoctests: No action because of --only-tags")
return
expected = expected.splitlines()
if args.keep_both:
test_lines = ([update_optional_tags(src_in_lines[first_line_num - 1],
add_tags=[f'GOT{venv_explainer}'])]
+ src_in_lines[first_line_num : line_num])
update_line_optional_tags(first_line_num - 1, add_tags=['EXPECTED'],
message="Marking the doctest with idempotent tag EXPECTED, creating another copy with idempotent tag GOT")
indent = (len(src_in_lines[line_num - 1]) - len(src_in_lines[line_num - 1].lstrip()))
line_num += len(expected) # skip to the last line of the expected output
append_to_line(line_num - 1, '\n'.join([''] + test_lines)) # 2nd copy of the test
# now line_num is the last line of the 2nd copy of the test
expected = []
# If we expected nothing, and got something, then we need to insert the line before line_num
# and match indentation with line number line_num-1
if not expected:
indent = (len(src_in_lines[first_line_num - 1]) - len(src_in_lines[first_line_num - 1].lstrip()))
append_to_line(line_num - 1,
'\n' + '\n'.join('%s%s' % (' ' * indent, line.lstrip()) for line in got),
message="Adding the new output")
return
# Guess how much extra indenting ``got`` needs to match with the indentation
# of src_in_lines - we match the indentation with the line in ``got`` which
# has the smallest indentation after lstrip(). Note that the amount of indentation
# required could be negative if the ``got`` block is indented. In this case
# ``indent`` is set to zero.
indent = max(0, (len(src_in_lines[line_num]) - len(src_in_lines[line_num].lstrip())
- min(len(got[j]) - len(got[j].lstrip()) for j in range(len(got)))))
# Double check that what was expected was indeed in the source file and if
# it is not then then print a warning for the user which contains the
# problematic lines.
if any(expected[i].strip() != src_in_lines[line_num + i].strip()
for i in range(len(expected))):
import warnings
txt = "Did not manage to replace\n%s\n%s\n%s\nwith\n%s\n%s\n%s"
warnings.warn(txt % ('>' * 40, '\n'.join(expected), '>' * 40,
'<' * 40, '\n'.join(got), '<' * 40))
return
# If we got nothing when we expected something then we delete the line from the
# output, otherwise, add all of what we `got` onto the end of src_in_lines[line_num]
if got == ['']:
update_line(line_num, None,
message="Expected something, got nothing; deleting the old output")
else:
update_line(line_num, '\n'.join((' ' * indent + got[i]) for i in range(len(got))),
message="Replacing the old expected output with the new output")
# Mark any remaining `expected` lines as ``None`` so as to preserve the line numbering
for i in range(1, len(expected)):
update_line(line_num + i, None)
# set input and output files
def output_filename(filename):
if len(args.filename) == 2 and not args.overwrite and not args.no_overwrite:
if args.filename[0] == filename:
return args.filename[1]
else:
return None
return filename + ".fixed"
if args.no_overwrite:
return filename + ".fixed"
return filename
tested_doctesters = set()
venv_files = {} # distribution -> files that are not yet known to be fixed points in venv; we add and remove items
venv_ignored_files = {} # distribution -> files that should be ignored; we only add items
unprocessed_files = set()
class BadDistribution(Exception):
pass
def doctest_blocks(args, input_filenames, distribution=None, venv=None, environment=None):
executable = f'{os.path.relpath(venv)}/bin/sage' if venv else 'sage'
environment_args = f'--environment {environment} ' if environment and environment != runtest_default_environment else ''
long_args = f'--long ' if args.long else ''
probe_args = f'--probe {shlex.quote(args.probe)} ' if args.probe else ''
lib_args = f'--if-installed ' if venv else ''
doc_file = tmp_filename()
if venv or environment_args:
# Test the doctester, putting the output of the test into sage's temporary directory
input = os.path.join(os.path.relpath(SAGE_ROOT), 'src', 'sage', 'version.py')
cmdline = f'{shlex.quote(executable)} -t {environment_args}{long_args}{probe_args}'.rstrip()
if cmdline not in tested_doctesters:
if args.verbose:
print(f'sage-fixdoctests: Checking whether the doctester "{cmdline}" works')
cmdline += f' {shlex.quote(input)}'
if status := os.waitstatus_to_exitcode(os.system(f'{cmdline} > {shlex.quote(doc_file)}')):
raise BadDistribution(f"Doctester exited with error status {status}")
tested_doctesters.add(cmdline)
# Run the doctester, putting the output of the test into sage's temporary directory
input_args = " ".join(shlex.quote(f) for f in input_filenames)
cmdline = f'{shlex.quote(executable)} -t -p {environment_args}{long_args}{probe_args}{lib_args}{input_args}'
print(f'Running "{cmdline}"')
os.system(f'{cmdline} > {shlex.quote(doc_file)}')
with open(doc_file, 'r') as doc:
doc_out = doc.read()
# Remove skipped files, echo control messages
for m in re.finditer(r"^Skipping '(.*?)'.*$", doc_out, re.MULTILINE):
print('sage-runtests: ' + m.group(0))
if distribution is not None:
venv_files[distribution].discard(m.group(1))
venv_ignored_files[distribution].add(m.group(1))
return doc_out.split(sep)
def block_filename(block):
if not (m := re.match('File "([^"]*)", line ([0-9]+), in ', block)):
return None
return m.group(1)
def expanded_filename_args():
DD = DocTestDefaults(optional='all', warn_long=10000)
DC = DocTestController(DD, input_filenames)
DC.add_files()
DC.expand_files_into_sources()
for source in DC.sources:
yield source.path
def process_grouped_blocks(grouped_iterator, distribution=None, venv=None, environment=None):
seen = set()
explainer = venv_explainer(distribution, venv, environment)
for input, blocks in grouped_iterator:
if not input: # Blocks of noise
continue
if input in seen:
continue
seen.add(input)
with open(input, 'r') as test_file:
src_in = test_file.read()
src_in_lines = src_in.splitlines()
shallow_copy_of_src_in_lines = list(src_in_lines)
file_optional_tags = set(parse_file_optional_tags(enumerate(src_in_lines)))
persistent_tags_counts = defaultdict(int)
tags_counts = defaultdict(int)
for block in blocks:
try:
process_block(block, src_in_lines, file_optional_tags, venv_explainer=explainer)
except Exception:
print('sage-fixdoctests: Failure to process block')
print(block)
# Now source line numbers do not matter any more, and lines can be real lines again
src_in_lines = list(itertools.chain.from_iterable(
[] if line is None else [''] if not line else line.splitlines()
for line in src_in_lines))
# Remove duplicate optional tags and rewrite all '# optional' that should be '# needs'
persistent_optional_tags = {}
persistent_optional_tags_counted = False
for i, line in enumerate(src_in_lines):
if m := re.match(' *sage: *(.*)#', line):
tags, line_sans_tags, is_persistent = parse_optional_tags(line, return_string_sans_tags=True)
if is_persistent:
persistent_optional_tags = {tag: explanation
for tag, explanation in tags.items()
if explanation or tag not in file_optional_tags}
persistent_optional_tags_counted = False
line = update_optional_tags(line, tags=persistent_optional_tags, force_rewrite='standard')
if re.fullmatch(' *sage: *', line):
# persistent (block-scoped or file-scoped) tag was removed, so remove the whole line
line = None
else:
tags = {tag: explanation
for tag, explanation in tags.items()
if explanation or (tag not in file_optional_tags
and tag not in persistent_optional_tags)}
line = update_optional_tags(line, tags=tags, force_rewrite='standard')
if not persistent_optional_tags_counted:
persistent_tags_counts[frozenset(persistent_optional_tags)] += 1
persistent_optional_tags_counted = True
tags_counts[frozenset(tags)] += 1
src_in_lines[i] = line
elif line.strip() in ['', '"""', "'''"]: # Blank line or end of docstring
persistent_optional_tags = {}
persistent_optional_tags_counted = False
elif re.match(' *sage: ', line):
if not persistent_optional_tags_counted:
persistent_tags_counts[frozenset(persistent_optional_tags)] += 1
persistent_optional_tags_counted = True
tags_counts[frozenset()] += 1
if src_in_lines != shallow_copy_of_src_in_lines:
if (output := output_filename(input)) is None:
print(f"sage-fixdoctests: Not saving modifications made in '{input}'")
else:
with open(output, 'w') as test_output:
for line in src_in_lines:
if line is None:
continue
test_output.write(line)
test_output.write('\n')
# Show summary of changes
if input != output:
print("sage-fixdoctests: The fixed doctests have been saved as '{0}'.".format(output))
else:
relative = os.path.relpath(output, SAGE_ROOT)
print(f"sage-fixdoctests: The input file '{output}' has been overwritten.")
if not args.no_diff and not relative.startswith('..'):
subprocess.call(['git', '--no-pager', 'diff', relative], cwd=SAGE_ROOT)
for other_distribution, file_set in venv_files.items():
if input not in venv_ignored_files[other_distribution]:
file_set.add(input)
else:
print(f"sage-fixdoctests: No fixes made in '{input}'")
if distribution is not None:
venv_files[distribution].discard(input)
unprocessed_files.discard(input)
if args.verbose:
if file_optional_tags:
print(f"File tags: ")
print(f" {' '.join(sorted(file_optional_tags))}")
if persistent_tags_counts:
print(f"Doctest blocks by persistent tags: ")
for tags, count in sorted(persistent_tags_counts.items(),
key=lambda i: i[1], reverse=True):
print(f"{count:6} {' '.join(sorted(tags)) or '(untagged)'}")
if tags_counts:
print(f"Doctest examples by tags: ")
for tags, count in sorted(tags_counts.items(),
key=lambda i: i[1], reverse=True):
print(f"{count:6} {' '.join(sorted(tags)) or '(untagged)'}")
def fix_with_distribution(file_set, distribution=None, verbose=False):
if verbose:
print("#" * 78)
print(f"sage-fixdoctests: Fixing with --distribution={shlex.quote(distribution)}")
default_venv, default_environment = default_venv_environment_from_distribution(distribution)
venv = args.venv or default_venv
environment = args.environment or default_environment
file_set_to_process = sorted(file_set)
file_set.clear()
try:
doctests = doctest_blocks(args, file_set_to_process,
distribution=distribution, venv=venv, environment=environment)
process_grouped_blocks(itertools.groupby(doctests, block_filename), # modifies file_set
distribution=distribution, venv=venv, environment=environment)
except BadDistribution as e:
if args.ignore_bad_distributions:
print(f"sage-fixdoctests: {e}, ignoring")
else:
sys.exit(f"sage-fixdoctests: {e}")
if __name__ == "__main__":
args = parser.parse_args()
if args.verbose:
args.no_diff = True
args.ignore_bad_distributions = False # This could also be a switch
args.update_failures_distribution = args.distribution
if args.distribution == ['all']:
args.distribution = ['sagemath-categories',
''] # monolithic distribution
args.update_failures_distribution = args.distribution + ['sagemath-repl', # not included above because it knows too little and complains too much
]
args.ignore_bad_distributions = True
if not args.filename:
if not args.update_known_test_failures:
sys.exit("sage-fixdoctests: At least one filename is required when --update-known-test-failures is not used")
if not args.distribution:
sys.exit("sage-fixdoctests: At least one --distribution argument is required for --update-known-test-failures")
if args.distribution or args.venv or args.environment:
args.keep_both = args.full_tracebacks = True
if len(args.distribution) > 1:
if args.venv or args.environment:
sys.exit("sage-fixdoctests: at most one --distribution argument can be combined with --venv and --environment")
elif not args.distribution:
args.distribution = ['']
if len(args.filename) == 2 and not args.overwrite and not args.no_overwrite:
print("sage-fixdoctests: When passing two filenames, the second one is taken as an output filename; "
"this is deprecated. To pass two input filenames, use the option --overwrite.")
input_filenames = [args.filename[0]]
else:
input_filenames = args.filename
try:
unprocessed_files = set(expanded_filename_args())
for distribution in args.distribution:
venv_files[distribution] = set(unprocessed_files) # make copies
venv_ignored_files[distribution] = set()
if args.no_test:
pass
elif len(args.distribution) == 1 and not args.fixed_point:
fix_with_distribution(set(unprocessed_files), args.distribution[0])
else:
for distribution, file_set in venv_files.items():
fix_with_distribution(file_set, distribution, verbose=True)
if args.fixed_point:
if args.probe:
print(f"sage-fixdoctests: Turning off --probe for the following iterations")
# This forces convergence to a fixed point
args.probe = ''
while True:
# Run a distribution with largest number of files remaining to be checked
# because of the startup overhead of sage-runtests
distribution, file_set = max(venv_files.items(), key=lambda df: len(df[1]))
if not file_set:
break
while file_set:
fix_with_distribution(file_set, distribution, verbose=True)
# Immediately re-run with the same distribution to continue chains of
# "NameError" / "variable was set only in doctest" fixes
# Each file must be processed by process_grouped_blocks at least once to clean up tags,
# even if sage-runtest does not have any complaints.
if unprocessed_files:
print(f"sage-fixdoctests: Processing unprocessed files")
process_grouped_blocks([(filename, [])
for filename in unprocessed_files])
if args.fixed_point:
print(f"sage-fixdoctests: Fixed point reached")
if args.update_known_test_failures:
if args.update_failures_distribution == ['']:
print("sage-fixdoctests: Ignoring switch --update-known-test-failures because no --distribution was given")
else:
for distribution in sorted(args.update_failures_distribution):
if distribution == '':
continue
plain_distribution, extras = plain_distribution_and_extras(distribution)
default_venv, _ = default_venv_environment_from_distribution(distribution)
venv = args.venv or default_venv
try:
stats_filename = os.path.join(default_venv, '.sage/timings2.json')
with open(stats_filename, 'r') as stats_file:
stats = json.load(stats_file)
except FileNotFoundError:
print(f"sage-fixdoctests: {os.path.relpath(stats_filename, SAGE_ROOT)} "
"does not exist (ignoring)")
else:
for d in stats.values():
del d['walltime']
stats = {k: d for k, d in stats.items()
if d.get('failed') or d.get('ntests', True)}
if extras:
extras_suffix = '--' + '--'.join(extras.split(','))
else:
extras_suffix = ''
failures_file = os.path.join(SAGE_ROOT, 'pkgs', plain_distribution,
f'known-test-failures{extras_suffix}.json')
with open(failures_file, 'w') as f:
json.dump(stats, f, sort_keys=True, indent=4)
print(f"sage-fixdoctests: Updated {os.path.relpath(failures_file, SAGE_ROOT)}")
except Exception:
print(f"sage-fixdoctests: Internal error")
raise
