Noorul Islam K M <noorul_at_collab.net> writes:
> Paul Burba <ptburba_at_gmail.com> writes:
>
>> If someone with the requisite linux skills/hardware could tweak
>> makefile.in so it can take advantage of the --milestone-filter option,
>> well that would be fabulous.
>>
>> Paul
>>
>> On Thu, Feb 17, 2011 at 5:09 PM, <pburba_at_apache.org> wrote:
>>> Author: pburba
>>> Date: Thu Feb 17 22:09:02 2011
>>> New Revision: 1071809
>>>
>>> URL: http://svn.apache.org/viewvc?rev=1071809&view=rev
>>> Log:
>>> Add an option (--milestone-filter=REGEX) to the Python tests so we can list a
>>> subset of the tests based on their associated issues' target milestone.
>>>
>>> This option is currently only available to win-tests.py and
>>> subversion/tests/cmdline/svntest/main.py. So it isn't quite as useful
>>> on non-Windows platforms just yet.
>>>
>>> Now we can easily answer questions like, "What xfailing merge tests need to
>>> be fixed before we can release 1.7?"
>>>
>>> C:\SVN\src-trunk>win-tests.py --list --milestone-filter="(1.7.*)|(---)"
>>> --mode-filter xfail --log-to-stdout --test merge
>>> Listing Debug configuration on local repository.
>>> LISTING: merge_tests.py
>>> Test # Mode Test Description
>>> ------ ----- ----------------
>>> 64 XFAIL merge target with non inheritable mergeinfo
>>> [#2970(blue-sky),#3642(1.7.0)]
>>> 75 XFAIL merge added subtree [#1962(1.7-consider)]
>>>
>>> * build/run_tests.py
>>>
>>> (TestHarness.__init__): Add mode_filter argument.
>>>
>>> (TestHarness._run_c_test): Issue warning that --milestone-filter doesn't
>>> work when listing C tests.
>>>
>>> (TestHarness._run_py_test): Accept --milestone-filter option.
>>>
>>> * subversion/tests/cmdline/svntest/main.py
>>>
>>> (global): Import xml and urllib.
>>>
>>> (TestSpawningThread.run_one): Support --milestone-filter option.
>>>
>>> (TestRunner.list): Add optional argument mapping issues to target
>>> milestones.
>>>
>>> (TestRunner.get_issues): New.
>>>
>>> (_create_parser): Handle --milestone-filter.
>>>
>>> (get_target_milestones_for_issues): New.
>>>
>>> (execute_tests): Handle --milestone-filter.
>>>
>>> * win-tests.py
>>>
>>> (_usage_exit): Add --milestone-filter to usage text.
>>>
>>> (milestone_filter): New global variable.
>>>
>>> (global): Accept --milestone-filter as a valid option, pass it to
>>> run_tests.TestHarness().
>>>
>>>
>>> Modified:
>>> subversion/trunk/build/run_tests.py
>>> subversion/trunk/subversion/tests/cmdline/svntest/main.py
>>> subversion/trunk/win-tests.py
>>>
>>> Modified: subversion/trunk/build/run_tests.py
>>> URL: http://svn.apache.org/viewvc/subversion/trunk/build/run_tests.py?rev=1071809&r1=1071808&r2=1071809&view=diff
>>> ==============================================================================
>>> --- subversion/trunk/build/run_tests.py (original)
>>> +++ subversion/trunk/build/run_tests.py Thu Feb 17 22:09:02 2011
>>> @@ -79,7 +79,8 @@ class TestHarness:
>>> server_minor_version=None, verbose=None,
>>> cleanup=None, enable_sasl=None, parallel=None, config_file=None,
>>> fsfs_sharding=None, fsfs_packing=None,
>>> - list_tests=None, svn_bin=None, mode_filter=None):
>>> + list_tests=None, svn_bin=None, mode_filter=None,
>>> + milestone_filter=None):
>>> '''Construct a TestHarness instance.
>>>
>>> ABS_SRCDIR and ABS_BUILDDIR are the source and build directories.
>>> @@ -91,8 +92,12 @@ class TestHarness:
>>> HTTP_LIBRARY is the HTTP library for DAV-based communications.
>>> SERVER_MINOR_VERSION is the minor version of the server being tested.
>>> SVN_BIN is the path where the svn binaries are installed.
>>> - mode_filter restricts the TestHarness to tests with the expected mode
>>> - XFail, Skip, Pass, or All tests (default).
>>> + MODE_FILTER restricts the TestHarness to tests with the expected mode
>>> + XFail, Skip, Pass, or All tests (default). MILESTONE_FILTER is a
>>> + string representation of a valid regular expression pattern; when used
>>> + in conjunction with LIST_TESTS, the only tests that are listed are
>>> + those with an associated issue in the tracker which has a target
>>> + milestone that matches the regex.
>>> '''
>>> self.srcdir = abs_srcdir
>>> self.builddir = abs_builddir
>>> @@ -114,6 +119,7 @@ class TestHarness:
>>> if config_file is not None:
>>> self.config_file = os.path.abspath(config_file)
>>> self.list_tests = list_tests
>>> + self.milestone_filter = milestone_filter
>>> self.svn_bin = svn_bin
>>> self.mode_filter = mode_filter
>>> self.log = None
>>> @@ -280,6 +286,8 @@ class TestHarness:
>>> if not self.list_tests:
>>> sys.stdout.write('.' * dot_count)
>>> sys.stdout.flush()
>>> + elif self.milestone_filter:
>>> + print 'WARNING: --milestone-filter option does not currently work with C tests'
>>>
>>> if os.access(progbase, os.X_OK):
>>> progname = './' + progbase
>>> @@ -349,6 +357,8 @@ class TestHarness:
>>> svntest.main.options.server_minor_version = self.server_minor_version
>>> if self.list_tests is not None:
>>> svntest.main.options.list_tests = True
>>> + if self.milestone_filter is not None:
>>> + svntest.main.options.milestone_filter = self.milestone_filter
>>> if self.svn_bin is not None:
>>> svntest.main.options.svn_bin = self.svn_bin
>>> if self.fsfs_sharding is not None:
>>>
>>> Modified: subversion/trunk/subversion/tests/cmdline/svntest/main.py
>>> URL: http://svn.apache.org/viewvc/subversion/trunk/subversion/tests/cmdline/svntest/main.py?rev=1071809&r1=1071808&r2=1071809&view=diff
>>> ==============================================================================
>>> --- subversion/trunk/subversion/tests/cmdline/svntest/main.py (original)
>>> +++ subversion/trunk/subversion/tests/cmdline/svntest/main.py Thu Feb 17 22:09:02 2011
>>> @@ -34,6 +34,8 @@ import time # for time()
>>> import traceback # for print_exc()
>>> import threading
>>> import optparse # for argument parsing
>>> +import xml
>>> +import urllib
>>>
>>> try:
>>> # Python >=3.0
>>> @@ -1132,6 +1134,8 @@ class TestSpawningThread(threading.Threa
>>> args.append('--server-minor-version=' + str(options.server_minor_version))
>>> if options.mode_filter:
>>> args.append('--mode-filter=' + options.mode_filter)
>>> + if options.milestone_filter:
>>> + args.append('--milestone-filter=' + options.milestone_filter)
>>>
>>> result, stdout_lines, stderr_lines = spawn_process(command, 0, 0, None,
>>> *args)
>>> @@ -1152,26 +1156,61 @@ class TestRunner:
>>> self.pred = svntest.testcase.create_test_case(func)
>>> self.index = index
>>>
>>> - def list(self):
>>> + def list(self, milestones_dict=None):
>>> + """Print test doc strings. MILESTONES_DICT is an optional mapping
>>> + of issue numbers to target milestones."""
>>> if options.mode_filter.upper() == 'ALL' \
>>> or options.mode_filter.upper() == self.pred.list_mode().upper() \
>>> or (options.mode_filter.upper() == 'PASS' \
>>> and self.pred.list_mode() == ''):
>>> + issues = []
>>> tail = ''
>>> if self.pred.issues:
>>> - tail += " [%s]" % ','.join(['#%d' % i for i in self.pred.issues])
>>> - if options.verbose and self.pred.inprogress:
>>> - tail += " [[%s]]" % self.pred.inprogress
>>> - else:
>>> - print(" %3d %-5s %s%s" % (self.index,
>>> - self.pred.list_mode(),
>>> - self.pred.description,
>>> - tail))
>>> + if not options.milestone_filter or milestones_dict is None:
>>> + issues = self.pred.issues
>>> + else: # Limit listing by requested target milestone(s).
>>> + filter_issues = []
>>> + matches_filter = False
>>> +
>>> + # Get the milestones for all the issues associated with this test.
>>> + # If any one of them matches the MILESTONE_FILTER then we'll print
>>> + # them all.
>>> + for issue in self.pred.issues:
>>> + # A safe starting assumption.
>>> + milestone = 'unknown'
>>> + if milestones_dict:
>>> + if milestones_dict.has_key(str(issue)):
>>> + milestone = milestones_dict[str(issue)]
>>> +
>>> + filter_issues.append(str(issue) + '(' + milestone + ')')
>>> + pattern = re.compile(options.milestone_filter)
>>> + if pattern.match(milestone):
>>> + matches_filter = True
>>> +
>>> + # Did at least one of the associated issues meet our filter?
>>> + if matches_filter:
>>> + issues = filter_issues
>>> +
>>> + tail += " [%s]" % ','.join(['#%s' % str(i) for i in issues])
>>> +
>>> + # If there is no filter or this test made if through
>>> + # the filter then print it!
>>> + if options.milestone_filter is None or len(issues):
>>> + if options.verbose and self.pred.inprogress:
>>> + tail += " [[%s]]" % self.pred.inprogress
>>> + else:
>>> + print(" %3d %-5s %s%s" % (self.index,
>>> + self.pred.list_mode(),
>>> + self.pred.description,
>>> + tail))
>>> sys.stdout.flush()
>>>
>>> def get_mode(self):
>>> return self.pred.list_mode()
>>>
>>> + def get_issues(self):
>>> + return self.pred.issues
>>> +
>>> def get_function_name(self):
>>> return self.pred.get_function_name()
>>>
>>> @@ -1376,6 +1415,8 @@ def _create_parser():
>>> parser = optparse.OptionParser(usage=usage)
>>> parser.add_option('-l', '--list', action='store_true', dest='list_tests',
>>> help='Print test doc strings instead of running them')
>>> + parser.add_option('--milestone-filter', action='store', dest='milestone_filter',
>>> + help='Limit --list to those with target milestone specified')
>>> parser.add_option('-v', '--verbose', action='store_true', dest='verbose',
>>> help='Print binary command-lines (not with --quiet)')
>>> parser.add_option('-q', '--quiet', action='store_true',
>>> @@ -1470,6 +1511,47 @@ def run_tests(test_list, serial_only = F
>>>
>>> sys.exit(execute_tests(test_list, serial_only))
>>>
>>> +def get_target_milestones_for_issues(issue_numbers):
>>> + xml_url = "http://subversion.tigris.org/issues/xml.cgi?id="
>>> + issue_dict = {}
>>> +
>>> + if isinstance(issue_numbers, int):
>>> + issue_numbers = [str(issue_numbers)]
>>> + elif isinstance(issue_numbers, str):
>>> + issue_numbers = [issue_numbers]
>>> +
>>> + if issue_numbers is None or len(issue_numbers) == 0:
>>> + return issue_dict
>>> +
>>> + for num in issue_numbers:
>>> + xml_url += str(num) + ','
>>> + issue_dict[str(num)] = 'unknown'
>>> +
>>> + try:
>>> + # Parse the xml for ISSUE_NO from the issue tracker into a Document.
>>> + issue_xml_f = urllib.urlopen(xml_url)
>>> + except:
>>> + print "WARNING: Unable to contact issue tracker; " \
>>> + "milestones defaulting to 'unknown'."
>>> + return issue_dict
>>> +
>>> + try:
>>> + xmldoc = xml.dom.minidom.parse(issue_xml_f)
>>> + issue_xml_f.close()
>>> +
>>> + # Get the target milestone for each issue.
>>> + issue_element = xmldoc.getElementsByTagName('issue')
>>> + for i in issue_element:
>>> + issue_id_element = i.getElementsByTagName('issue_id')
>>> + issue_id = issue_id_element[0].childNodes[0].nodeValue
>>> + milestone_element = i.getElementsByTagName('target_milestone')
>>> + milestone = milestone_element[0].childNodes[0].nodeValue
>>> + issue_dict[issue_id] = milestone
>>> + except:
>>> + print "ERROR: Unable to parse target milestones from issue tracker"
>>> + raise
>>> +
>>> + return issue_dict
>>>
>>> # Main func. This is the "entry point" that all the test scripts call
>>> # to run their list of tests.
>>> @@ -1586,6 +1668,23 @@ def execute_tests(test_list, serial_only
>>> testnums = list(range(1, len(test_list)))
>>>
>>> if options.list_tests:
>>> +
>>> + # If we want to list the target milestones, then get all the issues
>>> + # associated with all the individual tests.
>>> + milestones_dict = None
>>> + if options.milestone_filter:
>>> + issues_dict = {}
>>> + for testnum in testnums:
>>> + issues = TestRunner(test_list[testnum], testnum).get_issues()
>>> + test_mode = TestRunner(test_list[testnum], testnum).get_mode().upper()
>>> + if issues:
>>> + for issue in issues:
>>> + if (options.mode_filter.upper() == 'ALL' or
>>> + options.mode_filter.upper() == test_mode or
>>> + (options.mode_filter.upper() == 'PASS' and test_mode == '')):
>>> + issues_dict[issue]=issue
>>> + milestones_dict = get_target_milestones_for_issues(issues_dict.keys())
>>> +
>>> header = "Test # Mode Test Description\n" \
>>> "------ ----- ----------------"
>>> printed_header = False
>>> @@ -1597,7 +1696,7 @@ def execute_tests(test_list, serial_only
>>> if not printed_header:
>>> print header
>>> printed_header = True
>>> - TestRunner(test_list[testnum], testnum).list()
>>> + TestRunner(test_list[testnum], testnum).list(milestones_dict)
>>> # We are simply listing the tests so always exit with success.
>>> return 0
>>>
>>>
>>> Modified: subversion/trunk/win-tests.py
>>> URL: http://svn.apache.org/viewvc/subversion/trunk/win-tests.py?rev=1071809&r1=1071808&r2=1071809&view=diff
>>> ==============================================================================
>>> --- subversion/trunk/win-tests.py (original)
>>> +++ subversion/trunk/win-tests.py Thu Feb 17 22:09:02 2011
>>> @@ -79,6 +79,10 @@ def _usage_exit():
>>> print(" --http-library : dav library to use, neon (default) or serf")
>>> print(" --javahl : Run the javahl tests instead of the normal tests")
>>> print(" --list : print test doc strings only")
>>> + print(" --milestone-filter=RE : RE is a regular expression pattern that (when")
>>> + print(" used with --list) limits the tests listed to")
>>> + print(" those with an associated issue in the tracker")
>>> + print(" which has a target milestone that matches RE.")
>>> print(" --mode-filter=TYPE : limit tests to expected TYPE = XFAIL, SKIP, PASS,")
>>> print(" or 'ALL' (default)")
>>> print(" --enable-sasl : enable Cyrus SASL authentication for")
>>> @@ -123,7 +127,7 @@ opts, args = my_getopt(sys.argv[1:], 'hr
>>> 'fsfs-packing', 'fsfs-sharding=', 'javahl',
>>> 'list', 'enable-sasl', 'bin=', 'parallel',
>>> 'config-file=', 'server-minor-version=',
>>> - 'log-to-stdout', 'mode-filter='])
>>> + 'log-to-stdout', 'mode-filter=', 'milestone-filter='])
>>> if len(args) > 1:
>>> print('Warning: non-option arguments after the first one will be ignored')
>>>
>>> @@ -140,6 +144,7 @@ httpd_port = None
>>> httpd_service = None
>>> http_library = 'neon'
>>> list_tests = None
>>> +milestone_filter = None
>>> test_javahl = None
>>> enable_sasl = None
>>> svn_bin = None
>>> @@ -195,6 +200,8 @@ for opt, val in opts:
>>> test_javahl = 1
>>> elif opt == '--list':
>>> list_tests = 1
>>> + elif opt == '--milestone-filter':
>>> + milestone_filter = val
>>> elif opt == '--mode-filter':
>>> mode_filter = val
>>> elif opt == '--enable-sasl':
>>> @@ -688,7 +695,8 @@ if not test_javahl:
>>> server_minor_version, not quiet,
>>> cleanup, enable_sasl, parallel, config_file,
>>> fsfs_sharding, fsfs_packing,
>>> - list_tests, svn_bin, mode_filter)
>>> + list_tests, svn_bin, mode_filter,
>>> + milestone_filter)
>>> old_cwd = os.getcwd()
>>> try:
>>> os.chdir(abs_builddir)
>>>
>>>
>>>
>
> Please find attached patch for Makefile.in to make the same work on
> linux. I am not that proficient with make files. But still I think I did
> it right.
>
> Log
> [[[
>
> Follow-up to r1071809. Allow 'make check' to take MILESTON_FILTER and
> MODE_FILTER options.
>
> Now we can easily answer questions like, "What xfailing merge tests need to
> be fixed before we can release 1.7?"
>
> $ make check MILESTONE_FILTER="(1.7.*)|(---)" MODE_FILTER=xfail
> TESTS=subversion/tests/cmdline/merge_tests.py
>
> LISTING: merge_tests.py
> Test # Mode Test Description
> ------ ----- ----------------
> 64 XFAIL merge target with non inheritable mergeinfo
> [#2970(blue-sky),#3642(1.7.0)]
> 75 XFAIL merge added subtree [#1962(1.7-consider)]
>
> * Makefile.in
> (check): Pass --list, --milestone-filter, --mode-filter and
> --log-to-stdout to run_tests.py if MILESTONE_FILTER is set.
>
> * build/run_tests.py
> (__doc__): Add --list, --milestone-filter and --mode-filter options to
> usage doc.
>
> (main): Accept --list, --milestone-filter and --mode-filter as a valid
> option, pass it to TestHarness()
>
> Patch by: Noorul Islam K M <noorul{_AT_}collab.net>
> ]]]
>
> Index: Makefile.in
> ===================================================================
> --- Makefile.in (revision 1072234)
> +++ Makefile.in (working copy)
> @@ -473,6 +473,10 @@
> if test "$(LOG_TO_STDOUT)" != ""; then \
> flags="--log-to-stdout $$flags"; \
> fi; \
> + if test "$(MILESTONE_FILTER)" != ""; then \
> + flags="--list --milestone-filter=$(MILESTONE_FILTER) \
> + --mode-filter=$(MODE_FILTER) --log-to-stdout $$flags"; \
> + fi; \
> $(PYTHON) $(top_srcdir)/build/run_tests.py \
> --config-file $(top_srcdir)/subversion/tests/tests.conf \
> $$flags \
> Index: build/run_tests.py
> ===================================================================
> --- build/run_tests.py (revision 1072234)
> +++ build/run_tests.py (working copy)
> @@ -27,6 +27,7 @@
> [--verbose] [--log-to-stdout] [--cleanup] [--parallel]
> [--url=<base-url>] [--http-library=<http-library>] [--enable-sasl]
> [--fs-type=<fs-type>] [--fsfs-packing] [--fsfs-sharding=<n>]
> + [--list] [--milestone-filter=<regex>] [--mode-filter=<type>]
> [--server-minor-version=<version>]
> [--config-file=<file>]
> <abs_srcdir> <abs_builddir>
> @@ -522,7 +523,7 @@
> 'http-library=', 'server-minor-version=',
> 'fsfs-packing', 'fsfs-sharding=',
> 'enable-sasl', 'parallel', 'config-file=',
> - 'log-to-stdout'])
> + 'log-to-stdout', 'list', 'milestone-filter=', 'mode-filter='])
> except getopt.GetoptError:
> args = []
>
> @@ -532,9 +533,9 @@
>
> base_url, fs_type, verbose, cleanup, enable_sasl, http_library, \
> server_minor_version, fsfs_sharding, fsfs_packing, parallel, \
> - config_file, log_to_stdout = \
> + config_file, log_to_stdout, list_tests, mode_filter, milestone_filter= \
> None, None, None, None, None, None, None, None, None, None, None, \
> - None
> + None, None, None, None
> for opt, val in opts:
> if opt in ['-u', '--url']:
> base_url = val
> @@ -560,6 +561,12 @@
> config_file = val
> elif opt in ['--log-to-stdout']:
> log_to_stdout = 1
> + elif opt in ['--list']:
> + list_tests = 1
> + elif opt in ['--milestone-filter']:
> + milestone_filter = val
> + elif opt in ['--mode-filter']:
> + mode_filter = val
> else:
> raise getopt.GetoptError
>
> @@ -573,7 +580,8 @@
> th = TestHarness(args[0], args[1], logfile, faillogfile,
> base_url, fs_type, http_library, server_minor_version,
> verbose, cleanup, enable_sasl, parallel, config_file,
> - fsfs_sharding, fsfs_packing)
> + fsfs_sharding, fsfs_packing, list_tests,
> + mode_filter=mode_filter, milestone_filter=milestone_filter)
>
> failed = th.run(args[2:])
> if failed:
Pinging to get some attention to this thread.
Thanks and Regards
Noorul
Received on 2011-02-24 04:00:09 CET