Re: [PATCH] Re: svn commit: r1071809 - in /subversion/trunk: build/run_tests.py subversion/tests/cmdline/svntest/main.py win-tests.py

On Wed, Feb 23, 2011 at 9:59 PM, Noorul Islam K M <noorul_at_collab.net> wrote:
> Noorul Islam K M <noorul_at_collab.net> writes:
>
>> Paul Burba <ptburba_at_gmail.com> writes:
>>
>>> If someone with the requisite linux skills/hardware could tweak
>>> makefile.in so it can take advantage of the --milestone-filter option,
>>> well that would be fabulous.
>>>
>>> Paul
>>>
>>> On Thu, Feb 17, 2011 at 5:09 PM,  <pburba_at_apache.org> wrote:
>>>> Author: pburba
>>>> Date: Thu Feb 17 22:09:02 2011
>>>> New Revision: 1071809
>>>>
>>>> URL: http://svn.apache.org/viewvc?rev=1071809&view=rev
>>>> Log:
>>>> Add an option (--milestone-filter=REGEX) to the Python tests so we can list a
>>>> subset of the tests based on their associated issues' target milestone.
>>>>
>>>> This option is currently only available to win-tests.py and
>>>> subversion/tests/cmdline/svntest/main.py.  So it isn't quite as useful
>>>> on non-Windows platforms just yet.
>>>>
>>>> Now we can easily answer questions like, "What xfailing merge tests need to
>>>> be fixed before we can release 1.7?"
>>>>
>>>>  C:\SVN\src-trunk>win-tests.py --list --milestone-filter="(1.7.*)|(---)"
>>>>  --mode-filter xfail --log-to-stdout --test merge
>>>>  Listing Debug configuration on local repository.
>>>>  LISTING: merge_tests.py
>>>>  Test #  Mode   Test Description
>>>>  ------  -----  ----------------
>>>>    64    XFAIL  merge target with non inheritable mergeinfo
>>>>  [#2970(blue-sky),#3642(1.7.0)]
>>>>    75    XFAIL  merge added subtree [#1962(1.7-consider)]
>>>>
>>>> * build/run_tests.py
>>>>
>>>>  (TestHarness.__init__): Add mode_filter argument.
>>>>
>>>>  (TestHarness._run_c_test): Issue warning that --milestone-filter doesn't
>>>>   work when listing C tests.
>>>>
>>>>  (TestHarness._run_py_test): Accept --milestone-filter option.
>>>>
>>>> * subversion/tests/cmdline/svntest/main.py
>>>>
>>>>  (global): Import xml and urllib.
>>>>
>>>>  (TestSpawningThread.run_one): Support --milestone-filter option.
>>>>
>>>>  (TestRunner.list): Add optional argument mapping issues to target
>>>>   milestones.
>>>>
>>>>  (TestRunner.get_issues): New.
>>>>
>>>>  (_create_parser): Handle --milestone-filter.
>>>>
>>>>  (get_target_milestones_for_issues): New.
>>>>
>>>>  (execute_tests): Handle --milestone-filter.
>>>>
>>>> * win-tests.py
>>>>
>>>>  (_usage_exit): Add --milestone-filter to usage text.
>>>>
>>>>  (milestone_filter): New global variable.
>>>>
>>>>  (global): Accept --milestone-filter as a valid option, pass it to
>>>>   run_tests.TestHarness().
>>>>
>>>>
>>>> Modified:
>>>>    subversion/trunk/build/run_tests.py
>>>>    subversion/trunk/subversion/tests/cmdline/svntest/main.py
>>>>    subversion/trunk/win-tests.py
>>>>
>>>> Modified: subversion/trunk/build/run_tests.py
>>>> URL: http://svn.apache.org/viewvc/subversion/trunk/build/run_tests.py?rev=1071809&r1=1071808&r2=1071809&view=diff
>>>> ==============================================================================
>>>> --- subversion/trunk/build/run_tests.py (original)
>>>> +++ subversion/trunk/build/run_tests.py Thu Feb 17 22:09:02 2011
>>>> @@ -79,7 +79,8 @@ class TestHarness:
>>>>                server_minor_version=None, verbose=None,
>>>>                cleanup=None, enable_sasl=None, parallel=None, config_file=None,
>>>>                fsfs_sharding=None, fsfs_packing=None,
>>>> -               list_tests=None, svn_bin=None, mode_filter=None):
>>>> +               list_tests=None, svn_bin=None, mode_filter=None,
>>>> +               milestone_filter=None):
>>>>     '''Construct a TestHarness instance.
>>>>
>>>>     ABS_SRCDIR and ABS_BUILDDIR are the source and build directories.
>>>> @@ -91,8 +92,12 @@ class TestHarness:
>>>>     HTTP_LIBRARY is the HTTP library for DAV-based communications.
>>>>     SERVER_MINOR_VERSION is the minor version of the server being tested.
>>>>     SVN_BIN is the path where the svn binaries are installed.
>>>> -    mode_filter restricts the TestHarness to tests with the expected mode
>>>> -    XFail, Skip, Pass, or All tests (default).
>>>> +    MODE_FILTER restricts the TestHarness to tests with the expected mode
>>>> +    XFail, Skip, Pass, or All tests (default).  MILESTONE_FILTER is a
>>>> +    string representation of a valid regular expression pattern; when used
>>>> +    in conjunction with LIST_TESTS, the only tests that are listed are
>>>> +    those with an associated issue in the tracker which has a target
>>>> +    milestone that matches the regex.
>>>>     '''
>>>>     self.srcdir = abs_srcdir
>>>>     self.builddir = abs_builddir
>>>> @@ -114,6 +119,7 @@ class TestHarness:
>>>>     if config_file is not None:
>>>>       self.config_file = os.path.abspath(config_file)
>>>>     self.list_tests = list_tests
>>>> +    self.milestone_filter = milestone_filter
>>>>     self.svn_bin = svn_bin
>>>>     self.mode_filter = mode_filter
>>>>     self.log = None
>>>> @@ -280,6 +286,8 @@ class TestHarness:
>>>>     if not self.list_tests:
>>>>       sys.stdout.write('.' * dot_count)
>>>>       sys.stdout.flush()
>>>> +    elif self.milestone_filter:
>>>> +      print 'WARNING: --milestone-filter option does not currently work with C tests'
>>>>
>>>>     if os.access(progbase, os.X_OK):
>>>>       progname = './' + progbase
>>>> @@ -349,6 +357,8 @@ class TestHarness:
>>>>       svntest.main.options.server_minor_version = self.server_minor_version
>>>>     if self.list_tests is not None:
>>>>       svntest.main.options.list_tests = True
>>>> +    if self.milestone_filter is not None:
>>>> +      svntest.main.options.milestone_filter = self.milestone_filter
>>>>     if self.svn_bin is not None:
>>>>       svntest.main.options.svn_bin = self.svn_bin
>>>>     if self.fsfs_sharding is not None:
>>>>
>>>> Modified: subversion/trunk/subversion/tests/cmdline/svntest/main.py
>>>> URL: http://svn.apache.org/viewvc/subversion/trunk/subversion/tests/cmdline/svntest/main.py?rev=1071809&r1=1071808&r2=1071809&view=diff
>>>> ==============================================================================
>>>> --- subversion/trunk/subversion/tests/cmdline/svntest/main.py (original)
>>>> +++ subversion/trunk/subversion/tests/cmdline/svntest/main.py Thu Feb 17 22:09:02 2011
>>>> @@ -34,6 +34,8 @@ import time    # for time()
>>>>  import traceback # for print_exc()
>>>>  import threading
>>>>  import optparse # for argument parsing
>>>> +import xml
>>>> +import urllib
>>>>
>>>>  try:
>>>>   # Python >=3.0
>>>> @@ -1132,6 +1134,8 @@ class TestSpawningThread(threading.Threa
>>>>       args.append('--server-minor-version=' + str(options.server_minor_version))
>>>>     if options.mode_filter:
>>>>       args.append('--mode-filter=' + options.mode_filter)
>>>> +    if options.milestone_filter:
>>>> +      args.append('--milestone-filter=' + options.milestone_filter)
>>>>
>>>>     result, stdout_lines, stderr_lines = spawn_process(command, 0, 0, None,
>>>>                                                        *args)
>>>> @@ -1152,26 +1156,61 @@ class TestRunner:
>>>>     self.pred = svntest.testcase.create_test_case(func)
>>>>     self.index = index
>>>>
>>>> -  def list(self):
>>>> +  def list(self, milestones_dict=None):
>>>> +    """Print test doc strings.  MILESTONES_DICT is an optional mapping
>>>> +    of issue numbers to target milestones."""
>>>>     if options.mode_filter.upper() == 'ALL' \
>>>>        or options.mode_filter.upper() == self.pred.list_mode().upper() \
>>>>        or (options.mode_filter.upper() == 'PASS' \
>>>>            and self.pred.list_mode() == ''):
>>>> +      issues = []
>>>>       tail = ''
>>>>       if self.pred.issues:
>>>> -        tail += " [%s]" % ','.join(['#%d' % i for i in self.pred.issues])
>>>> -      if options.verbose and self.pred.inprogress:
>>>> -        tail += " [[%s]]" % self.pred.inprogress
>>>> -      else:
>>>> -        print(" %3d    %-5s  %s%s" % (self.index,
>>>> -                                      self.pred.list_mode(),
>>>> -                                      self.pred.description,
>>>> -                                      tail))
>>>> +        if not options.milestone_filter or milestones_dict is None:
>>>> +          issues = self.pred.issues
>>>> +        else: # Limit listing by requested target milestone(s).
>>>> +          filter_issues = []
>>>> +          matches_filter = False
>>>> +
>>>> +          # Get the milestones for all the issues associated with this test.
>>>> +          # If any one of them matches the MILESTONE_FILTER then we'll print
>>>> +          # them all.
>>>> +          for issue in self.pred.issues:
>>>> +            # A safe starting assumption.
>>>> +            milestone = 'unknown'
>>>> +            if milestones_dict:
>>>> +              if milestones_dict.has_key(str(issue)):
>>>> +                milestone = milestones_dict[str(issue)]
>>>> +
>>>> +            filter_issues.append(str(issue) + '(' + milestone + ')')
>>>> +            pattern = re.compile(options.milestone_filter)
>>>> +            if pattern.match(milestone):
>>>> +              matches_filter = True
>>>> +
>>>> +          # Did at least one of the associated issues meet our filter?
>>>> +          if matches_filter:
>>>> +            issues = filter_issues
>>>> +
>>>> +        tail += " [%s]" % ','.join(['#%s' % str(i) for i in issues])
>>>> +
>>>> +      # If there is no filter or this test made if through
>>>> +      # the filter then print it!
>>>> +      if options.milestone_filter is None or len(issues):
>>>> +        if options.verbose and self.pred.inprogress:
>>>> +          tail += " [[%s]]" % self.pred.inprogress
>>>> +        else:
>>>> +          print(" %3d    %-5s  %s%s" % (self.index,
>>>> +                                        self.pred.list_mode(),
>>>> +                                        self.pred.description,
>>>> +                                        tail))
>>>>     sys.stdout.flush()
>>>>
>>>>   def get_mode(self):
>>>>     return self.pred.list_mode()
>>>>
>>>> +  def get_issues(self):
>>>> +    return self.pred.issues
>>>> +
>>>>   def get_function_name(self):
>>>>     return self.pred.get_function_name()
>>>>
>>>> @@ -1376,6 +1415,8 @@ def _create_parser():
>>>>   parser = optparse.OptionParser(usage=usage)
>>>>   parser.add_option('-l', '--list', action='store_true', dest='list_tests',
>>>>                     help='Print test doc strings instead of running them')
>>>> +  parser.add_option('--milestone-filter', action='store', dest='milestone_filter',
>>>> +                    help='Limit --list to those with target milestone specified')
>>>>   parser.add_option('-v', '--verbose', action='store_true', dest='verbose',
>>>>                     help='Print binary command-lines (not with --quiet)')
>>>>   parser.add_option('-q', '--quiet', action='store_true',
>>>> @@ -1470,6 +1511,47 @@ def run_tests(test_list, serial_only = F
>>>>
>>>>   sys.exit(execute_tests(test_list, serial_only))
>>>>
>>>> +def get_target_milestones_for_issues(issue_numbers):
>>>> +  xml_url = "http://subversion.tigris.org/issues/xml.cgi?id="
>>>> +  issue_dict = {}
>>>> +
>>>> +  if isinstance(issue_numbers, int):
>>>> +    issue_numbers = [str(issue_numbers)]
>>>> +  elif isinstance(issue_numbers, str):
>>>> +    issue_numbers = [issue_numbers]
>>>> +
>>>> +  if issue_numbers is None or len(issue_numbers) == 0:
>>>> +    return issue_dict
>>>> +
>>>> +  for num in issue_numbers:
>>>> +    xml_url += str(num) + ','
>>>> +    issue_dict[str(num)] = 'unknown'
>>>> +
>>>> +  try:
>>>> +    # Parse the xml for ISSUE_NO from the issue tracker into a Document.
>>>> +    issue_xml_f = urllib.urlopen(xml_url)
>>>> +  except:
>>>> +    print "WARNING: Unable to contact issue tracker; " \
>>>> +          "milestones defaulting to 'unknown'."
>>>> +    return issue_dict
>>>> +
>>>> +  try:
>>>> +    xmldoc = xml.dom.minidom.parse(issue_xml_f)
>>>> +    issue_xml_f.close()
>>>> +
>>>> +    # Get the target milestone for each issue.
>>>> +    issue_element = xmldoc.getElementsByTagName('issue')
>>>> +    for i in issue_element:
>>>> +      issue_id_element = i.getElementsByTagName('issue_id')
>>>> +      issue_id = issue_id_element[0].childNodes[0].nodeValue
>>>> +      milestone_element = i.getElementsByTagName('target_milestone')
>>>> +      milestone = milestone_element[0].childNodes[0].nodeValue
>>>> +      issue_dict[issue_id] = milestone
>>>> +  except:
>>>> +    print "ERROR: Unable to parse target milestones from issue tracker"
>>>> +    raise
>>>> +
>>>> +  return issue_dict
>>>>
>>>>  # Main func.  This is the "entry point" that all the test scripts call
>>>>  # to run their list of tests.
>>>> @@ -1586,6 +1668,23 @@ def execute_tests(test_list, serial_only
>>>>     testnums = list(range(1, len(test_list)))
>>>>
>>>>   if options.list_tests:
>>>> +
>>>> +    # If we want to list the target milestones, then get all the issues
>>>> +    # associated with all the individual tests.
>>>> +    milestones_dict = None
>>>> +    if options.milestone_filter:
>>>> +      issues_dict = {}
>>>> +      for testnum in testnums:
>>>> +        issues = TestRunner(test_list[testnum], testnum).get_issues()
>>>> +        test_mode = TestRunner(test_list[testnum], testnum).get_mode().upper()
>>>> +        if issues:
>>>> +          for issue in issues:
>>>> +            if (options.mode_filter.upper() == 'ALL' or
>>>> +                options.mode_filter.upper() == test_mode or
>>>> +                (options.mode_filter.upper() == 'PASS' and test_mode == '')):
>>>> +              issues_dict[issue]=issue
>>>> +      milestones_dict = get_target_milestones_for_issues(issues_dict.keys())
>>>> +
>>>>     header = "Test #  Mode   Test Description\n" \
>>>>              "------  -----  ----------------"
>>>>     printed_header = False
>>>> @@ -1597,7 +1696,7 @@ def execute_tests(test_list, serial_only
>>>>         if not printed_header:
>>>>           print header
>>>>           printed_header = True
>>>> -        TestRunner(test_list[testnum], testnum).list()
>>>> +        TestRunner(test_list[testnum], testnum).list(milestones_dict)
>>>>     # We are simply listing the tests so always exit with success.
>>>>     return 0
>>>>
>>>>
>>>> Modified: subversion/trunk/win-tests.py
>>>> URL: http://svn.apache.org/viewvc/subversion/trunk/win-tests.py?rev=1071809&r1=1071808&r2=1071809&view=diff
>>>> ==============================================================================
>>>> --- subversion/trunk/win-tests.py (original)
>>>> +++ subversion/trunk/win-tests.py Thu Feb 17 22:09:02 2011
>>>> @@ -79,6 +79,10 @@ def _usage_exit():
>>>>   print("  --http-library         : dav library to use, neon (default) or serf")
>>>>   print("  --javahl               : Run the javahl tests instead of the normal tests")
>>>>   print("  --list                 : print test doc strings only")
>>>> +  print("  --milestone-filter=RE  : RE is a regular expression pattern that (when")
>>>> +  print("                           used with --list) limits the tests listed to")
>>>> +  print("                           those with an associated issue in the tracker")
>>>> +  print("                           which has a target milestone that matches RE.")
>>>>   print("  --mode-filter=TYPE     : limit tests to expected TYPE = XFAIL, SKIP, PASS,")
>>>>   print("                           or 'ALL' (default)")
>>>>   print("  --enable-sasl          : enable Cyrus SASL authentication for")
>>>> @@ -123,7 +127,7 @@ opts, args = my_getopt(sys.argv[1:], 'hr
>>>>                         'fsfs-packing', 'fsfs-sharding=', 'javahl',
>>>>                         'list', 'enable-sasl', 'bin=', 'parallel',
>>>>                         'config-file=', 'server-minor-version=',
>>>> -                        'log-to-stdout', 'mode-filter='])
>>>> +                        'log-to-stdout', 'mode-filter=', 'milestone-filter='])
>>>>  if len(args) > 1:
>>>>   print('Warning: non-option arguments after the first one will be ignored')
>>>>
>>>> @@ -140,6 +144,7 @@ httpd_port = None
>>>>  httpd_service = None
>>>>  http_library = 'neon'
>>>>  list_tests = None
>>>> +milestone_filter = None
>>>>  test_javahl = None
>>>>  enable_sasl = None
>>>>  svn_bin = None
>>>> @@ -195,6 +200,8 @@ for opt, val in opts:
>>>>     test_javahl = 1
>>>>   elif opt == '--list':
>>>>     list_tests = 1
>>>> +  elif opt == '--milestone-filter':
>>>> +    milestone_filter = val
>>>>   elif opt == '--mode-filter':
>>>>     mode_filter = val
>>>>   elif opt == '--enable-sasl':
>>>> @@ -688,7 +695,8 @@ if not test_javahl:
>>>>                              server_minor_version, not quiet,
>>>>                              cleanup, enable_sasl, parallel, config_file,
>>>>                              fsfs_sharding, fsfs_packing,
>>>> -                             list_tests, svn_bin, mode_filter)
>>>> +                             list_tests, svn_bin, mode_filter,
>>>> +                             milestone_filter)
>>>>   old_cwd = os.getcwd()
>>>>   try:
>>>>     os.chdir(abs_builddir)
>>>>
>>>>
>>>>
>>
>> Please find attached patch for Makefile.in to make the same work on
>> linux. I am not that proficient with make files. But still I think I did
>> it right.
>>
>> Log
>> [[[
>>
>> Follow-up to r1071809. Allow 'make check' to take MILESTON_FILTER and
>> MODE_FILTER options.
>>
>> Now we can easily answer questions like, "What xfailing merge tests need to
>> be fixed before we can release 1.7?"
>>
>> $ make check MILESTONE_FILTER="(1.7.*)|(---)" MODE_FILTER=xfail
>> TESTS=subversion/tests/cmdline/merge_tests.py
>>
>>   LISTING: merge_tests.py
>>   Test #  Mode   Test Description
>>   ------  -----  ----------------
>>     64    XFAIL  merge target with non inheritable mergeinfo
>>   [#2970(blue-sky),#3642(1.7.0)]
>>     75    XFAIL  merge added subtree [#1962(1.7-consider)]
>>
>> * Makefile.in
>>   (check): Pass --list, --milestone-filter, --mode-filter and
>>     --log-to-stdout to run_tests.py if MILESTONE_FILTER is set.
>>
>> * build/run_tests.py
>>   (__doc__): Add --list, --milestone-filter and --mode-filter options to
>>     usage doc.
>>
>>   (main): Accept --list, --milestone-filter and --mode-filter as a valid
>>    option, pass it to TestHarness()
>>
>> Patch by: Noorul Islam K M <noorul{_AT_}collab.net>
>> ]]]
>>
>> Index: Makefile.in
>> ===================================================================
>> --- Makefile.in       (revision 1072234)
>> +++ Makefile.in       (working copy)
>> @@ -473,6 +473,10 @@
>>         if test "$(LOG_TO_STDOUT)" != ""; then                             \
>>           flags="--log-to-stdout $$flags";                                 \
>>         fi;                                                                \
>> +       if test "$(MILESTONE_FILTER)" != ""; then                          \
>> +         flags="--list --milestone-filter=$(MILESTONE_FILTER)             \
>> +                --mode-filter=$(MODE_FILTER) --log-to-stdout $$flags";    \
>> +       fi;                                                                \
>>         $(PYTHON) $(top_srcdir)/build/run_tests.py                         \
>>                   --config-file $(top_srcdir)/subversion/tests/tests.conf  \
>>                   $$flags                                                  \
>> Index: build/run_tests.py
>> ===================================================================
>> --- build/run_tests.py        (revision 1072234)
>> +++ build/run_tests.py        (working copy)
>> @@ -27,6 +27,7 @@
>>              [--verbose] [--log-to-stdout] [--cleanup] [--parallel]
>>              [--url=<base-url>] [--http-library=<http-library>] [--enable-sasl]
>>              [--fs-type=<fs-type>] [--fsfs-packing] [--fsfs-sharding=<n>]
>> +            [--list] [--milestone-filter=<regex>] [--mode-filter=<type>]
>>              [--server-minor-version=<version>]
>>              [--config-file=<file>]
>>              <abs_srcdir> <abs_builddir>
>> @@ -522,7 +523,7 @@
>>                              'http-library=', 'server-minor-version=',
>>                              'fsfs-packing', 'fsfs-sharding=',
>>                              'enable-sasl', 'parallel', 'config-file=',
>> -                            'log-to-stdout'])
>> +                            'log-to-stdout', 'list', 'milestone-filter=', 'mode-filter='])
>>    except getopt.GetoptError:
>>      args = []
>>
>> @@ -532,9 +533,9 @@
>>
>>    base_url, fs_type, verbose, cleanup, enable_sasl, http_library, \
>>      server_minor_version, fsfs_sharding, fsfs_packing, parallel, \
>> -    config_file, log_to_stdout = \
>> +    config_file, log_to_stdout, list_tests, mode_filter, milestone_filter= \
>>              None, None, None, None, None, None, None, None, None, None, None, \
>> -            None
>> +            None, None, None, None
>>    for opt, val in opts:
>>      if opt in ['-u', '--url']:
>>        base_url = val
>> @@ -560,6 +561,12 @@
>>        config_file = val
>>      elif opt in ['--log-to-stdout']:
>>        log_to_stdout = 1
>> +    elif opt in ['--list']:
>> +      list_tests = 1
>> +    elif opt in ['--milestone-filter']:
>> +      milestone_filter = val
>> +    elif opt in ['--mode-filter']:
>> +      mode_filter = val
>>      else:
>>        raise getopt.GetoptError
>>
>> @@ -573,7 +580,8 @@
>>    th = TestHarness(args[0], args[1], logfile, faillogfile,
>>                     base_url, fs_type, http_library, server_minor_version,
>>                     verbose, cleanup, enable_sasl, parallel, config_file,
>> -                   fsfs_sharding, fsfs_packing)
>> +                   fsfs_sharding, fsfs_packing, list_tests,
>> +                   mode_filter=mode_filter, milestone_filter=milestone_filter)
>>
>>    failed = th.run(args[2:])
>>    if failed:
>
> Pinging to get some attention to this thread.

Hi Noorul,

I can't evaluate this patch since I don't have a linux box.

This may be a silly question, but have you tried it out? Can you
generate a listing of xfailing tests that have associated issues with
the 1.7 milestone for example?

Paul

> Thanks and Regards
> Noorul
>
Received on 2011-02-27 20:28:23 CET