Completed implementation making run_ not required.

RobBagby · RobBagby · commit 7501f31062a8 · 2020-05-06T13:40:29.000-07:00
diff --git a/README.md b/README.md
@@ -108,13 +108,13 @@ test_name (19.43149897100011 seconds)
 
 ### Test Cases
 
-A test fixture can contain 1 or mote test cases.  Test cases are discovered when execute_tests() is called on the test fixture.  Every test case is comprised of 2 required and 2 optional methods and are discovered by the following convention: prefix_testname, where valid prefixes are: before_, run_, assertion_, and after_.  A test fixture that has run_fred and assertion_fred methods has 1 test case called 'fred'.  The following are details about test case methods:  
+A test fixture can contain 1 or mote test cases.  Test cases are discovered when execute_tests() is called on the test fixture.  Every test case is comprised of 1 required and 3 optional methods and are discovered by the following convention: prefix_testname, where valid prefixes are: before_, run_, assertion_, and after_.  A test fixture that has run_fred and assertion_fred methods has 1 test case called 'fred'.  The following are details about test case methods:  
 
 * _before\_(testname)_ - (optional) - if provided, is run prior to the 'run_' method.  This method can be used to setup any test pre-conditions
 
-* _run\_(testname)_ - (required) - run after 'before_' if before was provided, otherwise run first.  This method typically runs the notebook under test
+* _run\_(testname)_ - (optional) - if provider, is run after 'before_' if before was provided, otherwise run first.  This method is typically used to run the notebook under test
 
-* _assertion\_(testname)_ (required) - run after 'run_'.  This method typically contains the test assertions
+* _assertion\_(testname)_ (required) - run after 'run_', if run was provided.  This method typically contains the test assertions
 
 __Note:__  You can assert test scenarios using the standard ``` assert ``` statement or the assertion capabilities from a package of your choice.
 
@@ -144,7 +144,7 @@ print(result.to_string())
 
 ### before_all and after_all
 
-Test Fixtures also can have a before_all() method which is run prior to all tests and an after_all() which is run after all tests.  
+Test Fixtures also can have a before_all() method which is run prior to all tests and an after_all() which is run after all tests.
 
 ``` Python
 from runtime.nutterfixture import NutterFixture, tag
@@ -162,6 +162,42 @@ class MultiTestFixture(NutterFixture):
       …
 ```
 
+### Multiple test assertions pattern with before_all
+
+It is possible to support multiple assertions for a test by implementing a before_all method, no run methods and multiple assertion methods.  In this pattern, the before_all method runs the notebook under test.  There are no run methods.  The assertion methods simply assert against what was done in before_all. 
+
+``` Python
+from runtime.nutterfixture import NutterFixture, tag
+class MultiTestFixture(NutterFixture):
+   def before_all(self):
+     dbutils.notebook.run('notebook_under_test', 600, args) 
+      …
+
+   def assertion_test_case_1(self):
+      …
+
+   def assertion_test_case_2(self):
+     …
+
+   def after_all(self):
+      …
+```
+
+### Guaranteed test order
+
+After test cases are loaded, Nutter uses a sorted dictionary to order them by name.  Therefore test cases will be executed in alphabetical order.
+
+### Sharing state between test cases
+
+It is possible to share state across test cases via instance variables.  Generally, these should be set in the constructor.  Please see below:
+
+```Python
+class TestFixture(NutterFixture):
+  def __init__(self):
+    self.file = '/data/myfile'
+    NutterFixture.__init__(self)
+```
+
 ## Nutter CLI
 
 The Nutter CLI is a command line interface that allows you to execute and list tests via a Command Prompt.
diff --git a/cli/nuttercli.py b/cli/nuttercli.py
@@ -17,7 +17,7 @@
 from .reportsman import ReportWriters
 from . import reportsman as reports
 
-__version__ = '0.1.32'
+__version__ = '0.1.33'
 
 BUILD_NUMBER_ENV_VAR = 'NUTTER_BUILD_NUMBER'
 
diff --git a/runtime/nutterfixture.py b/runtime/nutterfixture.py
@@ -8,6 +8,7 @@
 from common.testresult import TestResults
 from .fixtureloader import FixtureLoader
 from common.testexecresults import TestExecResults
+from collections import OrderedDict
 
 
 def tag(the_tag):
@@ -57,7 +58,7 @@ def __load_fixture(self):
         if test_case_dict is None:
             logging.fatal("Invalid Test Fixture")
             raise InvalidTestFixtureException("Invalid Test Fixture")
-        self.__test_case_dict = test_case_dict
+        self.__test_case_dict = OrderedDict(sorted(test_case_dict.items(), key=lambda t: t[0]))
 
         logging.debug("Found {} test cases".format(len(test_case_dict)))
         for key, value in self.__test_case_dict.items():
diff --git a/runtime/testcase.py b/runtime/testcase.py
@@ -17,8 +17,6 @@ def get_testcase(test_name):
 
 
 class TestCase():
-    ERROR_MESSAGE_RUN_MISSING = """ TestCase does not contain a run function.
-                                      Please pass a function to set_run"""
     ERROR_MESSAGE_ASSERTION_MISSING = """ TestCase does not contain an assertion function.
                                             Please pass a function to set_assertion """
 
@@ -27,6 +25,7 @@ def __init__(self, test_name):
         self.before = None
         self.__before_set = False
         self.run = None
+        self.__run_set = False
         self.assertion = None
         self.after = None
         self.__after_set = False
@@ -39,6 +38,7 @@ def set_before(self, before):
 
     def set_run(self, run):
         self.run = run
+        self.__run_set = True
 
     def set_assertion(self, assertion):
         self.assertion = assertion
@@ -60,7 +60,8 @@ def execute_test(self):
                     "Both a run and an assertion are required for every test")
             if self.__before_set and self.before is not None:
                 self.before()
-            self.run()
+            if self.__run_set:
+                self.run()
             self.assertion()
             if self.__after_set and self.after is not None:
                 self.after()
@@ -76,10 +77,6 @@ def execute_test(self):
     def is_valid(self):
         is_valid = True
 
-        if self.run is None:
-            self.__add_message_to_error(self.ERROR_MESSAGE_RUN_MISSING)
-            is_valid = False
-
         if self.assertion is None:
             self.__add_message_to_error(self.ERROR_MESSAGE_ASSERTION_MISSING)
             is_valid = False
diff --git a/tests/runtime/test_fixtureloader.py b/tests/runtime/test_fixtureloader.py
@@ -134,6 +134,29 @@ def test__load_fixture__two_assertion_one_run_method__adds_two_testclass_to_dict
     __assert_test_case_from_dict(loaded_fixture, test_name_1, True, False, False, True)
     __assert_test_case_from_dict(loaded_fixture, test_name_2, True, True, False, True)
 
+def test__load_fixture__three_assertion_methods__adds_three_testclass_to_dictionary():
+    # Arrange
+    test_name_1 = "fred"
+    test_name_2 = "hank"
+    test_name_3 = "bert"
+    new_class = TestNutterFixtureBuilder() \
+        .with_name("MyClass") \
+        .with_assertion(test_name_1) \
+        .with_assertion(test_name_2) \
+        .with_assertion(test_name_3) \
+        .build()
+
+    loader = FixtureLoader()
+
+    # Act 
+    loaded_fixture = loader.load_fixture(new_class())
+
+    # Assert
+    assert len(loaded_fixture) == 3
+    __assert_test_case_from_dict(loaded_fixture, test_name_1, True, True, False, True)
+    __assert_test_case_from_dict(loaded_fixture, test_name_2, True, True, False, True)
+    __assert_test_case_from_dict(loaded_fixture, test_name_3, True, True, False, True)
+
 def test__load_fixture__three_with_all_methods__adds_three_testclass_to_dictionary():
     # Arrange
     test_name_1 = "fred"
@@ -166,7 +189,6 @@ def test__load_fixture__three_with_all_methods__adds_three_testclass_to_dictiona
     __assert_test_case_from_dict(loaded_fixture, test_name_2, False, False, False, False)
     __assert_test_case_from_dict(loaded_fixture, test_name_3, False, False, False, False)
 
-
 def __assert_test_case_from_dict(test_case_dict, expected_name, before_none, run_none, assertion_none, after_none):
     assert expected_name in test_case_dict
 
diff --git a/tests/runtime/test_nutterfixture.py b/tests/runtime/test_nutterfixture.py
@@ -6,6 +6,7 @@
 import pytest
 from runtime.nutterfixture import NutterFixture, tag, InvalidTestFixtureException
 from runtime.testcase import TestCase
+from runtime.fixtureloader import FixtureLoader
 from common.testresult  import TestResult, TestResults
 from tests.runtime.testnutterfixturebuilder import TestNutterFixtureBuilder
 from common.apiclientresults import ExecuteNotebookResult
@@ -228,6 +229,15 @@ def test__execute_tests__two_test_cases__returns_test_results_with_2_test_result
     # Assert 
     assert len(result.test_results.results) == 2
 
+def test__execute_tests__test_names_not_in_order_in_class__tests_executed_in_alphabetical_order():
+    # Arrange
+    fix = OutOfOrderTestFixture()
+
+    # Act
+    fix.execute_tests()
+
+    # Assert
+    assert '1wxyz' == fix.get_method_order()
 
 def test__run_test_method__has_list_tag_decorator__list_set_on_method():
     # Arrange
@@ -296,7 +306,8 @@ def assertion_test(self):
 
 def __get_test_case(name, setrun, setassert):
     tc = TestCase(name)
-    tc.set_run(setrun)
+    if setrun != None:
+        tc.set_run(setrun)
     tc.set_assertion(setassert)
 
     return tc
@@ -338,3 +349,30 @@ def run_test_with_valid_decorator(self):
     def run_test_with_invalid_decorator(self):
         pass
 
+class OutOfOrderTestFixture(NutterFixture):
+    def __init__(self):
+        super(OutOfOrderTestFixture, self).__init__()
+        self.__method_order = ''
+
+    def assertion_y(self):
+        self.__method_order += 'y'
+        assert 1 == 1
+
+    def assertion_z(self):
+        self.__method_order += 'z'
+        assert 1 == 1
+
+    def assertion_1(self):
+        self.__method_order += '1'
+        assert 1 == 1
+        
+    def assertion_w(self):
+        self.__method_order += 'w'
+        assert 1 == 1
+
+    def assertion_x(self):
+        self.__method_order += 'x'
+        assert 1 == 1
+    
+    def get_method_order(self):
+        return self.__method_order
diff --git a/tests/runtime/test_nutterfixure_fullroundtriptests.py b/tests/runtime/test_nutterfixure_fullroundtriptests.py
@@ -154,6 +154,30 @@ def test__execute_tests__one_test_case_with_all_methods__all_methods_called(mock
     test_fixture.after_test.assert_called_once_with()
     test_fixture.after_all.assert_called_once_with()
 
+def test__execute_tests__one_beforeall_2_assertions__all_methods_called(mocker):
+    # Arrange
+    test_name_1 = "test"
+    test_name_2 = "test2"
+    
+    test_fixture = TestNutterFixtureBuilder() \
+        .with_name("MyClass") \
+        .with_before_all() \
+        .with_assertion(test_name_1) \
+        .with_assertion(test_name_2) \
+        .build()
+
+    mocker.patch.object(test_fixture, 'before_all')
+    mocker.patch.object(test_fixture, 'assertion_test')
+    mocker.patch.object(test_fixture, 'assertion_test2')
+
+    # Act 
+    result = test_fixture().execute_tests()
+
+    # Assert
+    test_fixture.before_all.assert_called_once_with()
+    test_fixture.assertion_test.assert_called_once_with()
+    test_fixture.assertion_test.assert_called_once_with()
+
 def __item_in_list_equalto(list, expected_item):
     for item in list:
         if (item == expected_item):
diff --git a/tests/runtime/test_testcase.py b/tests/runtime/test_testcase.py
@@ -10,7 +10,7 @@
 from runtime.nutterfixture import tag
 from runtime.testcase import TestCase, NoTestCasesFoundError
 
-def test__isvalid_rundoesntexist_returnsfalse():
+def test__isvalid_assertionexistsrundoesntexist_returnstrue():
     # Arrange
     tc = TestCase("Test Name")
     fixture = TestFixture()
@@ -21,14 +21,16 @@ def test__isvalid_rundoesntexist_returnsfalse():
     isvalid = tc.is_valid()
 
     # Assert
-    assert False == isvalid 
+    assert True == isvalid 
 
 def test__isvalid_assertiondoesntexist_returnsfalse():
     # Arrange
     tc = TestCase("Test Name")
     fixture = TestFixture()
     
+    tc.set_before(fixture.before_test)
     tc.set_run(fixture.run_test)
+    tc.set_after(fixture.after_test)
 
     # Act
     isvalid = tc.is_valid()
@@ -50,21 +52,6 @@ def test__isvalid_runandassertionexist_returnstrue():
     # Assert
     assert True == isvalid
 
-def test__getinvalidmessage_rundoesntexist_returnsrunerrormessage():
-    # Arrange
-    tc = TestCase("Test Name")
-    fixture = TestFixture()
-    
-    tc.set_assertion(fixture.assertion_test)
-
-    expected_message = tc.ERROR_MESSAGE_RUN_MISSING 
-
-    # Act
-    invalid_message = tc.get_invalid_message()
-
-    # Assert
-    assert expected_message == invalid_message
-
 def test__getinvalidmessage_assertiondoesntexist_returnsassertionerrormessage():
     # Arrange
     tc = TestCase("Test Name")
@@ -80,21 +67,6 @@ def test__getinvalidmessage_assertiondoesntexist_returnsassertionerrormessage():
     # Assert
     assert expected_message == invalid_message
 
-def test__getinvalidmessage_runandassertiondontexist_returnsrunandassertionerrormessage():
-    # Arrange
-    tc = TestCase("Test Name")
-    fixture = TestFixture()
-    
-    # Act
-    invalid_message = tc.get_invalid_message()
-
-    # Assert
-    assertion_message_exists = tc.ERROR_MESSAGE_ASSERTION_MISSING in invalid_message
-    run_message_exists = tc.ERROR_MESSAGE_RUN_MISSING in invalid_message
-
-    assert assertion_message_exists == True
-    assert run_message_exists == True
-
 def test__set_run__function_passed__sets_run_function():
     # Arrange
     tc = TestCase("Test Name")
@@ -168,6 +140,33 @@ def test__execute_test__before_not_set__does_not_call_before(mocker):
     # Assert
     tc.before.assert_not_called()
 
+def test__execute_test__run_set__calls_run(mocker):
+    # Arrange
+    tc = TestCase("TestName")
+
+    tc.set_run(lambda: 1 == 1)
+    tc.set_assertion(lambda: 1 == 1)
+    mocker.patch.object(tc, 'run')
+
+    # Act
+    test_result = tc.execute_test()
+
+    # Assert
+    tc.run.assert_called_once_with()
+
+def test__execute_test__run_not_set__does_not_call_run(mocker):
+    # Arrange
+    tc = TestCase("TestName")
+
+    tc.set_assertion(lambda: 1 == 1)
+    mocker.patch.object(tc, 'run')
+
+    # Act
+    test_result = tc.execute_test()
+
+    # Assert
+    tc.run.assert_not_called()    
+
 def test__execute_test__after_set__calls_after(mocker):
     # Arrange
     tc = TestCase("TestName")
