Presented at NDC London (2019-01-31)
Video at https://www.youtube.com/watch?v=tWn8RA_DEic
Throw a line of code into many codebases and it's sure to hit one or more testing frameworks. There's no shortage of frameworks for testing, each with their particular spin and set of conventions, but that glut is not always matched by a clear vision of how to structure and use tests — a framework is a vehicle, but you still need to know how to drive. The computer science classic, Structure and Interpretation of Computer Programs, points out that "Programs must be written for people to read, and only incidentally for machines to execute". The same is true of test code.
This talk takes a deep dive into unit testing, looking at examples and counterexamples across a number of languages and frameworks, from naming to nesting, exploring the benefits of data-driven testing, the trade-offs between example-based and property-based testing, how to get the most out of the common given–when–then refrain and knowing how far to follow it.
6. So who should you be writing the
tests for? For the person trying to
understand your code.
Good tests act as documentation
for the code they are testing.
Gerard Meszaros
7. Very many people say "TDD"
when they really mean, "I have
good unit tests" ("I have GUTs"?).
Alistair Cockburn
The modern programming professional has GUTs
9. A unit test is a test of behaviour
whose success or failure is wholly
determined by the correctness of
the test and the correctness of
the unit under test.
Kevlin Henney
https://www.theregister.co.uk/2007/07/28/what_are_your_units/
12. The programmer in me made unit testing more
about applying and exercising frameworks.
I had essentially reduced my concept of unit
testing to the basic mechanics of exercising
xUnit to verify the behavior of my classes.
My mindset had me thinking far too narrowly
about what it meant to write good unit tests.
Tod Golding
Tapping into Testing Nirvana
40. For tests to drive development they must
do more than just test that code performs
its required functionality: they must clearly
express that required functionality to the
reader.
That is, they must be clear specifications
of the required functionality.
Nat Pryce & Steve Freeman
Are your tests really driving your development?
41. namespace Leap_year_spec
{
[TestFixture]
public class A_year_is_a_leap_year
{
[Test]
public void if_it_is_divisible_by_4_but_not_by_100()
[Test]
public void if_it_is_divisible_by_400()
}
[TestFixture]
public class A_year_is_not_a_leap_year
{
[Test]
public void if_it_is_not_divisible_by_4()
[Test]
public void if_it_is_divisible_by_100_but_not_by_400()
}
}
42. namespace Leap_year_spec
{
[TestFixture]
public class A_year_is_a_leap_year
{
[Test]
public void if_it_is_divisible_by_4_but_not_by_100()
[Test]
public void if_it_is_divisible_by_400()
}
[TestFixture]
public class A_year_is_not_a_leap_year
{
[Test]
public void if_it_is_not_divisible_by_4()
[Test]
public void if_it_is_divisible_by_100_but_not_by_400()
}
}
43. namespace Leap_year_spec
{
[TestFixture]
public class A_year_is_a_leap_year
[TestFixture]
public class A_year_is_not_a_leap_year
{
[Test]
public void if_it_is_not_divisible_by_4()
{
Assert.IsFalse(IsLeapYear(2018));
}
[Test]
public void if_it_is_divisible_by_100_but_not_by_400()
{
Assert.IsFalse(IsLeapYear(1900));
}
}
}
44. namespace Leap_year_spec
{
[TestFixture]
public class A_year_is_a_leap_year
[TestFixture]
public class A_year_is_not_a_leap_year
{
[Test]
public void if_it_is_not_divisible_by_4()
{
Assert.IsFalse(isLeapYear(2018));
}
[Test]
public void if_it_is_divisible_by_100_but_not_by_400()
{
Assert.IsFalse(isLeapYear(1900));
}
}
}
45. namespace Leap_year_spec
{
[TestFixture]
public class A_year_is_a_leap_year
[TestFixture]
public class A_year_is_not_a_leap_year
{
[Test]
public void if_it_is_not_divisible_by_4()
{
Assert.IsFalse(isLeapYear(42));
}
[Test]
public void if_it_is_divisible_by_100_but_not_by_400()
{
Assert.IsFalse(isLeapYear(100));
}
}
}
46.
47. namespace Leap_year_spec
{
[TestFixture]
public class A_year_is_a_leap_year
[TestFixture]
public class A_year_is_not_a_leap_year
{
[Test]
public void if_it_is_not_divisible_by_4()
{
Assert.IsFalse(IsLeapYear(2018));
Assert.IsFalse(IsLeapYear(2017));
Assert.IsFalse(IsLeapYear(42));
Assert.IsFalse(IsLeapYear(1));
}
[Test]
public void if_it_is_divisible_by_100_but_not_by_400()
}
}
48. namespace Leap_year_spec
{
[TestFixture]
public class A_year_is_a_leap_year
[TestFixture]
public class A_year_is_not_a_leap_year
{
[TestCase(2018)]
[TestCase(2017)]
[TestCase(42)]
[TestCase(1)]
public void if_it_is_not_divisible_by_4(int year)
{
Assert.IsFalse(IsLeapYear(year));
}
[Test]
public void if_it_is_divisible_by_100_but_not_by_400()
}
}
49. namespace Leap_year_spec
{
[TestFixture]
public class A_year_is_a_leap_year
[TestFixture]
public class A_year_is_not_a_leap_year
{
[Test]
public void if_it_is_not_divisible_by_4(
[Values(2018, 2017, 42, 1)] int year)
{
Assert.IsFalse(IsLeapYear(year));
}
[Test]
public void if_it_is_divisible_by_100_but_not_by_400()
}
}
50. namespace Leap_year_spec
{
[TestFixture]
public class A_year_is_a_leap_year
{
[Test]
public void if_it_is_divisible_by_4_but_not_by_100(
[Values(2016, 1984, 4)] int year)
[Test]
public void if_it_is_divisible_by_400(
[Range(400, 2400, 400)] int year)
}
[TestFixture]
public class A_year_is_not_a_leap_year
{
[Test]
public void if_it_is_not_divisible_by_4(
[Values(2018, 2017, 42, 1)] int year)
[Test]
public void if_it_is_divisible_by_100_but_not_by_400(
[Values(2100, 1900, 100)] int year)
}
}
51. namespace Leap_year_spec
{
[TestFixture]
public class A_year_is_a_leap_year
{
[Test]
public void if_it_is_divisible_by_4_but_not_by_100(
[Values(2016, 1984, 4)] int year)
[Test]
public void if_it_is_divisible_by_400(
[Range(400, 2400, 400)] int year)
}
[TestFixture]
public class A_year_is_not_a_leap_year
{
[Test]
public void if_it_is_not_divisible_by_4(
[Values(2018, 2017, 42, 1)] int year)
[Test]
public void if_it_is_divisible_by_100_but_not_by_400(
[Values(2100, 1900, 100)] int year)
}
}
54. namespace Leap_year_spec
{
[TestFixture]
public class A_year
{
[Test]
public void is_either_a_leap_year_or_not([Range(1, 10000)] int year)
{
Assert.AreEqual(
year % 4 == 0 && year % 100 != 0 || year % 400 == 0,
IsLeapYear(year));
}
}
}
55. namespace Leap_year_spec
{
[TestFixture]
public class A_year
{
[Test]
public void is_either_a_leap_year_or_not([Range(1, 10000)] int year)
{
Assert.AreEqual(LeapYearExpectation(year), IsLeapYear(year));
}
public static bool LeapYearExpectation(int year)
{
return year % 4 == 0 && year % 100 != 0 || year % 400 == 0;
}
}
}
56. public static bool IsLeapYear(int year)
{
return year % 4 == 0 && year % 100 != 0 || year % 400 == 0;
}
57. All happy families are alike;
each unhappy family is
unhappy in its own way.
Leo Tolstoy
Anna Karenina
58. namespace Leap_year_spec
{
[TestFixture]
public class A_year_is_a_leap_year
[TestFixture]
public class A_year_is_not_a_leap_year
[TestFixture]
public class A_year_is_not_supported
{
[Test]
public void if_it_is_0()
{
Assert.Throws<ArgumentException>(() => IsLeapYear(0));
}
}
}
59. namespace Leap_year_spec
{
[TestFixture]
public class A_year_is_a_leap_year
[TestFixture]
public class A_year_is_not_a_leap_year
[TestFixture]
public class A_year_is_not_supported
{
[Test]
public void if_it_is_0()
{
Assert.Catch<ArgumentException>(() => IsLeapYear(0));
}
}
}
60. namespace Leap_year_spec
{
[TestFixture]
public class A_year_is_a_leap_year
[TestFixture]
public class A_year_is_not_a_leap_year
[TestFixture]
public class A_year_is_not_supported
{
[Test]
public void if_it_is_0()
{
Assert.Catch<ArgumentException>(() => IsLeapYear(0));
}
[Test]
public void if_it_is_negative(
[Values(-1, -4, -100, -400)] int year)
{
Assert.Catch<ArgumentException>(() => IsLeapYear(year));
}
}
}
61. namespace Leap_year_spec
{
[TestFixture]
public class A_year_is_a_leap_year
[TestFixture]
public class A_year_is_not_a_leap_year
[TestFixture]
public class A_year_is_supported
{
[Test]
public void if_it_is_positive(
[Values(1, int.MaxValue)] int year)
{
Assert.DoesNotThrow(() => IsLeapYear(year));
}
}
[TestFixture]
public class A_year_is_not_supported
}
64. An abstract data type defines a
class of abstract objects which
is completely characterized by
the operations available on
those objects.
Barbara Liskov
Programming with Abstract Data Types
65. T • Stack
{
new: Stack[T],
push: Stack[T] T → Stack[T],
pop: Stack[T] ⇸ Stack[T],
depth: Stack[T] → Integer,
top: Stack[T] ⇸ T
}
75. TEST_CASE(“stack tests”)
{
SECTION(“can be constructed”)
SECTION(“can be pushed”)
SECTION(“can sometimes be popped”)
SECTION(“has depth”)
SECTION(“sometimes has a top”)
}
78. Given
When
Then
an empty stack
an item is pushed
if it's OK with you, I
think that, perhaps, it
should probably not be
empty, don't you think?
79. Make definite assertions.
Avoid tame, colourless, hesitating, noncommittal
language.
When a sentence is made stronger, it usually becomes
shorter. Thus brevity is a by-product of vigour.
William Strunk and E B White
The Elements of Style
88. TEST_CASE(“Stack specification”)
{
SECTION(“An empty stack acquires depth by retaining a pushed item as its top")
{
stack<std::string> stack;
stack.push(“NDC");
REQUIRE(stack.depth() == 1);
REQUIRE(stack.top() == “NDC");
}
}
89. TEST_CASE(“Stack specification”)
{
SECTION(“A new stack is empty”)
SECTION(“An empty stack throws when queried for its top item”)
SECTION(“An empty stack throws when popped”)
SECTION(“An empty stack acquires depth by retaining a pushed item as its top”)
SECTION(“A non-empty stack becomes deeper by retaining a pushed item as its top”)
SECTION(“A non-empty stack on popping reveals tops in reverse order of pushing”)
}
90. TEST_CASE(“Stack specification”)
{
SECTION(“A new stack is empty”)
SECTION(“An empty stack throws when queried for its top item”)
SECTION(“An empty stack throws when popped”)
SECTION(“An empty stack acquires depth by retaining a pushed item as its top”)
SECTION(“A non-empty stack becomes deeper by retaining a pushed item as its top”)
SECTION(“A non-empty stack on popping reveals tops in reverse order of pushing”)
}
91. TEST_CASE(“Stack specification”)
{
SECTION(“An empty stack acquires depth by retaining a pushed item as its top")
{
stack<std::string> stack;
stack.push(“NDC");
REQUIRE(stack.depth() == 1);
REQUIRE(stack.top() == “NDC");
}
}
92. For each usage scenario, the test(s):
▪ Describe the context, starting point, or
preconditions that must be satisfied
▪ Illustrate how the software is invoked
▪ Describe the expected results or
postconditions to be verified
Gerard Meszaros
93. TEST_CASE(“Stack specification”)
{
SECTION(“An empty stack acquires depth by retaining a pushed item as its top")
{
// Arrange:
stack<std::string> stack;
// Act:
stack.push(“NDC");
// Assert:
REQUIRE(stack.depth() == 1);
REQUIRE(stack.top() == “NDC");
}
}
94. TEST_CASE(“Stack specification”)
{
SECTION(“An empty stack acquires depth by retaining a pushed item as its top")
{
// Arrange:
stack<std::string> stack;
// Act:
stack.push(“NDC");
// Assert:
REQUIRE(stack.depth() == 1);
REQUIRE(stack.top() == “NDC");
}
}
95. TEST_CASE(“Stack specification”)
{
SECTION(“An empty stack acquires depth by retaining a pushed item as its top")
{
// Given:
stack<std::string> stack;
// When:
stack.push(“NDC");
// Then:
REQUIRE(stack.depth() == 1);
REQUIRE(stack.top() == “NDC");
}
}
103. TEST_CASE("Stack specification")
{
SECTION("A new stack")
{
SECTION("is empty") ...
}
SECTION("An empty stack")
{
SECTION("throws when queried for its top item") ...
SECTION("throws when popped") ...
SECTION("acquires depth by retaining a pushed item as its top") ...
}
SECTION("A non empty stack")
{
SECTION("becomes deeper by retaining a pushed item as its top") ...
SECTION("on popping reveals tops in reverse order of pushing") ...
}
}
104. TEST_CASE("Stack specification")
{
SECTION("A new stack")
{
SECTION("is empty") ...
}
SECTION("An empty stack")
{
SECTION("throws when queried for its top item") ...
SECTION("throws when popped") ...
SECTION("acquires depth by retaining a pushed item as its top") ...
}
SECTION("A non empty stack")
{
SECTION("becomes deeper by retaining a pushed item as its top") ...
SECTION("on popping reveals tops in reverse order of pushing") ...
}
}
105. TEST_CASE("Stack specification")
{
SECTION("A new stack")
{
...
}
SECTION("An empty stack")
{
...
SECTION("acquires depth by retaining a pushed item as its top")
{
stack<std::string> stack;
stack.push(“NDC");
REQUIRE(stack.depth() == 1);
REQUIRE(stack.top() == “NDC");
}
}
SECTION("A non empty stack")
{
...
}
}
106. TEST_CASE("Stack specification")
{
SECTION("A new stack")
{
...
}
SECTION("An empty stack")
{
stack<std::string> stack;
...
SECTION("acquires depth by retaining a pushed item as its top")
{
stack.push(“NDC");
REQUIRE(stack.depth() == 1);
REQUIRE(stack.top() == “NDC");
}
}
SECTION("A non empty stack")
{
...
}
}
107. SCENARIO("Stack specification")
{
...
GIVEN("An empty stack")
{
stack<std::string> stack;
...
WHEN("an item is pushed")
{
stack.push(“NDC");
THEN(“acquires depth by retaining the new item as its top”)
{
REQUIRE(stack.depth() == 1);
REQUIRE(stack.top() == “NDC");
}
}
}
...
}
108. SCENARIO("Stack specification")
{
...
GIVEN("An empty stack")
{
stack<std::string> stack;
...
WHEN("an item is pushed")
{
stack.push(“NDC");
THEN(“acquires depth”)
{
REQUIRE(stack.depth() == 1);
}
THEN(“retains the new item as its top”)
{
REQUIRE(stack.top() == “NDC");
}
}
}
...
}
109. SCENARIO("Stack specification")
{
...
GIVEN("An empty stack")
{
stack<std::string> stack;
...
WHEN("an item is pushed")
{
stack.push(“NDC");
THEN(“acquires depth”)
REQUIRE(stack.depth() == 1);
THEN(“retains the new item as its top”)
REQUIRE(stack.top() == “NDC");
}
}
...
}
110. Given can be used to group
tests for operations with
respect to common
initial state
111. When can be used to group
tests by operation,
regardless of initial state
or outcome
112. Then can be used to group
tests by common
outcome, regardless of
operation or initial state
121. public class Queue<T>
{
...
public Queue(int capacity) ...
public int length() ...
public int capacity() ...
public boolean enqueue(T last) ...
public Optional<T> dequeue() ...
}
124. public class Queue_spec
public class A_new_queue
public void is_empty()
public void preserves_positive_bounding_capacity(int capacity)
public void cannot_be_created_with_non_positive_bounding_capacity(int capacity)
public class An_empty_queue
public void dequeues_an_empty_value()
public void remains_empty_when_null_enqueued()
public void becomes_non_empty_when_non_null_value_enqueued(String value)
public class A_non_empty_queue
public class that_is_not_full
public void becomes_longer_when_non_null_value_enqueued(String value)
public void becomes_full_when_enqueued_up_to_capacity()
public class that_is_full
public void ignores_further_enqueued_values()
public void becomes_non_full_when_dequeued()
public void dequeues_values_in_order_enqueued()
public void remains_unchanged_when_null_enqueued()
125. public class Queue_spec
public class A_new_queue
public void is_empty()
public void preserves_positive_bounding_capacity(int capacity)
public void cannot_be_created_with_non_positive_bounding_capacity(int capacity)
public class An_empty_queue
public void dequeues_an_empty_value()
public void remains_empty_when_null_enqueued()
public void becomes_non_empty_when_non_null_value_enqueued(String value)
public class A_non_empty_queue
public class that_is_not_full
public void becomes_longer_when_non_null_value_enqueued(String value)
public void becomes_full_when_enqueued_up_to_capacity()
public class that_is_full
public void ignores_further_enqueued_values()
public void becomes_non_full_when_dequeued()
public void dequeues_values_in_order_enqueued()
public void remains_unchanged_when_null_enqueued()
126. public class Queue<T>
{
private Deque<T> items = new LinkedList<>();
private final int capacity;
public Queue(int boundingCapacity)
{
if (boundingCapacity < 1)
throw new IllegalArgumentException();
capacity = boundingCapacity;
}
public int length()
{
return items.size();
}
public int capacity()
{
return capacity;
}
public boolean enqueue(T last)
{
boolean enqueuing = last != null && length() < capacity();
if (enqueuing)
items.addLast(last);
return enqueuing;
}
public Optional<T> dequeue()
{
return Optional.ofNullable(items.pollFirst());
}
}