from pluralsight
modules are singleton
Closures and decorators #
closure maintains references to objects from earlier scopes
def raise_to(exp):
def raise_to_exp(x):
return pow(x, exp)
return raise_to_exp
square = raise_to(2) # square is now a function
assert square(2) == 4
cube = raise_to(3)
assert cube(3) == 9
lambdas are one-line function without a name
length = lambda name: len(name)
assert length("oui") == 3
def function(a, b):
print("A function")
def local_function(x, y):
print("A local function")
return x*y + a*b
return local_function
lambdas // TODO function factories decorators
def logging_decorator(f):
@functools.wraps(f) #forwards __doc__ and __name__
def logging_wrapper():
print(f"{f.__name__} function was called")
return f
return logging_wrapper
#@second_decorator - decorators are called in reverse-order
@logging_decorator
def hello():
'''print hello'''
print("Hello")
hello()
wrappers
properties and class methods #
class_attribute instance_attribute
class Foo:
a_class_attribute = 0
def __init__(self):
self.an_instance_attribute = 42
Foo.a_class_attribute = 64
self.a_class_attribute = "actually doesnt change the class attribute"
@staticmethod
def a_static_method():
return "No self, can't use the class attributes"
@classmethod
def a_class_method(cls):
cls.a_class_attribute += 1
@classmethod
def a_named_constructor(cls):
return cls()
def _get_a_property(self):
return self.an_instance_attribute
@property
def a_property(self):
self._get_a_property()
def _set_a_property(self, value):
self.an_instance_attribute = value
@a_property.setter
def a_property(self, value):
self._set_a_property(value)
class Bar(Foo):
# template method design pattern - override
def _get_a_property(self):
r = super()._get_a_property()
# do more things with it
return r
def _set_a_property(self, value):
# added logic, like checks
super()._set_a_property(value)
Strings and representations #
repr() > repr #
unambiguous, precise, include type, for developers and debugging where information is more important than readabiility
- reprlib
str() > str #
for humans, if not set, uses repr
“{:parameter}".format > format(self, f) #
Numeric and scalar types #
Decimal type #
from datetime import datetime as Datetime // useful
Iterables and Iteration #
l = [(x,y) for x in range(5) if x < 10 for y in range (3)]
# l = [(x,y)
# for x in range(5)
# if x < 10
# for y in range (3)]
# is equivalent to
ll = []
for x in range(5):
if x < 10:
for y in range (3):
ll.append((x,y))
map(ord, “oui la voix”) returns a lazily-evaluated iterator filter : filter via a boolean function reduce : accumulation
iterator : next(), iter(), getitem() sequence : len(), getitem() all sequences are iterable
Inheritance and Subtype Polymorphism #
python sub-class does not call base class initializer automatically - must use super()
Collections #
container = in and not-in sized = len iterable = iter + “for in” sequence = count, access by index, reversable, slicable, concatenation and repetition set
class SortedSet(Sequence, Set):
def __init__(self, items=None):
self._items = sorted(set(items)) if items is not None else []
def __contains__(self, item): #container
index = bisect_left(self._items, item)
return (index != len(self._items)) and (self._items[index] == item)
def __len__(self): #sized
return len(self._items)
def __iter__(self): #iterable
return iter(self._items)
def __getitem__(self, index):
result = self._items[index]
return SortedSet(result) if isinstance(index, slice) else result
def __repr__(self):
return "Sorted({})".format(
repr(self._items) if self._items else ''
)
def __eq__(self, rhs):
if not isinstance(rhs, SortedSet):
return NotImplemented #different than raise NotImplementedError
return self._items == rhs._items
def __ne__(self, rhs):
if not isinstance(rhs, SortedSet):
return NotImplemented #different than raise NotImplementedError
return self._items != rhs._items
def index(self, item):
index = bisect_left(self._items, item)
if (index != len(self._items)) and (self._items[index] == item):
return index
raise ValueError("{} not found".format(repr(item)))
def _is_unique_and_sorted(self):
return all(self[i] < self[i-1] for i in range(len(self) -1))
def count(self, item):
assert self._is_unique_and_sorted()
return int(item in self)
def __add__(self, rhs):
return SortedSet(itertools.chain(self._items, rhs._items))
def __mul__(self, rhs):
return self if rhs > 0 else SortedSet()
def __rmul__(self, lhs):
return self * lhs
def issubset(self, iterable):
return self <= SortedSet(iterable)
def issuperset(self, iterable):
return self >= SortedSet(iterable)
def intersection(self, iterable):
return self & SortedSet(iterable)
def union(self, iterable):
return self | SortedSet(iterable)
def symmetric_difference(self, iterable):
return self ^ SortedSet(iterable)
def difference(self, iterable):
return self - SortedSet(iterable)
Exceptions and errors #
never catch ALL exceptions
https://docs.python.org/3/library/exceptions.html#exception-hierarchy
context cause traceback
Defining context managers #
# 1st solution
class ContextManager:
# PEP 343
def __enter__(self):
pass
# can return a ContextManager which will be used in `with ... as`
# usually return itself with set variables
def __exit__(self, exception_type, exception_val, exception_tb):
# propagate Exception by default, return False to not propagate
# must not re-raise Exception except if fails itself
if exception_type is None:
pass
else:
pass
# 2nd solution
# useful for a stateful context manager
@contextlib.contextmanager # for a function that yield something
def my_context_manager():
# <ENTER>
try:
yield [value] #will be bound to `with ... as`
# <NORMAL EXIT>
except:
# <EXCEPTIONAL EXIT>
# need to explicitly re-raise the exception
raise
with context_manager as x:
# context_manager.begin()
pass
# context_manager.end()
with cm as a, cm as b :
pass
with cm() as a,\
cm(a) as b:
pass
Introspection #
Easier to Ask Forgiveness Than Permission vs Look Before You Leap
It is more pythonic to assume that a certain type is passed as an argument
rather than checking that the passed argument has all the necessary attributes
assume that this certain type is passed
and with a try ... except
block, raise a TypeError if necessary
(becausse hasattr uses try - except
anyway)
Best practices for code quality #
PEP8 official python style guidelines
Formatting Whitespace Naming
pylint pycodestyle black
Sphinx with reStructuredText + apidoc (for code)
Types #
mypy or PyCharm change pycharm settings to treat type-warnings as errors
https://mypy.readthedocs.io/en/stable/cheat_sheet_py3.html
def plus(num1: int, my_float: float = 3.5) -> float:
result: float
result = num1 + my_float
return result
Virtualenv #
always use python3 -m pip [...]
always use a venv
use python3 -m venv [...]
Tox #
let’s you test your code against different versions of python with venvs
tox.ini
poetry
Unit Testing #
automated unit test
- designed by a human
- runs without intervention
- reports if fail or pass
all tests are independant
first Arrange : set up objects then Act : launch the unit test then Assert : make claims about what happened
check for regression testing
Test Driven Development (better than Test First and Test Last)
- pluralsight course about TDD by the same author https://martinfowler.com/articles/continuousIntegration.html
Unittest module #
# test_xxxx.py
import unittest
class PhoneBookTest(unittest.TestCase):
def setUp(self) -> None: # fixture
self.att = NewAtt()
def tearDown(self) -> None:
pass
def test_scenario(self):
self.att.something()
self.assertEqual("xxx", x)
@unittest.skip("WIP")
def test_wip(self):
self.att.something()
pass
Pytest module #
report as HTML with pytest-html
run with python3 -m pytest --html=report.html
PYTHONPATH=. python3 -m pytest tests/
# pytest.ini
[pytest]
addopts = --strict
markers =
slow: Run tests that are slow because
# tests/conftest.py
@pytest.fixture
def obj(tmpdir):
# setup
yield obj
# teardown
# shutil.rmtree(str(tmpdir))
# tests/1.py
def test_scenario(obj):
assert "xxx" == x
with pytest.raises(KeyError):
pass
@pytest.mark.slow
def test_slow(obj):
pass
@pytest.mark.skip("WIP")
def test_skipped(obj):
pass
Testable Documentation with doctest #
- docstring maintenance : keep docstring synchronised with code
- regression testing (approval testing)
- tutorial documentation
python3 -m pytest --doctest-modules
# pytest.ini
[pytest]
addopts = --strict
markers =
slow: Run tests that are slow because
def my_func(a, b):
"""
Do something
>>> result = my_func("test",
... "other") # doctest: +ELLIPSIS
>>> (print r for r in result)
xxx ... yyy
"""
return f"{a}-{b}"
def random_choice(obj):
"""
Randomly choose something
>>> random.seed(1234) # deterministic choice
>>> random_choice("test")
"xxx"
"""
return f"{a}-{b}"
def traceback(obj):
"""
Will fail
>>> traceback(123)
Traceback (most recent call last):
...
TypeError: must be str, not int
"""
return f"{a}-{b}"
Test Doubles : Mocks, Stubs and Fakes #
3 kinds of Assert : return value - state change - method call
Dummy -> Stub -> Fake
Stub #
a Stub has the same methods as the class it replaces, but the implementation is very simple. s Stub won’t fail the test.
from sensor import sensor
# hand-coded stub
class StubSensor:
def sample_pressure(self):
return 15
class Alarm:
# NEW : a stub sensor can be provided
def __init__(self, sensor=None):
self._sensor = sensor or Sensor()
self._low_pressure_threshold = 17
self._high_pressure_threshold = 21
self._is_alarm_on = False
# OLD : can't specify a stub sensor as it is instanciated directly
def __init__(self):
self._sensor = Sensor()
self._low_pressure_threshold = 17
self._high_pressure_threshold = 21
self._is_alarm_on = False
def test_low_pressure_activates_alarm():
alarm = Alarm(sensor=StubSensor())
alarm.check()
assert alarm.is_alarm_on
def test_normal_pressure_alarm_stays_off():
stub_sensor = unittest.mock.Mock(Sensor)
stub_sensor.sample_pressure.return_value = 18
alarm = Alarm(sensor=stub_sensor)
alarm.check()
assert not alarm.is_alarm_on
Fake #
a Fake has the same methods as the class it replaces, with a real implementation with logic and behaviour
- replace File with StringIO
- replace MySQL with a in-memory-database
- replace real webserver with a lightweight-webserver
def test_convert_quotes():
fake_file = io.StringIO("quote: ' ")
converter = HtmlPagesConverter(open_file=fake_file)
converted_text = converter.get_html_page(0)
assert converted_text == "quote: ' "
def access_second_page():
fake_file = io.StringIO("""\
page one
PAGE_BREAK
page two
PAGE_BREAK
page three
""")
converter = HtmlPagesConverter(open_file=fake_file)
converted_text = converter.get_html_page(1)
assert converted_text == "page two<br />"
Dummy #
a Dummy is not used, usually is None, because the argument is mandatory
Spy #
a Spy records the method calls it receives
class SingleSignOnRegistry:
pass
class MyService:
def __init__(self, sso_registry):
self.sso_registry = sso_registry
def handle(self, request, sso_token):
if sso_token:
return Response("Yes")
else:
return Response("No")
class Request:
def __init__(self, name):
self.name = name
class Response:
def __init__(self, text):
self.text = text
def test_sso():
spy_sso_registry = Mock(SingleSignOnRegistry)
service = MyService(spy_sso_registry)
token = SSOToken
service.handle(Request("Emily"), token)
spy_sso_registry.is_valid.assert_called_with(token)
Mock #
a Mock expect certain method calls, otherwise raise an error
def confirm_token(token):
def is_valid(actual_token):
if actual_token != token:
raise ValueError("wrong token")
return is_valid
def test_sso():
mock_sso_registry = Mock(SingleSignOnRegistry)
token = SSOToken
mock_sso_registry.is_valid = Mock(side_effect=confirm_token(token))
service = MyService(mock_sso_registry)
service.handle(Request("Emily"), token)
mock_sso_registry.is_valid.assert_called_with(token)
Monkey Patching #
exchange code after runtime, for example
# can't replace this one
def __init__(self):
self._sensor = Sensor()
# by this one
def __init__(self, sensor=None):
self._sensor = sensor or Sensor()
from unittest.mock import patch
def test_alarm():
with patch(package.Class) as test_package_class:
test_class_instance = Mock()
test_class_instance.method_or_attribute.return_value = 22
test_package_class.return_value = test_class_instance
@patch("alarm.Sensor")
def test_alarm(test_package_class):
with patch(package.Class) as test_package_class:
test_class_instance = Mock()
test_class_instance.method_or_attribute.return_value = 22
test_package_class.return_value = test_class_instance
@patch("builtins.open", new_callable=mock_open, read_data="quote : ' ")
def test_builtin(fake_file):
assert
Parameterized tests and test coverage #
useful :
- to spot missing tests for new code
- to add tests to existing code
- to track trends over time
evaluating test quality :
- bugs in production
- confidence to refactor
- unrealiable tests
- code review
- onboarding time
- mutation testing
pytest --cov-report html:cov_html --cov-branch --cov=package_name .
@pytest.mark.parametrize("p1, p2",
[(2,1),
(4,3)
])
def test_parameterized(p1, p2):
assert p1 > p2
Advanced Python #
While else #
should not be used, but it exists
while condition:
execute_if_true()
else: # nobreak
execute_when_false()
For else #
for item in iterable:
if item is genial:
break
else: #nobreak - no match found
pass
Try else #
try:
pass
except:
pass
else:
pass
Dispath on Type #
useful for separation of concerns
class A:
pass
class B:
pass
from functools import singledispatch
@singledispatch
def do(s):
return "unknown type"
@do.register(A)
def _do_A(s):
print(s)
@do.register(B)
def _do_B(s):
print(s)
# if need to use dispatch in a class method
class S:
def do(self, s):
return dispatch_do(s, self) # double dispatch class.method(other_class)
Byte oriented programming #
bb = b"This is okay because it's 7bit ASCII"
bb[0] = an integer
bb[2:4] = bytes[]
Interpreting binary structures #
struct Vector {
float x; // 32-bit (!= 64-bit in python)
float y;
float z;
};
struct Color {
unsigned short int red; // 16-bit
unsigned short int green;
unsigned short int blue;
};
struct Vertex {
struct Vector position;
struct Color color;
}
import struct
# https://docs.python.org/3/library/struct.html
with open('data.bin', 'rb') as f:
buffer = f.read()
hex_buffer = hexlify(buffer).decode('ascii')
hex_pairs = ' '.join(hex_buffer[i:i+2] for i in range(0, len(hex_buffer), 2))
items = struct.unpack_from("@fffHHHxx", buffer)
# @ = native (little-endian = "<")
# f = C-float - 3f
# H = C-unsigned short int - 3H
# x = padding (C wants to pad struct to a multiple of 4 bits)
v = []
for items in struct.iter_unpack("@fffHHHxx", buffer):
v.append(newVertex(items))
We can also use memoryview
to avoid duplication of data (zero-copy)
mem = memoryview(buffer)
code.interact(local=locals())
# stop the program and launch the python interpreter
# with the variables loaded
mem[0:12].cast("f")
# a slice of a memoryview is a memoryview
Memory-mapped files are useful to avoid copying the data
import mmap
with open('data.bin', 'rb') as f:
with mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) as buffer:
# buffer is now a memory-mapped file - zero copy
mem = memoryview(buffer)
# dont forget to delete pointer to the buffer and memory views
del mem
Memory efficiency #
class Resistor:
__slots__ = ['resistance', 'tolerance', 'power']
# set the final number of attributes, no moore attribute can be added
# size from 152 bytes to 64 bytes
def __init__(self, resistance, tolerance, power):
self.resistance = resistance
self.tolerance = tolerance
self.power = power
Descriptors #
from weakref import WeakKeyDictionary
class Positive:
def __init__(self):
self._instance_data = WeakKeyDictionary()
def __get__(self, instance, owner):
return self._instance_data[instance]
def __set__(self, instance, value):
if value <= 0:
raise ValueError()
self._instance_data[instance] = value
def __delete__(self, instance):
raise AttributeError()
class A:
def __init__(self, a):
self.a = a # set through descriptor
# replacing property by our own descriptor Positive
@property
def a(self):
return self._a
@a.setter
def a(self, value):
if a <= 0:
raise ValueError
self._a = a
a = Positive()
Instance creatioon #
Class is instanciated via the __new__
method inherited from object
.
Metaclasses #
class Widget:
pass
name = 'Widget'
metaclass = type
bases = ()
kwargs = {}
namespace = metaclass.__prepare__(name, bases, **kwargs)
Widget = metaclass.__new__(metaclass, name, bases, namespace, **kwargs)
metaclass.__init__(Widget, name, bases, namespace, **kwargs)
class Widget(object, metaclass=type):
def __new__(cls, *args, **kwargs):
return type.__new__(cls)
def __init__(self):
pass
# type example
class A:
def __call__(cls, *args, **kwargs):
obj = cls.__new__(*args, **kwargs)
obj.__init__(*args, **kwargs)
return obj
w = Widget()
triggers metaclass.__call__()
which calls Widget.__new__()
and Widget.__init__()