From 1cbea513e6fe1d9cd8ec5fffffda2bc3fc1e53a2 Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Fri, 14 Jan 2022 00:57:08 -0700 Subject: [PATCH 01/11] Make netfilterqueue a package and add type hints --- CHANGES.txt | 4 +- MANIFEST.in | 9 ++-- ci.sh | 26 +++++++---- .../__init__.pxd | 2 +- netfilterqueue/__init__.pyi | 45 +++++++++++++++++++ .../__init__.pyx | 5 ++- netfilterqueue/_version.py | 4 ++ netfilterqueue/py.typed | 0 setup.py | 26 +++++++---- test-requirements.in | 1 + test-requirements.txt | 14 ++++-- tests/conftest.py | 40 +++++++++-------- 12 files changed, 127 insertions(+), 49 deletions(-) rename netfilterqueue.pxd => netfilterqueue/__init__.pxd (99%) create mode 100644 netfilterqueue/__init__.pyi rename netfilterqueue.pyx => netfilterqueue/__init__.pyx (99%) create mode 100644 netfilterqueue/_version.py create mode 100644 netfilterqueue/py.typed diff --git a/CHANGES.txt b/CHANGES.txt index 011053d..89fc98a 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,11 +1,13 @@ v1.0.0, unreleased Propagate exceptions raised by the user's packet callback - Warn about exceptions raised by the packet callback during queue unbinding + Avoid calls to the packet callback during queue unbinding Raise an error if a packet verdict is set after its parent queue is closed set_payload() now affects the result of later get_payload() Handle signals received when run() is blocked in recv() Accept packets in COPY_META mode, only failing on an attempt to access the payload Add a parameter NetfilterQueue(sockfd=N) that uses an already-opened Netlink socket + Add type hints + Remove the Packet.payload attribute; it was never safe (treated as a char* but not NUL-terminated) nor documented, but was exposed in the API (perhaps inadvertently). v0.9.0, 12 Jan 2021 Improve usability when Packet objects are retained past the callback diff --git a/MANIFEST.in b/MANIFEST.in index f461248..4d1b190 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,6 +1,3 @@ -include *.txt -include *.rst -include *.c -include *.pyx -include *.pxd -recursive-include tests/ *.py +include LICENSE.txt README.rst CHANGES.txt +recursive-include netfilterqueue *.py *.pyx *.pxd *.c *.pyi py.typed +recursive-include tests *.py diff --git a/ci.sh b/ci.sh index d836d17..a4dc63e 100755 --- a/ci.sh +++ b/ci.sh @@ -11,34 +11,42 @@ python setup.py sdist --formats=zip # ... but not to install it pip uninstall -y cython +python setup.py build_ext pip install dist/*.zip pip install -Ur test-requirements.txt if [ "$CHECK_LINT" = "1" ]; then error=0 - if ! black --check setup.py tests; then + black_files="setup.py tests netfilterqueue" + if ! black --check $black_files; then + error=$? + black --diff $black_files + fi + mypy --strict -p netfilterqueue || error=$? + ( mkdir empty; cd empty; python -m mypy.stubtest netfilterqueue ) || error=$? + + if [ $error -ne 0 ]; then cat < Optional[bytes]: ... + def get_payload(self) -> bytes: ... + def get_payload_len(self) -> int: ... + def get_timestamp(self) -> float: ... + def get_mark(self) -> int: ... + def set_payload(self, payload: bytes) -> None: ... + def set_mark(self, mark: int) -> None: ... + def retain(self) -> None: ... + def accept(self) -> None: ... + def drop(self) -> None: ... + def repeat(self) -> None: ... + +class NetfilterQueue: + def __new__(self, *, af: int = ..., sockfd: int = ...) -> NetfilterQueue: ... + def bind( + self, + queue_num: int, + user_callback: Callable[[Packet], None], + max_len: int = ..., + mode: int = COPY_PACKET, + range: int = ..., + sock_len: int = ..., + ) -> None: ... + def unbind(self) -> None: ... + def get_fd(self) -> int: ... + def run(self, block: bool = ...) -> None: ... + def run_socket(self, s: socket.socket) -> None: ... + +PROTOCOLS: Dict[int, str] diff --git a/netfilterqueue.pyx b/netfilterqueue/__init__.pyx similarity index 99% rename from netfilterqueue.pyx rename to netfilterqueue/__init__.pyx index 0de0476..6cee14e 100644 --- a/netfilterqueue.pyx +++ b/netfilterqueue/__init__.pyx @@ -5,7 +5,6 @@ function. Copyright: (c) 2011, Kerkhoff Technologies Inc. License: MIT; see LICENSE.txt """ -VERSION = (0, 9, 0) # Constants for module users COPY_NONE = 0 @@ -24,6 +23,10 @@ DEF SockCopySize = MaxCopySize + SockOverhead # Socket queue should hold max number of packets of copysize bytes DEF SockRcvSize = DEFAULT_MAX_QUEUELEN * SockCopySize // 2 +__package__ = "netfilterqueue" + +from ._version import __version__, VERSION + from cpython.exc cimport PyErr_CheckSignals # A negative return value from this callback will stop processing and diff --git a/netfilterqueue/_version.py b/netfilterqueue/_version.py new file mode 100644 index 0000000..5a5b5fe --- /dev/null +++ b/netfilterqueue/_version.py @@ -0,0 +1,4 @@ +# This file is imported from __init__.py and exec'd from setup.py + +__version__ = "0.9.0+dev" +VERSION = (0, 9, 0) diff --git a/netfilterqueue/py.typed b/netfilterqueue/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/setup.py b/setup.py index ad7d22d..124036b 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ import os, sys from setuptools import setup, Extension -VERSION = "0.9.0" # Remember to change CHANGES.txt and netfilterqueue.pyx when version changes. +exec(open("netfilterqueue/_version.py", encoding="utf-8").read()) setup_requires = [] try: @@ -10,7 +10,9 @@ ext_modules = cythonize( Extension( - "netfilterqueue", ["netfilterqueue.pyx"], libraries=["netfilter_queue"] + "netfilterqueue.__init__", + ["netfilterqueue/__init__.pyx"], + libraries=["netfilter_queue"], ), compiler_directives={"language_level": "3str"}, ) @@ -21,7 +23,7 @@ # setup_requires below. setup_requires = ["cython"] elif not os.path.exists( - os.path.join(os.path.dirname(__file__), "netfilterqueue.c") + os.path.join(os.path.dirname(__file__), "netfilterqueue/__init__.c") ): sys.stderr.write( "You must have Cython installed (`pip install cython`) to build this " @@ -31,21 +33,27 @@ ) sys.exit(1) ext_modules = [ - Extension("netfilterqueue", ["netfilterqueue.c"], libraries=["netfilter_queue"]) + Extension( + "netfilterqueue.__init__", + ["netfilterqueue/__init__.c"], + libraries=["netfilter_queue"], + ) ] setup( - ext_modules=ext_modules, - setup_requires=setup_requires, - python_requires=">=3.6", name="NetfilterQueue", - version=VERSION, + version=__version__, license="MIT", author="Matthew Fox", author_email="matt@tansen.ca", url="https://github.com/oremanj/python-netfilterqueue", description="Python bindings for libnetfilter_queue", - long_description=open("README.rst").read(), + long_description=open("README.rst", encoding="utf-8").read(), + packages=["netfilterqueue"], + ext_modules=ext_modules, + include_package_data=True, + setup_requires=setup_requires, + python_requires=">=3.6", classifiers=[ "Development Status :: 5 - Production/Stable", "License :: OSI Approved :: MIT License", diff --git a/test-requirements.in b/test-requirements.in index 21b675e..2a63073 100644 --- a/test-requirements.in +++ b/test-requirements.in @@ -5,3 +5,4 @@ pytest-trio async_generator black platformdirs <= 2.4.0 # needed by black; 2.4.1+ don't support py3.6 +mypy; implementation_name == "cpython" diff --git a/test-requirements.txt b/test-requirements.txt index b5acfd3..ebd46de 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -22,8 +22,12 @@ idna==3.3 # via trio iniconfig==1.1.1 # via pytest +mypy==0.931 ; implementation_name == "cpython" + # via -r test-requirements.in mypy-extensions==0.4.3 - # via black + # via + # black + # mypy outcome==1.1.0 # via # pytest-trio @@ -57,10 +61,14 @@ sortedcontainers==2.4.0 toml==0.10.2 # via pytest tomli==1.2.3 - # via black + # via + # black + # mypy trio==0.19.0 # via # -r test-requirements.in # pytest-trio typing-extensions==4.0.1 - # via black + # via + # black + # mypy diff --git a/tests/conftest.py b/tests/conftest.py index 0d94e9e..e7f832b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,12 +5,12 @@ import subprocess import sys import trio -import unshare +import unshare # type: ignore import netfilterqueue from functools import partial -from typing import AsyncIterator, Callable, Optional, Tuple +from typing import Any, AsyncIterator, Callable, Dict, Optional, Tuple from async_generator import asynccontextmanager -from pytest_trio.enable_trio_mode import * +from pytest_trio.enable_trio_mode import * # type: ignore # We'll create three network namespaces, representing a router (which @@ -45,8 +45,8 @@ def enter_netns() -> None: subprocess.run("/sbin/ip link set lo up".split(), check=True) -@pytest.hookimpl(tryfirst=True) -def pytest_runtestloop(): +@pytest.hookimpl(tryfirst=True) # type: ignore +def pytest_runtestloop() -> None: if os.getuid() != 0: # Create a new user namespace for the whole test session outer = {"uid": os.getuid(), "gid": os.getgid()} @@ -93,7 +93,9 @@ async def peer_main(idx: int, parent_fd: int) -> None: await peer.connect((peer_ip, peer_port)) # Enter the message-forwarding loop - async def proxy_one_way(src, dest): + async def proxy_one_way( + src: trio.socket.SocketType, dest: trio.socket.SocketType + ) -> None: while src.fileno() >= 0: try: msg = await src.recv(4096) @@ -121,13 +123,13 @@ def _default_capture_cb( class Harness: - def __init__(self): - self._received = {} - self._conn = {} - self.dest_addr = {} + def __init__(self) -> None: + self._received: Dict[int, trio.MemoryReceiveChannel[bytes]] = {} + self._conn: Dict[int, trio.socket.SocketType] = {} + self.dest_addr: Dict[int, Tuple[str, int]] = {} self.failed = False - async def _run_peer(self, idx: int, *, task_status): + async def _run_peer(self, idx: int, *, task_status: Any) -> None: their_ip = PEER_IP[idx] my_ip = ROUTER_IP[idx] conn, child_conn = trio.socket.socketpair(socket.AF_UNIX, socket.SOCK_SEQPACKET) @@ -169,10 +171,10 @@ async def _run_peer(self, idx: int, *, task_status): # and its netns goes away. check=False to suppress that error. await trio.run_process(f"ip link delete veth{idx}".split(), check=False) - async def _manage_peer(self, idx: int, *, task_status): + async def _manage_peer(self, idx: int, *, task_status: Any) -> None: async with trio.open_nursery() as nursery: await nursery.start(self._run_peer, idx) - packets_w, packets_r = trio.open_memory_channel(math.inf) + packets_w, packets_r = trio.open_memory_channel[bytes](math.inf) self._received[idx] = packets_r task_status.started() async with packets_w: @@ -183,7 +185,7 @@ async def _manage_peer(self, idx: int, *, task_status): await packets_w.send(msg) @asynccontextmanager - async def run(self): + async def run(self) -> AsyncIterator[None]: async with trio.open_nursery() as nursery: async with trio.open_nursery() as start_nursery: start_nursery.start_soon(nursery.start, self._manage_peer, 1) @@ -258,14 +260,14 @@ async def capture_packets_to( **options: int, ) -> AsyncIterator["trio.MemoryReceiveChannel[netfilterqueue.Packet]"]: - packets_w, packets_r = trio.open_memory_channel(math.inf) + packets_w, packets_r = trio.open_memory_channel[netfilterqueue.Packet](math.inf) queue_num, nfq = self.bind_queue(partial(cb, packets_w), **options) try: async with self.enqueue_packets_to(idx, queue_num): async with packets_w, trio.open_nursery() as nursery: @nursery.start_soon - async def listen_for_packets(): + async def listen_for_packets() -> None: while True: await trio.lowlevel.wait_readable(nfq.get_fd()) nfq.run(block=False) @@ -275,7 +277,7 @@ async def listen_for_packets(): finally: nfq.unbind() - async def expect(self, idx: int, *packets: bytes): + async def expect(self, idx: int, *packets: bytes) -> None: for expected in packets: with trio.move_on_after(5) as scope: received = await self._received[idx].receive() @@ -291,13 +293,13 @@ async def expect(self, idx: int, *packets: bytes): f"received {received!r}" ) - async def send(self, idx: int, *packets: bytes): + async def send(self, idx: int, *packets: bytes) -> None: for packet in packets: await self._conn[3 - idx].send(packet) @pytest.fixture -async def harness() -> Harness: +async def harness() -> AsyncIterator[Harness]: h = Harness() async with h.run(): yield h From 541c9e76480700f81804e3598a3e3a40d6f45e08 Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Fri, 14 Jan 2022 12:58:56 -0700 Subject: [PATCH 02/11] Switch back to _impl and fix names --- netfilterqueue/__init__.py | 12 ++++++++++++ netfilterqueue/{__init__.pxd => _impl.pxd} | 0 netfilterqueue/{__init__.pyi => _impl.pyi} | 0 netfilterqueue/{__init__.pyx => _impl.pyx} | 19 +++++++++++++++---- setup.py | 10 +++++----- 5 files changed, 32 insertions(+), 9 deletions(-) create mode 100644 netfilterqueue/__init__.py rename netfilterqueue/{__init__.pxd => _impl.pxd} (100%) rename netfilterqueue/{__init__.pyi => _impl.pyi} (100%) rename netfilterqueue/{__init__.pyx => _impl.pyx} (96%) diff --git a/netfilterqueue/__init__.py b/netfilterqueue/__init__.py new file mode 100644 index 0000000..2a92dec --- /dev/null +++ b/netfilterqueue/__init__.py @@ -0,0 +1,12 @@ +from ._impl import ( + COPY_NONE as COPY_NONE, + COPY_META as COPY_META, + COPY_PACKET as COPY_PACKET, + Packet as Packet, + NetfilterQueue as NetfilterQueue, + PROTOCOLS as PROTOCOLS, +) +from ._version import ( + VERSION as VERSION, + __version__ as __version__, +) diff --git a/netfilterqueue/__init__.pxd b/netfilterqueue/_impl.pxd similarity index 100% rename from netfilterqueue/__init__.pxd rename to netfilterqueue/_impl.pxd diff --git a/netfilterqueue/__init__.pyi b/netfilterqueue/_impl.pyi similarity index 100% rename from netfilterqueue/__init__.pyi rename to netfilterqueue/_impl.pyi diff --git a/netfilterqueue/__init__.pyx b/netfilterqueue/_impl.pyx similarity index 96% rename from netfilterqueue/__init__.pyx rename to netfilterqueue/_impl.pyx index 6cee14e..d656d5b 100644 --- a/netfilterqueue/__init__.pyx +++ b/netfilterqueue/_impl.pyx @@ -23,12 +23,12 @@ DEF SockCopySize = MaxCopySize + SockOverhead # Socket queue should hold max number of packets of copysize bytes DEF SockRcvSize = DEFAULT_MAX_QUEUELEN * SockCopySize // 2 -__package__ = "netfilterqueue" - -from ._version import __version__, VERSION - from cpython.exc cimport PyErr_CheckSignals +cdef extern from "Python.h": + ctypedef struct PyTypeObject: + const char* tp_name + # A negative return value from this callback will stop processing and # make nfq_handle_packet return -1, so we use that as the error flag. cdef int global_callback(nfq_q_handle *qh, nfgenmsg *nfmsg, @@ -346,6 +346,17 @@ cdef class NetfilterQueue: else: nfq_handle_packet(self.h, buf, len(buf)) +cdef void _fix_names(): + # Avoid ._impl showing up in reprs. This doesn't work on PyPy; there we would + # need to modify the name before PyType_Ready(), but I can't find any way to + # write Cython code that would execute at that time. + cdef PyTypeObject* tp = Packet + tp.tp_name = "netfilterqueue.Packet" + tp = NetfilterQueue + tp.tp_name = "netfilterqueue.NetfilterQueue" + +_fix_names() + PROTOCOLS = { 0: "HOPOPT", 1: "ICMP", diff --git a/setup.py b/setup.py index 124036b..81bc2fc 100644 --- a/setup.py +++ b/setup.py @@ -10,8 +10,8 @@ ext_modules = cythonize( Extension( - "netfilterqueue.__init__", - ["netfilterqueue/__init__.pyx"], + "netfilterqueue._impl", + ["netfilterqueue/_impl.pyx"], libraries=["netfilter_queue"], ), compiler_directives={"language_level": "3str"}, @@ -23,7 +23,7 @@ # setup_requires below. setup_requires = ["cython"] elif not os.path.exists( - os.path.join(os.path.dirname(__file__), "netfilterqueue/__init__.c") + os.path.join(os.path.dirname(__file__), "netfilterqueue/_impl.c") ): sys.stderr.write( "You must have Cython installed (`pip install cython`) to build this " @@ -34,8 +34,8 @@ sys.exit(1) ext_modules = [ Extension( - "netfilterqueue.__init__", - ["netfilterqueue/__init__.c"], + "netfilterqueue._impl", + ["netfilterqueue/_impl.c"], libraries=["netfilter_queue"], ) ] From db80c853baf461349bf7e2bcd3008f1f4d6d6128 Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Fri, 14 Jan 2022 13:01:43 -0700 Subject: [PATCH 03/11] Fix stub --- netfilterqueue/_impl.pyi | 3 --- 1 file changed, 3 deletions(-) diff --git a/netfilterqueue/_impl.pyi b/netfilterqueue/_impl.pyi index c9d561a..3cba8fc 100644 --- a/netfilterqueue/_impl.pyi +++ b/netfilterqueue/_impl.pyi @@ -2,9 +2,6 @@ import socket from enum import IntEnum from typing import Callable, Dict, Optional, Tuple -__version__: str -VERSION: Tuple[int, ...] - COPY_NONE: int COPY_META: int COPY_PACKET: int From e1e20d4aba9f5e1f60560c8b1b304f756a67380d Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Fri, 14 Jan 2022 13:11:17 -0700 Subject: [PATCH 04/11] Don't install _impl.c --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 81bc2fc..8d95a9a 100644 --- a/setup.py +++ b/setup.py @@ -52,6 +52,7 @@ packages=["netfilterqueue"], ext_modules=ext_modules, include_package_data=True, + exclude_package_data={"netfilterqueue": ["*.c"]}, setup_requires=setup_requires, python_requires=">=3.6", classifiers=[ From c03aec2e88c3d3cce5b4ef784bc562f41945bfca Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Fri, 14 Jan 2022 13:24:18 -0700 Subject: [PATCH 05/11] Release v1.0.0 --- CHANGES.txt | 4 ++-- README.rst | 11 +++++++++++ netfilterqueue/_version.py | 4 ++-- setup.py | 4 ++-- 4 files changed, 17 insertions(+), 6 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 89fc98a..37c092f 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,4 @@ -v1.0.0, unreleased +v1.0.0, 14 Jan 2022 Propagate exceptions raised by the user's packet callback Avoid calls to the packet callback during queue unbinding Raise an error if a packet verdict is set after its parent queue is closed @@ -9,7 +9,7 @@ v1.0.0, unreleased Add type hints Remove the Packet.payload attribute; it was never safe (treated as a char* but not NUL-terminated) nor documented, but was exposed in the API (perhaps inadvertently). -v0.9.0, 12 Jan 2021 +v0.9.0, 12 Jan 2022 Improve usability when Packet objects are retained past the callback Add Packet.retain() to save the packet contents in such cases Eliminate warnings during build on py3 diff --git a/README.rst b/README.rst index a5b60da..275e173 100644 --- a/README.rst +++ b/README.rst @@ -1,3 +1,11 @@ +.. image:: https://img.shields.io/pypi/v/netfilterqueue.svg + :target: https://pypi.org/project/netfilterqueue + :alt: Latest PyPI version + +.. image:: https://github.com/oremanj/python-netfilterqueue/actions/workflows/ci.yml/badge.svg?branch=master + :target: https://github.com/oremanj/python-netfilterqueue/actions?query=branch%3Amaster + :alt: Automated test status + ============== NetfilterQueue ============== @@ -9,6 +17,9 @@ or given a mark. libnetfilter_queue (the netfilter library, not this module) is part of the `Netfilter project `_. +The current version of NetfilterQueue requires Python 3.6 or later. +The last version with support for Python 2.7 was 0.9.0. + Example ======= diff --git a/netfilterqueue/_version.py b/netfilterqueue/_version.py index 5a5b5fe..478d388 100644 --- a/netfilterqueue/_version.py +++ b/netfilterqueue/_version.py @@ -1,4 +1,4 @@ # This file is imported from __init__.py and exec'd from setup.py -__version__ = "0.9.0+dev" -VERSION = (0, 9, 0) +__version__ = "1.0.0" +VERSION = (1, 0, 0) diff --git a/setup.py b/setup.py index 8d95a9a..053b802 100644 --- a/setup.py +++ b/setup.py @@ -44,8 +44,8 @@ name="NetfilterQueue", version=__version__, license="MIT", - author="Matthew Fox", - author_email="matt@tansen.ca", + author="Matthew Fox , Joshua Oreman ", + author_email="oremanj@gmail.com", url="https://github.com/oremanj/python-netfilterqueue", description="Python bindings for libnetfilter_queue", long_description=open("README.rst", encoding="utf-8").read(), From 69436c13287e6e011d59295cedbe0f67510cf8f3 Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Fri, 14 Jan 2022 13:31:43 -0700 Subject: [PATCH 06/11] Bump version to 1.0.0+dev post release --- netfilterqueue/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netfilterqueue/_version.py b/netfilterqueue/_version.py index 478d388..aa14a86 100644 --- a/netfilterqueue/_version.py +++ b/netfilterqueue/_version.py @@ -1,4 +1,4 @@ # This file is imported from __init__.py and exec'd from setup.py -__version__ = "1.0.0" +__version__ = "1.0.0+dev" VERSION = (1, 0, 0) From 9f460d220a818ef534f2c7b92d73c7f1f62daeec Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Tue, 19 Apr 2022 08:01:36 -0700 Subject: [PATCH 07/11] Check whether payload is NULL before accessing it in __str__ --- netfilterqueue/_impl.pyx | 10 +++++++++- setup.py | 4 ++-- tests/test_basic.py | 3 +++ 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/netfilterqueue/_impl.pyx b/netfilterqueue/_impl.pyx index d656d5b..a8e1960 100644 --- a/netfilterqueue/_impl.pyx +++ b/netfilterqueue/_impl.pyx @@ -57,7 +57,15 @@ cdef class Packet: self._given_payload = None def __str__(self): - cdef iphdr *hdr = self.payload + cdef unsigned char *payload = NULL + if self._owned_payload: + payload = self._owned_payload + elif self.payload != NULL: + payload = self.payload + else: + return "%d byte packet, contents unretained" % (self.payload_len,) + + cdef iphdr *hdr = payload protocol = PROTOCOLS.get(hdr.protocol, "Unknown protocol") return "%s packet, %s bytes" % (protocol, self.payload_len) diff --git a/setup.py b/setup.py index 053b802..ae2997f 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ exec(open("netfilterqueue/_version.py", encoding="utf-8").read()) -setup_requires = [] +setup_requires = ["wheel"] try: # Use Cython from Cython.Build import cythonize @@ -21,7 +21,7 @@ if "egg_info" in sys.argv: # We're being run by pip to figure out what we need. Request cython in # setup_requires below. - setup_requires = ["cython"] + setup_requires += ["cython"] elif not os.path.exists( os.path.join(os.path.dirname(__file__), "netfilterqueue/_impl.c") ): diff --git a/tests/test_basic.py b/tests/test_basic.py index 8ec7492..99a331b 100644 --- a/tests/test_basic.py +++ b/tests/test_basic.py @@ -23,6 +23,7 @@ async def test_comms_without_queue(harness): async def test_queue_dropping(harness): async def drop(packets, msg): async for packet in packets: + assert "UDP packet" in str(packet) if packet.get_payload()[28:] == msg: packet.drop() else: @@ -190,6 +191,7 @@ async def test_errors(harness): async def test_unretained(harness): def cb(chan, pkt): # Can access payload within callback + assert "UDP packet" in str(pkt) assert pkt.get_payload()[-3:] in (b"one", b"two") chan.send_nowait(pkt) @@ -202,6 +204,7 @@ def cb(chan, pkt): RuntimeError, match="Payload data is no longer available" ): p.get_payload() + assert "contents unretained" in str(p) # Can still issue verdicts though if accept: p.accept() From 49e16812f7890a986fd68ddb1a8c33cecdca4c52 Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Wed, 1 Mar 2023 00:42:20 -0700 Subject: [PATCH 08/11] Add Packet accessors for interface indices --- .github/workflows/ci.yml | 3 ++- CHANGES.txt | 3 +++ README.rst | 12 ++++++++++++ netfilterqueue/_impl.pxd | 20 ++++++++++---------- netfilterqueue/_impl.pyi | 5 +++++ netfilterqueue/_impl.pyx | 4 ++++ tests/test_basic.py | 18 +++++++++++++++++- 7 files changed, 53 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6c0ce29..e181cea 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,13 +15,14 @@ jobs: fail-fast: false matrix: python: - - '3.6' - '3.7' - '3.8' - '3.9' - '3.10' + - '3.11' - 'pypy-3.7' - 'pypy-3.8' + - 'pypy-3.9' check_lint: ['0'] extra_name: [''] include: diff --git a/CHANGES.txt b/CHANGES.txt index 37c092f..efa6dec 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,3 +1,6 @@ +v1.1.0, unreleased + Add Packet accessors for {indev, outdev, physindev, physoutdev} interface indices + v1.0.0, 14 Jan 2022 Propagate exceptions raised by the user's packet callback Avoid calls to the packet callback during queue unbinding diff --git a/README.rst b/README.rst index 275e173..1ddafda 100644 --- a/README.rst +++ b/README.rst @@ -217,6 +217,18 @@ Objects of this type are passed to your callback. into our queue. Values 0 through 4 correspond to PREROUTING, INPUT, FORWARD, OUTPUT, and POSTROUTING respectively. +``Packet.indev``, ``Packet.outdev``, ``Packet.physindev``, ``Packet.physoutdev`` + The interface indices on which the packet arrived (``indev``) or is slated + to depart (``outdev``). These are integers, which can be converted to + names like "eth0" by using ``socket.if_indextoname()``. Zero means + no interface is applicable, either because the packet was locally generated + or locally received, or because the interface information wasn't available + when the packet was queued (for example, ``PREROUTING`` rules don't yet + know the ``outdev``). If the ``indev`` or ``outdev`` refers to a bridge + device, then the corresponding ``physindev`` or ``physoutdev`` will name + the bridge member on which the actual traffic occurred; otherwise + ``physindev`` and ``physoutdev`` will be zero. + ``Packet.retain()`` Allocate a copy of the packet payload for use after the callback has returned. ``get_payload()`` will raise an exception at that diff --git a/netfilterqueue/_impl.pxd b/netfilterqueue/_impl.pxd index 54c2fa3..d202b41 100644 --- a/netfilterqueue/_impl.pxd +++ b/netfilterqueue/_impl.pxd @@ -153,7 +153,11 @@ cdef extern from "libnetfilter_queue/libnetfilter_queue.h": int nfq_get_payload(nfq_data *nfad, unsigned char **data) int nfq_get_timestamp(nfq_data *nfad, timeval *tv) nfqnl_msg_packet_hw *nfq_get_packet_hw(nfq_data *nfad) - int nfq_get_nfmark (nfq_data *nfad) + int nfq_get_nfmark(nfq_data *nfad) + u_int32_t nfq_get_indev(nfq_data *nfad) + u_int32_t nfq_get_outdev(nfq_data *nfad) + u_int32_t nfq_get_physindev(nfq_data *nfad) + u_int32_t nfq_get_physoutdev(nfq_data *nfad) nfnl_handle *nfq_nfnlh(nfq_handle *h) # Dummy defines from linux/socket.h: @@ -184,8 +188,7 @@ cdef class NetfilterQueue: cdef class Packet: cdef NetfilterQueue _queue - cdef bint _verdict_is_set # True if verdict has been issued, - # false otherwise + cdef bint _verdict_is_set # True if verdict has been issued, false otherwise cdef bint _mark_is_set # True if a mark has been given, false otherwise cdef bint _hwaddr_is_set cdef bint _timestamp_is_set @@ -204,13 +207,10 @@ cdef class Packet: cdef unsigned char *payload cdef timeval timestamp cdef u_int8_t hw_addr[8] - - # TODO: implement these - #cdef readonly u_int32_t nfmark - #cdef readonly u_int32_t indev - #cdef readonly u_int32_t physindev - #cdef readonly u_int32_t outdev - #cdef readonly u_int32_t physoutdev + cdef readonly u_int32_t indev + cdef readonly u_int32_t physindev + cdef readonly u_int32_t outdev + cdef readonly u_int32_t physoutdev cdef set_nfq_data(self, NetfilterQueue queue, nfq_data *nfa) cdef drop_refs(self) diff --git a/netfilterqueue/_impl.pyi b/netfilterqueue/_impl.pyi index 3cba8fc..28c3080 100644 --- a/netfilterqueue/_impl.pyi +++ b/netfilterqueue/_impl.pyi @@ -11,6 +11,11 @@ class Packet: hw_protocol: int id: int mark: int + # These are ifindexes, pass to socket.if_indextoname() to get names: + indev: int + outdev: int + physindev: int + physoutdev: int def get_hw(self) -> Optional[bytes]: ... def get_payload(self) -> bytes: ... def get_payload_len(self) -> int: ... diff --git a/netfilterqueue/_impl.pyx b/netfilterqueue/_impl.pyx index a8e1960..24b0bed 100644 --- a/netfilterqueue/_impl.pyx +++ b/netfilterqueue/_impl.pyx @@ -99,6 +99,10 @@ cdef class Packet: nfq_get_timestamp(nfa, &self.timestamp) self.mark = nfq_get_nfmark(nfa) + self.indev = nfq_get_indev(nfa) + self.outdev = nfq_get_outdev(nfa) + self.physindev = nfq_get_physindev(nfa) + self.physoutdev = nfq_get_physoutdev(nfa) cdef drop_refs(self): """ diff --git a/tests/test_basic.py b/tests/test_basic.py index 99a331b..dbcafa1 100644 --- a/tests/test_basic.py +++ b/tests/test_basic.py @@ -119,11 +119,13 @@ def cb(chan, pkt): assert t0 < timestamps[0] < t1 -async def test_hwaddr(harness): +async def test_hwaddr_and_inoutdev(harness): hwaddrs = [] + inoutdevs = [] def cb(pkt): hwaddrs.append((pkt.get_hw(), pkt.hook, pkt.get_payload()[28:])) + inoutdevs.append((pkt.indev, pkt.outdev)) pkt.accept() queue_num, nfq = harness.bind_queue(cb) @@ -162,6 +164,20 @@ async def listen_for_packets(): (None, OUTPUT, b"four"), ] + if sys.implementation.name != "pypy": + # pypy doesn't appear to provide if_nametoindex() + iface1 = socket.if_nametoindex("veth1") + iface2 = socket.if_nametoindex("veth2") + else: + iface1, iface2 = inoutdevs[0] + assert 0 != iface1 != iface2 != 0 + assert inoutdevs == [ + (iface1, iface2), + (iface1, iface2), + (0, iface2), + (0, iface2), + ] + async def test_errors(harness): with pytest.warns(RuntimeWarning, match="rcvbuf limit is") as record: From 84676504dec3da1ebfd0c78d8c8ee51b9ead70ff Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Wed, 1 Mar 2023 00:51:59 -0700 Subject: [PATCH 09/11] Update README: python-dev is no more --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 1ddafda..d1303e4 100644 --- a/README.rst +++ b/README.rst @@ -79,7 +79,7 @@ Before installing, ensure you have: On Debian or Ubuntu, install these files with:: - apt-get install build-essential python-dev libnetfilter-queue-dev + apt-get install build-essential python3-dev libnetfilter-queue-dev From PyPI --------- From e38557217daa66d350dc009a74629d06d25c665d Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Wed, 1 Mar 2023 00:56:40 -0700 Subject: [PATCH 10/11] Bump version to 1.1.0 for release --- CHANGES.txt | 2 +- netfilterqueue/_version.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index efa6dec..61273ae 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,4 @@ -v1.1.0, unreleased +v1.1.0, 1 Mar 2023 Add Packet accessors for {indev, outdev, physindev, physoutdev} interface indices v1.0.0, 14 Jan 2022 diff --git a/netfilterqueue/_version.py b/netfilterqueue/_version.py index aa14a86..aa81e31 100644 --- a/netfilterqueue/_version.py +++ b/netfilterqueue/_version.py @@ -1,4 +1,4 @@ # This file is imported from __init__.py and exec'd from setup.py -__version__ = "1.0.0+dev" -VERSION = (1, 0, 0) +__version__ = "1.1.0" +VERSION = (1, 1, 0) From e11aaf6b17684f656292a806aa253185dbd83c3c Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Wed, 1 Mar 2023 01:02:21 -0700 Subject: [PATCH 11/11] Bump version to 1.1.0+dev post release --- netfilterqueue/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netfilterqueue/_version.py b/netfilterqueue/_version.py index aa81e31..cb817a6 100644 --- a/netfilterqueue/_version.py +++ b/netfilterqueue/_version.py @@ -1,4 +1,4 @@ # This file is imported from __init__.py and exec'd from setup.py -__version__ = "1.1.0" +__version__ = "1.1.0+dev" VERSION = (1, 1, 0)