Add rate limits and move dev requirements into pyproject.toml

.github/workflows/main.yml

@@ -23,8 +23,7 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install pylint black ratelimit requests urllib3 python-squarelet python_dateutil pytest
-
+        pip install -e ".[dev]"
     - name: Run Pylint on muckrock directory
       run: |
         pylint --disable=missing-function-docstring,too-many-instance-attributes,too-many-arguments,too-many-positional-arguments src/muckrock
@@ -46,15 +45,14 @@ jobs:
     steps:
       - name: Check out code
         uses: actions/checkout@v2
-
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install -r requirements.txt
+          pip install -e ".[dev]"
       - name: Run tests
         run: |
           pytest src/muckrock/tests.py

docs/changelog.rst

@@ -1,5 +1,8 @@
 Changelog
 ---------
+2.3.0
+~~~~~
+* Adds sane burst rate limits to endpoints. 
 
 2.2.0
 ~~~~~

docs/conf.py

@@ -55,9 +55,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = "2.2"
+version = "2.3"
 # The full version, including alpha/beta/rc tags.
-release = "2.2.0"
+release = "2.3.0"
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

pyproject.toml

@@ -3,14 +3,14 @@ requires = ["hatchling"]
 build-backend = "hatchling.build"
 
 [project]
-name = "python-muckrock" 
-version = "2.2.0"
+name = "python-muckrock"
+version = "2.3.0"
 authors = [
   { name="duckduckgrayduck", email="sanjin@muckrock.com" },
 ]
 description = "A simple Python wrapper for the MuckRock API v2"
 readme = "README.md"
-requires-python = ">=3.7,<=3.12"
+requires-python = ">=3.7,<3.13"
 classifiers = [
     "Programming Language :: Python :: 3",
     "License :: OSI Approved :: MIT License",
@@ -20,13 +20,22 @@ dependencies = [
     "requests",
     "ratelimit",
     "urllib3",
-    "python-squarelet"
+    "python-squarelet",
+    "token-bucket",
+    "python-dateutil",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pylint",
+    "black",
+    "pytest",
 ]
 
 [project.urls]
 "Homepage" = "https://github.com/MuckRock/python-muckrock"
 "Bug Tracker" = "https://github.com/MuckRock/python-muckrock/issues"
-"Documentation" ="https://python-muckrock.readthedocs.io/en/latest/"
+"Documentation" = "https://python-muckrock.readthedocs.io/en/latest/"
 
 [tool.hatch.build.targets.wheel]
-packages = ["src/muckrock"]
+packages = ["src/muckrock"]

src/muckrock/client.py

@@ -1,9 +1,13 @@
-""" Provides the client wrapper with Squarelet """
+"""
+Provides the client wrapper with Squarelet
+"""
 
 # Standard Library
 import logging
+import time
 
 # Third Party
+import token_bucket
 from squarelet import SquareletClient
 
 from .agencies import AgencyClient
@@ -12,19 +16,44 @@
 from .jurisdictions import JurisdictionClient
 from .organizations import OrganizationClient
 from .projects import ProjectClient
+
 # Local Imports
 from .requests import RequestClient
 from .users import UserClient
 
 logger = logging.getLogger("muckrock")
 
+# Per-endpoint rate limits.
+# Format: (url_pattern, rate_per_second, capacity)
+#
+# Endpoint          Rate        Burst   Notes
+# --------          ----        -----   -----
+# requests/          15/min      100
+# communications/    15/min      100
+# agencies/          15/min      100
+# files/             15/min      100
+# jurisdictions/     15/min      100
+# projects/          15/min      100
+# organizations/      5/min        5    Heavy rate limit, minimal burst
+# users/              5/min        5    Heavy rate limit, minimal burst
+ENDPOINT_RATE_LIMITS = [
+    ("organizations", 5 / 60, 5),
+    ("users", 5 / 60, 5),
+    ("requests", 15 / 60, 100),
+    ("communications", 15 / 60, 100),
+    ("agencies", 15 / 60, 100),
+    ("files", 15 / 60, 100),
+    ("jurisdictions", 15 / 60, 100),
+    ("projects", 15 / 60, 100),
+]
+
 
-class MuckRock(SquareletClient):
+class MuckRock(SquareletClient):  # pylint:disable=too-many-instance-attributes
     """
     The public interface for the MuckRock API, now integrated with SquareletClient
     """
 
-    # pylint:disable=too-many-positional-arguments
+    # pylint:disable=too-many-positional-arguments, too-many-arguments
     def __init__(
         self,
         username=None,
@@ -56,6 +85,17 @@ def __init__(
         else:
             logger.addHandler(logging.NullHandler())
 
+        # Build per-endpoint token bucket rate limiters
+        storage = token_bucket.MemoryStorage()
+        self._endpoint_limiters = [
+            (
+                pattern,
+                token_bucket.Limiter(rate=rate, capacity=capacity, storage=storage),
+                pattern,
+            )
+            for pattern, rate, capacity in ENDPOINT_RATE_LIMITS
+        ]
+
         self.requests = RequestClient(self)
         self.jurisdictions = JurisdictionClient(self)
         self.agencies = AgencyClient(self)
@@ -64,3 +104,13 @@ def __init__(
         self.organizations = OrganizationClient(self)
         self.users = UserClient(self)
         self.projects = ProjectClient(self)
+
+    def request(self, method, url, raise_error=True, **kwargs):
+        for pattern, limiter, bucket_key in self._endpoint_limiters:
+            if pattern in url:
+                if not limiter.consume(bucket_key):
+                    logger.warning("Rate limit reached for %s, throttling...", pattern)
+                    while not limiter.consume(bucket_key):
+                        time.sleep(0.1)
+                return super().request(method, url, raise_error=raise_error, **kwargs)
+        return super().request(method, url, raise_error=raise_error, **kwargs)