Major cosmetic changes to limits, but little-to-no functional changes. MUCH better testability now, no more relying on system time to tick by for limit testing.
This commit is contained in:
+1
-1
@@ -79,7 +79,7 @@ paste.filter_factory = nova.api.openstack:FaultWrapper.factory
|
||||
paste.filter_factory = nova.api.openstack.auth:AuthMiddleware.factory
|
||||
|
||||
[filter:ratelimit]
|
||||
paste.filter_factory = nova.api.openstack.ratelimiting:RateLimitingMiddleware.factory
|
||||
paste.filter_factory = nova.api.openstack.limits:RateLimitingMiddleware.factory
|
||||
|
||||
[app:osapiapp]
|
||||
paste.app_factory = nova.api.openstack:APIRouter.factory
|
||||
|
||||
@@ -33,6 +33,7 @@ from nova.api.openstack import backup_schedules
|
||||
from nova.api.openstack import consoles
|
||||
from nova.api.openstack import flavors
|
||||
from nova.api.openstack import images
|
||||
from nova.api.openstack import limits
|
||||
from nova.api.openstack import servers
|
||||
from nova.api.openstack import shared_ip_groups
|
||||
from nova.api.openstack import users
|
||||
@@ -114,12 +115,17 @@ class APIRouter(wsgi.Router):
|
||||
|
||||
mapper.resource("image", "images", controller=images.Controller(),
|
||||
collection={'detail': 'GET'})
|
||||
|
||||
mapper.resource("flavor", "flavors", controller=flavors.Controller(),
|
||||
collection={'detail': 'GET'})
|
||||
|
||||
mapper.resource("shared_ip_group", "shared_ip_groups",
|
||||
collection={'detail': 'GET'},
|
||||
controller=shared_ip_groups.Controller())
|
||||
|
||||
_limits = limits.LimitsController()
|
||||
mapper.resource("limit", "limits", controller=_limits)
|
||||
|
||||
super(APIRouter, self).__init__(mapper)
|
||||
|
||||
|
||||
|
||||
@@ -61,3 +61,24 @@ class Fault(webob.exc.HTTPException):
|
||||
content_type = req.best_match_content_type()
|
||||
self.wrapped_exc.body = serializer.serialize(fault_data, content_type)
|
||||
return self.wrapped_exc
|
||||
|
||||
|
||||
class OverLimitFault(webob.exc.HTTPException):
|
||||
"""
|
||||
Rate-limited request response.
|
||||
"""
|
||||
|
||||
wrapped_exc = webob.exc.HTTPForbidden()
|
||||
|
||||
def __init__(self, message, details, retry_time):
|
||||
"""
|
||||
Initialize new `OverLimitFault` with relevant information.
|
||||
"""
|
||||
self.message = message
|
||||
self.details = details
|
||||
self.retry_time = retry_time
|
||||
|
||||
@webob.dec.wsgify(RequestClass=wsgi.Request)
|
||||
def __call__(self, request):
|
||||
"""Currently just return the wrapped exception."""
|
||||
return self.wrapped_exc
|
||||
|
||||
@@ -0,0 +1,342 @@
|
||||
# vim: tabstop=4 shiftwidth=4 softtabstop=4
|
||||
|
||||
# Copyright 2010 OpenStack LLC.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.import datetime
|
||||
|
||||
"""
|
||||
Module dedicated functions/classes dealing with rate limiting requests.
|
||||
"""
|
||||
|
||||
import copy
|
||||
import httplib
|
||||
import json
|
||||
import math
|
||||
import re
|
||||
import time
|
||||
import urllib
|
||||
import webob.exc
|
||||
|
||||
from collections import defaultdict
|
||||
|
||||
from webob.dec import wsgify
|
||||
|
||||
from nova import wsgi
|
||||
from nova.api.openstack import faults
|
||||
from nova.wsgi import Controller
|
||||
from nova.wsgi import Middleware
|
||||
|
||||
|
||||
# Convenience constants for the limits dictionary passed to Limiter().
|
||||
PER_SECOND = 1
|
||||
PER_MINUTE = 60
|
||||
PER_HOUR = 60 * 60
|
||||
PER_DAY = 60 * 60 * 24
|
||||
|
||||
|
||||
class LimitsController(Controller):
|
||||
"""
|
||||
Controller for accessing limits in the OpenStack API.
|
||||
"""
|
||||
|
||||
def index(self, req):
|
||||
"""
|
||||
Return all global and rate limit information.
|
||||
"""
|
||||
abs_limits = {}
|
||||
rate_limits = req.environ.get("nova.limits", {})
|
||||
|
||||
return {
|
||||
"limits" : {
|
||||
"rate" : rate_limits,
|
||||
"absolute" : abs_limits,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class Limit(object):
|
||||
"""
|
||||
Stores information about a limit for HTTP requets.
|
||||
"""
|
||||
|
||||
UNITS = {
|
||||
1 : "SECOND",
|
||||
60 : "MINUTE",
|
||||
60 * 60 : "HOUR",
|
||||
60 * 60 * 24 : "DAY",
|
||||
}
|
||||
|
||||
def __init__(self, verb, uri, regex, value, unit):
|
||||
"""
|
||||
Initialize a new `Limit`.
|
||||
|
||||
@param verb: HTTP verb (POST, PUT, etc.)
|
||||
@param uri: Human-readable URI
|
||||
@param regex: Regular expression format for this limit
|
||||
@param value: Integer number of requests which can be made
|
||||
@param unit: Unit of measure for the value parameter
|
||||
"""
|
||||
self.verb = verb
|
||||
self.uri = uri
|
||||
self.regex = regex
|
||||
self.value = int(value)
|
||||
self.unit = unit
|
||||
self.remaining = int(value)
|
||||
|
||||
if value <= 0:
|
||||
raise ValueError("Limit value must be > 0")
|
||||
|
||||
self.last_request = None
|
||||
self.next_request = None
|
||||
|
||||
self.water_level = 0
|
||||
self.capacity = float(self.unit)
|
||||
self.request_value = float(self.capacity) / float(self.value)
|
||||
|
||||
def __call__(self, verb, url):
|
||||
"""
|
||||
Represents a call to this limit from a relevant request.
|
||||
|
||||
@param verb: string http verb (POST, GET, etc.)
|
||||
@param url: string URL
|
||||
"""
|
||||
if self.verb != verb or not re.match(self.regex, url):
|
||||
return
|
||||
|
||||
now = self._get_time()
|
||||
|
||||
if self.last_request is None:
|
||||
self.last_request = now
|
||||
|
||||
leak_value = now - self.last_request
|
||||
|
||||
self.water_level -= leak_value
|
||||
self.water_level = max(self.water_level, 0)
|
||||
self.water_level += self.request_value
|
||||
|
||||
difference = self.water_level - self.capacity
|
||||
|
||||
self.last_request = now
|
||||
|
||||
if difference > 0:
|
||||
self.water_level -= self.request_value
|
||||
self.next_request = now + difference
|
||||
return difference
|
||||
|
||||
cap = self.capacity
|
||||
water = self.water_level
|
||||
val = self.value
|
||||
|
||||
self.remaining = math.floor((cap - water) / cap * val)
|
||||
print "Remaining:", self.remaining
|
||||
self.next_request = now
|
||||
|
||||
def _get_time(self):
|
||||
"""Retrieve the current time. Broken out for testability."""
|
||||
return time.time()
|
||||
|
||||
def display_unit(self):
|
||||
"""Display the string name of the unit."""
|
||||
return self.UNITS.get(self.unit, "UNKNOWN")
|
||||
|
||||
def display(self):
|
||||
"""Return a useful representation of this class."""
|
||||
return {
|
||||
"verb" : self.verb,
|
||||
"uri" : self.uri,
|
||||
"regex" : self.regex,
|
||||
"value" : self.value,
|
||||
"remaining" : int(self.remaining),
|
||||
"unit" : self.display_unit(),
|
||||
"resetTime" : int(self.next_request or self._get_time()),
|
||||
}
|
||||
|
||||
|
||||
|
||||
# "Limit" format is a dictionary with the HTTP verb, human-readable URI,
|
||||
# a regular-expression to match, value and unit of measure (PER_DAY, etc.)
|
||||
|
||||
DEFAULT_LIMITS = [
|
||||
Limit("POST", "*", ".*", 10, PER_MINUTE),
|
||||
Limit("POST", "*/servers", "^/servers", 50, PER_DAY),
|
||||
Limit("PUT", "*", ".*", 10, PER_MINUTE),
|
||||
Limit("GET", "*changes-since*", ".*changes-since.*", 3, PER_MINUTE),
|
||||
Limit("DELETE", "*", ".*", 100, PER_MINUTE),
|
||||
]
|
||||
|
||||
|
||||
class RateLimitingMiddleware(Middleware):
|
||||
"""
|
||||
Rate-limits requests passing through this middleware. All limit information
|
||||
is stored in memory for this implementation.
|
||||
"""
|
||||
|
||||
def __init__(self, application, limits=None):
|
||||
"""
|
||||
Initialize new `RateLimitingMiddleware`, which wraps the given WSGI
|
||||
application and sets up the given limits.
|
||||
|
||||
@param application: WSGI application to wrap
|
||||
@param limits: List of dictionaries describing limits
|
||||
"""
|
||||
Middleware.__init__(self, application)
|
||||
self._limiter = Limiter(limits or DEFAULT_LIMITS)
|
||||
|
||||
@wsgify(RequestClass=wsgi.Request)
|
||||
def __call__(self, req):
|
||||
"""
|
||||
Represents a single call through this middleware. We should record the
|
||||
request if we have a limit relevant to it. If no limit is relevant to
|
||||
the request, ignore it.
|
||||
|
||||
If the request should be rate limited, return a fault telling the user
|
||||
they are over the limit and need to retry later.
|
||||
"""
|
||||
verb = req.method
|
||||
url = req.url
|
||||
username = req.environ["nova.context"].user_id
|
||||
|
||||
delay = self._limiter.check_for_delay(verb, url, username)
|
||||
|
||||
if delay:
|
||||
msg = "This request was rate-limited."
|
||||
details = "Error details."
|
||||
retry = time.time() + delay
|
||||
return faults.OverLimitFault(msg, details, retry)
|
||||
|
||||
req.environ["nova.limits"] = self._limiter.get_limits(username)
|
||||
|
||||
return self.application
|
||||
|
||||
|
||||
class Limiter(object):
|
||||
"""
|
||||
Rate-limit checking class which handles limits in memory.
|
||||
"""
|
||||
|
||||
def __init__(self, limits):
|
||||
"""
|
||||
Initialize the new `Limiter`.
|
||||
|
||||
@param limits: List of `Limit` objects
|
||||
"""
|
||||
self.limits = copy.deepcopy(limits)
|
||||
self.levels = defaultdict(lambda: copy.deepcopy(limits))
|
||||
|
||||
def get_limits(self, username=None):
|
||||
"""
|
||||
Return the limits for a given user.
|
||||
"""
|
||||
return [limit.display() for limit in self.levels[username]]
|
||||
|
||||
def check_for_delay(self, verb, url, username=None):
|
||||
"""
|
||||
Check the given verb/user/user triplet for limit.
|
||||
"""
|
||||
def _get_delay_list():
|
||||
"""Yield limit delays."""
|
||||
for limit in self.levels[username]:
|
||||
delay = limit(verb, url)
|
||||
if delay:
|
||||
yield delay
|
||||
|
||||
delays = list(_get_delay_list())
|
||||
|
||||
if delays:
|
||||
delays.sort()
|
||||
return delays[0]
|
||||
|
||||
|
||||
class WsgiLimiter(object):
|
||||
"""
|
||||
Rate-limit checking from a WSGI application. Uses an in-memory `Limiter`.
|
||||
|
||||
To use:
|
||||
POST /<username> with JSON data such as:
|
||||
{
|
||||
"verb" : GET,
|
||||
"path" : "/servers"
|
||||
}
|
||||
|
||||
and receive a 204 No Content, or a 403 Forbidden with an X-Wait-Seconds
|
||||
header containing the number of seconds to wait before the action would
|
||||
succeed.
|
||||
"""
|
||||
|
||||
def __init__(self, limits=None):
|
||||
"""
|
||||
Initialize the new `WsgiLimiter`.
|
||||
|
||||
@param limits: List of `Limit` objects
|
||||
"""
|
||||
self._limiter = Limiter(limits or DEFAULT_LIMITS)
|
||||
|
||||
@wsgify(RequestClass=wsgi.Request)
|
||||
def __call__(self, request):
|
||||
"""
|
||||
Handles a call to this application. Returns 204 if the request is
|
||||
acceptable to the limiter, else a 403 is returned with a relevant
|
||||
header indicating when the request *will* succeed.
|
||||
"""
|
||||
if request.method != "POST":
|
||||
raise webob.exc.HTTPMethodNotAllowed()
|
||||
|
||||
try:
|
||||
info = dict(json.loads(request.body))
|
||||
except ValueError:
|
||||
raise webob.exc.HTTPBadRequest()
|
||||
|
||||
username = request.path_info_pop()
|
||||
verb = info.get("verb")
|
||||
path = info.get("path")
|
||||
|
||||
delay = self._limiter.check_for_delay(verb, path, username)
|
||||
|
||||
if delay:
|
||||
headers = {"X-Wait-Seconds": "%.2f" % delay}
|
||||
return webob.exc.HTTPForbidden(headers=headers)
|
||||
else:
|
||||
return webob.exc.HTTPNoContent()
|
||||
|
||||
|
||||
class WsgiLimiterProxy(object):
|
||||
"""
|
||||
Rate-limit requests based on answers from a remote source.
|
||||
"""
|
||||
|
||||
def __init__(self, limiter_address):
|
||||
"""
|
||||
Initialize the new `WsgiLimiterProxy`.
|
||||
|
||||
@param limiter_address: IP/port combination of where to request limit
|
||||
"""
|
||||
self.limiter_address = limiter_address
|
||||
|
||||
def check_for_delay(self, verb, path, username=None):
|
||||
body = json.dumps({"verb":verb,"path":path})
|
||||
headers = {"Content-Type" : "application/json"}
|
||||
|
||||
conn = httplib.HTTPConnection(self.limiter_address)
|
||||
|
||||
if username:
|
||||
conn.request("POST", "/%s" % (username), body, headers)
|
||||
else:
|
||||
conn.request("POST", "/", body, headers)
|
||||
|
||||
resp = conn.getresponse()
|
||||
|
||||
if 200 >= resp.status < 300:
|
||||
return None
|
||||
|
||||
return resp.getheader("X-Wait-Seconds")
|
||||
@@ -20,7 +20,7 @@ from nova import test
|
||||
|
||||
from nova import context
|
||||
from nova import flags
|
||||
from nova.api.openstack.ratelimiting import RateLimitingMiddleware
|
||||
from nova.api.openstack.limits import RateLimitingMiddleware
|
||||
from nova.api.openstack.common import limited
|
||||
from nova.tests.api.openstack import fakes
|
||||
from webob import Request
|
||||
|
||||
@@ -34,7 +34,7 @@ from nova import utils
|
||||
import nova.api.openstack.auth
|
||||
from nova.api import openstack
|
||||
from nova.api.openstack import auth
|
||||
from nova.api.openstack import ratelimiting
|
||||
from nova.api.openstack import limits
|
||||
from nova.auth.manager import User, Project
|
||||
from nova.image import glance
|
||||
from nova.image import local
|
||||
@@ -79,7 +79,7 @@ def wsgi_app(inner_application=None):
|
||||
inner_application = openstack.APIRouter()
|
||||
mapper = urlmap.URLMap()
|
||||
api = openstack.FaultWrapper(auth.AuthMiddleware(
|
||||
ratelimiting.RateLimitingMiddleware(inner_application)))
|
||||
limits.RateLimitingMiddleware(inner_application)))
|
||||
mapper['/v1.0'] = api
|
||||
mapper['/'] = openstack.FaultWrapper(openstack.Versions())
|
||||
return mapper
|
||||
@@ -110,13 +110,13 @@ def stub_out_auth(stubs):
|
||||
|
||||
def stub_out_rate_limiting(stubs):
|
||||
def fake_rate_init(self, app):
|
||||
super(ratelimiting.RateLimitingMiddleware, self).__init__(app)
|
||||
super(limits.RateLimitingMiddleware, self).__init__(app)
|
||||
self.application = app
|
||||
|
||||
stubs.Set(nova.api.openstack.ratelimiting.RateLimitingMiddleware,
|
||||
stubs.Set(nova.api.openstack.limits.RateLimitingMiddleware,
|
||||
'__init__', fake_rate_init)
|
||||
|
||||
stubs.Set(nova.api.openstack.ratelimiting.RateLimitingMiddleware,
|
||||
stubs.Set(nova.api.openstack.limits.RateLimitingMiddleware,
|
||||
'__call__', fake_wsgi)
|
||||
|
||||
|
||||
|
||||
@@ -23,7 +23,6 @@ from paste import urlmap
|
||||
from nova import flags
|
||||
from nova import test
|
||||
from nova.api import openstack
|
||||
from nova.api.openstack import ratelimiting
|
||||
from nova.api.openstack import auth
|
||||
from nova.tests.api.openstack import fakes
|
||||
|
||||
|
||||
@@ -1,178 +1,321 @@
|
||||
# vim: tabstop=4 shiftwidth=4 softtabstop=4
|
||||
|
||||
# Copyright 2010 OpenStack LLC.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""
|
||||
Tests dealing with HTTP rate-limiting.
|
||||
"""
|
||||
|
||||
import httplib
|
||||
import json
|
||||
import StringIO
|
||||
import stubout
|
||||
import time
|
||||
import webob
|
||||
|
||||
from nova import test
|
||||
import nova.api.openstack.ratelimiting as ratelimiting
|
||||
from nova.api.openstack import limits
|
||||
from nova.api.openstack.limits import Limit
|
||||
|
||||
|
||||
TEST_LIMITS = [
|
||||
Limit("GET", "/delayed", "^/delayed", 1, limits.PER_MINUTE),
|
||||
Limit("POST", "*", ".*", 7, limits.PER_MINUTE),
|
||||
Limit("POST", "/servers", "^/servers", 3, limits.PER_MINUTE),
|
||||
Limit("PUT", "*", "", 10, limits.PER_MINUTE),
|
||||
Limit("PUT", "/servers", "^/servers", 5, limits.PER_MINUTE),
|
||||
]
|
||||
|
||||
class LimiterTest(test.TestCase):
|
||||
"""
|
||||
Tests for the in-memory `limits.Limiter` class.
|
||||
"""
|
||||
|
||||
def setUp(self):
|
||||
super(LimiterTest, self).setUp()
|
||||
self.limits = {
|
||||
'a': (5, ratelimiting.PER_SECOND),
|
||||
'b': (5, ratelimiting.PER_MINUTE),
|
||||
'c': (5, ratelimiting.PER_HOUR),
|
||||
'd': (1, ratelimiting.PER_SECOND),
|
||||
'e': (100, ratelimiting.PER_SECOND)}
|
||||
self.rl = ratelimiting.Limiter(self.limits)
|
||||
"""Run before each test."""
|
||||
test.TestCase.setUp(self)
|
||||
self.time = 0.0
|
||||
self.stubs = stubout.StubOutForTesting()
|
||||
self.stubs.Set(limits.Limit, "_get_time", self._get_time)
|
||||
self.limiter = limits.Limiter(TEST_LIMITS)
|
||||
|
||||
def exhaust(self, action, times_until_exhausted, **kwargs):
|
||||
for i in range(times_until_exhausted):
|
||||
when = self.rl.perform(action, **kwargs)
|
||||
self.assertEqual(when, None)
|
||||
num, period = self.limits[action]
|
||||
delay = period * 1.0 / num
|
||||
# Verify that we are now thoroughly delayed
|
||||
for i in range(10):
|
||||
when = self.rl.perform(action, **kwargs)
|
||||
self.assertAlmostEqual(when, delay, 2)
|
||||
def tearDown(self):
|
||||
"""Run after each test."""
|
||||
self.stubs.UnsetAll()
|
||||
|
||||
def test_second(self):
|
||||
self.exhaust('a', 5)
|
||||
time.sleep(0.2)
|
||||
self.exhaust('a', 1)
|
||||
time.sleep(1)
|
||||
self.exhaust('a', 5)
|
||||
def _get_time(self):
|
||||
"""Return the "time" according to this test suite."""
|
||||
return self.time
|
||||
|
||||
def test_minute(self):
|
||||
self.exhaust('b', 5)
|
||||
def _check(self, num, verb, url, username=None):
|
||||
"""Check and yield results from checks."""
|
||||
for x in xrange(num):
|
||||
yield self.limiter.check_for_delay(verb, url, username)
|
||||
|
||||
def test_one_per_period(self):
|
||||
def allow_once_and_deny_once():
|
||||
when = self.rl.perform('d')
|
||||
self.assertEqual(when, None)
|
||||
when = self.rl.perform('d')
|
||||
self.assertAlmostEqual(when, 1, 2)
|
||||
return when
|
||||
time.sleep(allow_once_and_deny_once())
|
||||
time.sleep(allow_once_and_deny_once())
|
||||
allow_once_and_deny_once()
|
||||
def _check_sum(self, num, verb, url, username=None):
|
||||
"""Check and sum results from checks."""
|
||||
results = self._check(num, verb, url, username)
|
||||
return sum(filter(lambda x: x != None, results))
|
||||
|
||||
def test_we_can_go_indefinitely_if_we_spread_out_requests(self):
|
||||
for i in range(200):
|
||||
when = self.rl.perform('e')
|
||||
self.assertEqual(when, None)
|
||||
time.sleep(0.01)
|
||||
def test_no_delay_GET(self):
|
||||
"""
|
||||
Simple test to ensure no delay on a single call for a limit verb we
|
||||
didn"t set.
|
||||
"""
|
||||
delay = self.limiter.check_for_delay("GET", "/anything")
|
||||
self.assertEqual(delay, None)
|
||||
|
||||
def test_users_get_separate_buckets(self):
|
||||
self.exhaust('c', 5, username='alice')
|
||||
self.exhaust('c', 5, username='bob')
|
||||
self.exhaust('c', 5, username='chuck')
|
||||
self.exhaust('c', 0, username='chuck')
|
||||
self.exhaust('c', 0, username='bob')
|
||||
self.exhaust('c', 0, username='alice')
|
||||
def test_no_delay_PUT(self):
|
||||
"""
|
||||
Simple test to ensure no delay on a single call for a known limit.
|
||||
"""
|
||||
delay = self.limiter.check_for_delay("PUT", "/anything")
|
||||
self.assertEqual(delay, None)
|
||||
|
||||
def test_delay_PUT(self):
|
||||
"""
|
||||
Ensure the 11th PUT will result in a delay of 6.0 seconds until
|
||||
the next request will be granced.
|
||||
"""
|
||||
expected = [None] * 10 + [6.0]
|
||||
results = list(self._check(11, "PUT", "/anything"))
|
||||
|
||||
class FakeLimiter(object):
|
||||
"""Fake Limiter class that you can tell how to behave."""
|
||||
self.assertEqual(expected, results)
|
||||
|
||||
def __init__(self, test):
|
||||
self._action = self._username = self._delay = None
|
||||
self.test = test
|
||||
def test_delay_POST(self):
|
||||
"""
|
||||
Ensure the 8th POST will result in a delay of 6.0 seconds until
|
||||
the next request will be granced.
|
||||
"""
|
||||
expected = [None] * 7
|
||||
results = list(self._check(7, "POST", "/anything"))
|
||||
self.assertEqual(expected, results)
|
||||
|
||||
def mock(self, action, username, delay):
|
||||
self._action = action
|
||||
self._username = username
|
||||
self._delay = delay
|
||||
expected = 60.0 / 7.0
|
||||
results = self._check_sum(1, "POST", "/anything")
|
||||
self.failUnlessAlmostEqual(expected, results, 8)
|
||||
|
||||
def test_delay_GET(self):
|
||||
"""
|
||||
Ensure the 11th GET will result in NO delay.
|
||||
"""
|
||||
expected = [None] * 11
|
||||
results = list(self._check(11, "GET", "/anything"))
|
||||
|
||||
def perform(self, action, username):
|
||||
self.test.assertEqual(action, self._action)
|
||||
self.test.assertEqual(username, self._username)
|
||||
return self._delay
|
||||
self.assertEqual(expected, results)
|
||||
|
||||
def test_delay_PUT_servers(self):
|
||||
"""
|
||||
Ensure PUT on /servers limits at 5 requests, and PUT elsewhere is still
|
||||
OK after 5 requests...but then after 11 total requests, PUT limiting
|
||||
kicks in.
|
||||
"""
|
||||
# First 6 requests on PUT /servers
|
||||
expected = [None] * 5 + [12.0]
|
||||
results = list(self._check(6, "PUT", "/servers"))
|
||||
self.assertEqual(expected, results)
|
||||
|
||||
class WSGIAppTest(test.TestCase):
|
||||
# Next 5 request on PUT /anything
|
||||
expected = [None] * 4 + [6.0]
|
||||
results = list(self._check(5, "PUT", "/anything"))
|
||||
self.assertEqual(expected, results)
|
||||
|
||||
def test_delay_PUT_wait(self):
|
||||
"""
|
||||
Ensure after hitting the limit and then waiting for the correct
|
||||
amount of time, the limit will be lifted.
|
||||
"""
|
||||
expected = [None] * 10 + [6.0]
|
||||
results = list(self._check(11, "PUT", "/anything"))
|
||||
self.assertEqual(expected, results)
|
||||
|
||||
# Advance time
|
||||
self.time += 6.0
|
||||
|
||||
expected = [None, 6.0]
|
||||
results = list(self._check(2, "PUT", "/anything"))
|
||||
self.assertEqual(expected, results)
|
||||
|
||||
def test_multiple_delays(self):
|
||||
"""
|
||||
Ensure multiple requests still get a delay.
|
||||
"""
|
||||
expected = [None] * 10 + [6.0] * 10
|
||||
results = list(self._check(20, "PUT", "/anything"))
|
||||
self.assertEqual(expected, results)
|
||||
|
||||
self.time += 1.0
|
||||
|
||||
expected = [5.0] * 10
|
||||
results = list(self._check(10, "PUT", "/anything"))
|
||||
self.assertEqual(expected, results)
|
||||
|
||||
def test_multiple_users(self):
|
||||
"""
|
||||
Tests involving multiple users.
|
||||
"""
|
||||
# User1
|
||||
expected = [None] * 10 + [6.0] * 10
|
||||
results = list(self._check(20, "PUT", "/anything", "user1"))
|
||||
self.assertEqual(expected, results)
|
||||
|
||||
# User2
|
||||
expected = [None] * 10 + [6.0] * 5
|
||||
results = list(self._check(15, "PUT", "/anything", "user2"))
|
||||
self.assertEqual(expected, results)
|
||||
|
||||
self.time += 1.0
|
||||
|
||||
# User1 again
|
||||
expected = [5.0] * 10
|
||||
results = list(self._check(10, "PUT", "/anything", "user1"))
|
||||
self.assertEqual(expected, results)
|
||||
|
||||
self.time += 1.0
|
||||
|
||||
# User1 again
|
||||
expected = [4.0] * 5
|
||||
results = list(self._check(5, "PUT", "/anything", "user2"))
|
||||
self.assertEqual(expected, results)
|
||||
|
||||
|
||||
class WsgiLimiterTest(test.TestCase):
|
||||
"""
|
||||
Tests for `limits.WsgiLimiter` class.
|
||||
"""
|
||||
|
||||
def setUp(self):
|
||||
super(WSGIAppTest, self).setUp()
|
||||
self.limiter = FakeLimiter(self)
|
||||
self.app = ratelimiting.WSGIApp(self.limiter)
|
||||
"""Run before each test."""
|
||||
test.TestCase.setUp(self)
|
||||
self.time = 0.0
|
||||
self.app = limits.WsgiLimiter(TEST_LIMITS)
|
||||
self.app._limiter._get_time = self._get_time
|
||||
|
||||
def test_invalid_methods(self):
|
||||
requests = []
|
||||
for method in ['GET', 'PUT', 'DELETE']:
|
||||
req = webob.Request.blank('/limits/michael/breakdance',
|
||||
dict(REQUEST_METHOD=method))
|
||||
requests.append(req)
|
||||
for req in requests:
|
||||
self.assertEqual(req.get_response(self.app).status_int, 405)
|
||||
def _get_time(self):
|
||||
"""Return the "time" according to this test suite."""
|
||||
return self.time
|
||||
|
||||
def test_invalid_urls(self):
|
||||
requests = []
|
||||
for prefix in ['limit', '', 'limiter2', 'limiter/limits', 'limiter/1']:
|
||||
req = webob.Request.blank('/%s/michael/breakdance' % prefix,
|
||||
dict(REQUEST_METHOD='POST'))
|
||||
requests.append(req)
|
||||
for req in requests:
|
||||
self.assertEqual(req.get_response(self.app).status_int, 404)
|
||||
def _request_data(self, verb, path):
|
||||
"""Get data decribing a limit request verb/path."""
|
||||
return json.dumps({"verb":verb, "path":path})
|
||||
|
||||
def verify(self, url, username, action, delay=None):
|
||||
def _request(self, verb, url, username=None):
|
||||
"""Make sure that POSTing to the given url causes the given username
|
||||
to perform the given action. Make the internal rate limiter return
|
||||
delay and make sure that the WSGI app returns the correct response.
|
||||
"""
|
||||
req = webob.Request.blank(url, dict(REQUEST_METHOD='POST'))
|
||||
self.limiter.mock(action, username, delay)
|
||||
resp = req.get_response(self.app)
|
||||
if not delay:
|
||||
self.assertEqual(resp.status_int, 200)
|
||||
if username:
|
||||
request = webob.Request.blank("/%s" % username)
|
||||
else:
|
||||
self.assertEqual(resp.status_int, 403)
|
||||
self.assertEqual(resp.headers['X-Wait-Seconds'], "%.2f" % delay)
|
||||
request = webob.Request.blank("/")
|
||||
|
||||
request.method = "POST"
|
||||
request.body = self._request_data(verb, url)
|
||||
response = request.get_response(self.app)
|
||||
|
||||
def test_good_urls(self):
|
||||
self.verify('/limiter/michael/hoot', 'michael', 'hoot')
|
||||
if "X-Wait-Seconds" in response.headers:
|
||||
self.assertEqual(response.status_int, 403)
|
||||
return response.headers["X-Wait-Seconds"]
|
||||
|
||||
self.assertEqual(response.status_int, 204)
|
||||
|
||||
def test_invalid_methods(self):
|
||||
"""Only POSTs should work."""
|
||||
requests = []
|
||||
for method in ["GET", "PUT", "DELETE", "HEAD", "OPTIONS"]:
|
||||
request = webob.Request.blank("/")
|
||||
request.body = self._request_data("GET", "/something")
|
||||
response = request.get_response(self.app)
|
||||
self.assertEqual(response.status_int, 405)
|
||||
|
||||
def test_good_url(self):
|
||||
delay = self._request("GET", "/something")
|
||||
self.assertEqual(delay, None)
|
||||
|
||||
def test_escaping(self):
|
||||
self.verify('/limiter/michael/jump%20up', 'michael', 'jump up')
|
||||
delay = self._request("GET", "/something/jump%20up")
|
||||
self.assertEqual(delay, None)
|
||||
|
||||
def test_response_to_delays(self):
|
||||
self.verify('/limiter/michael/hoot', 'michael', 'hoot', 1)
|
||||
self.verify('/limiter/michael/hoot', 'michael', 'hoot', 1.56)
|
||||
self.verify('/limiter/michael/hoot', 'michael', 'hoot', 1000)
|
||||
delay = self._request("GET", "/delayed")
|
||||
self.assertEqual(delay, None)
|
||||
|
||||
delay = self._request("GET", "/delayed")
|
||||
self.assertEqual(delay, '60.00')
|
||||
|
||||
def test_response_to_delays_usernames(self):
|
||||
delay = self._request("GET", "/delayed", "user1")
|
||||
self.assertEqual(delay, None)
|
||||
|
||||
delay = self._request("GET", "/delayed", "user2")
|
||||
self.assertEqual(delay, None)
|
||||
|
||||
delay = self._request("GET", "/delayed", "user1")
|
||||
self.assertEqual(delay, '60.00')
|
||||
|
||||
delay = self._request("GET", "/delayed", "user2")
|
||||
self.assertEqual(delay, '60.00')
|
||||
|
||||
|
||||
class FakeHttplibSocket(object):
|
||||
"""a fake socket implementation for httplib.HTTPResponse, trivial"""
|
||||
"""
|
||||
Fake `httplib.HTTPResponse` replacement.
|
||||
"""
|
||||
|
||||
def __init__(self, response_string):
|
||||
"""Initialize new `FakeHttplibSocket`."""
|
||||
self._buffer = StringIO.StringIO(response_string)
|
||||
|
||||
def makefile(self, _mode, _other):
|
||||
"""Returns the socket's internal buffer"""
|
||||
"""Returns the socket's internal buffer."""
|
||||
return self._buffer
|
||||
|
||||
|
||||
class FakeHttplibConnection(object):
|
||||
"""A fake httplib.HTTPConnection
|
||||
|
||||
Requests made via this connection actually get translated and routed into
|
||||
our WSGI app, we then wait for the response and turn it back into
|
||||
an httplib.HTTPResponse.
|
||||
"""
|
||||
def __init__(self, app, host, is_secure=False):
|
||||
Fake `httplib.HTTPConnection`.
|
||||
"""
|
||||
|
||||
def __init__(self, app, host):
|
||||
"""
|
||||
Initialize `FakeHttplibConnection`.
|
||||
"""
|
||||
self.app = app
|
||||
self.host = host
|
||||
|
||||
def request(self, method, path, data='', headers={}):
|
||||
def request(self, method, path, body="", headers={}):
|
||||
"""
|
||||
Requests made via this connection actually get translated and routed into
|
||||
our WSGI app, we then wait for the response and turn it back into
|
||||
an `httplib.HTTPResponse`.
|
||||
"""
|
||||
req = webob.Request.blank(path)
|
||||
req.method = method
|
||||
req.body = data
|
||||
req.headers = headers
|
||||
req.host = self.host
|
||||
# Call the WSGI app, get the HTTP response
|
||||
req.body = body
|
||||
|
||||
resp = str(req.get_response(self.app))
|
||||
# For some reason, the response doesn't have "HTTP/1.0 " prepended; I
|
||||
# guess that's a function the web server usually provides.
|
||||
resp = "HTTP/1.0 %s" % resp
|
||||
sock = FakeHttplibSocket(resp)
|
||||
self.http_response = httplib.HTTPResponse(sock)
|
||||
self.http_response.begin()
|
||||
|
||||
def getresponse(self):
|
||||
"""Return our generated response from the request."""
|
||||
return self.http_response
|
||||
|
||||
|
||||
@@ -208,36 +351,36 @@ def wire_HTTPConnection_to_WSGI(host, app):
|
||||
httplib.HTTPConnection = HTTPConnectionDecorator(httplib.HTTPConnection)
|
||||
|
||||
|
||||
class WSGIAppProxyTest(test.TestCase):
|
||||
class WsgiLimiterProxyTest(test.TestCase):
|
||||
"""
|
||||
Tests for the `limits.WsgiLimiterProxy` class.
|
||||
"""
|
||||
|
||||
def setUp(self):
|
||||
"""Our WSGIAppProxy is going to call across an HTTPConnection to a
|
||||
WSGIApp running a limiter. The proxy will send input, and the proxy
|
||||
should receive that same input, pass it to the limiter who gives a
|
||||
result, and send the expected result back.
|
||||
|
||||
The HTTPConnection isn't real -- it's monkeypatched to point straight
|
||||
at the WSGIApp. And the limiter isn't real -- it's a fake that
|
||||
behaves the way we tell it to.
|
||||
"""
|
||||
super(WSGIAppProxyTest, self).setUp()
|
||||
self.limiter = FakeLimiter(self)
|
||||
app = ratelimiting.WSGIApp(self.limiter)
|
||||
wire_HTTPConnection_to_WSGI('100.100.100.100:80', app)
|
||||
self.proxy = ratelimiting.WSGIAppProxy('100.100.100.100:80')
|
||||
Do some nifty HTTP/WSGI magic which allows for WSGI to be called
|
||||
directly by something like the `httplib` library.
|
||||
"""
|
||||
test.TestCase.setUp(self)
|
||||
self.time = 0.0
|
||||
self.app = limits.WsgiLimiter(TEST_LIMITS)
|
||||
self.app._limiter._get_time = self._get_time
|
||||
wire_HTTPConnection_to_WSGI("169.254.0.1:80", self.app)
|
||||
self.proxy = limits.WsgiLimiterProxy("169.254.0.1:80")
|
||||
|
||||
def _get_time(self):
|
||||
"""Return the "time" according to this test suite."""
|
||||
return self.time
|
||||
|
||||
def test_200(self):
|
||||
self.limiter.mock('conquer', 'caesar', None)
|
||||
when = self.proxy.perform('conquer', 'caesar')
|
||||
self.assertEqual(when, None)
|
||||
"""Successful request test."""
|
||||
delay = self.proxy.check_for_delay("GET", "/anything")
|
||||
self.assertEqual(delay, None)
|
||||
|
||||
def test_403(self):
|
||||
self.limiter.mock('grumble', 'proletariat', 1.5)
|
||||
when = self.proxy.perform('grumble', 'proletariat')
|
||||
self.assertEqual(when, 1.5)
|
||||
"""Forbidden request test."""
|
||||
delay = self.proxy.check_for_delay("GET", "/delayed")
|
||||
self.assertEqual(delay, None)
|
||||
|
||||
def test_failure(self):
|
||||
def shouldRaise():
|
||||
self.limiter.mock('murder', 'brutus', None)
|
||||
self.proxy.perform('stab', 'brutus')
|
||||
self.assertRaises(AssertionError, shouldRaise)
|
||||
delay = self.proxy.check_for_delay("GET", "/delayed")
|
||||
self.assertEqual(delay, '60.00')
|
||||
|
||||
Reference in New Issue
Block a user