patx/mrhttp-asgi
Chore: Cleanup base dir
Commit 2a96227 · Mark · 2024-09-20T20:45:06-07:00
Comments
No comments yet.
Diff
diff --git a/curl.sh b/curl.sh
deleted file mode 100755
index a9c7f15..0000000
--- a/curl.sh
+++ /dev/null
@@ -1,3 +0,0 @@
-for n in {1..10}; do
- curl -d "param1=value1¶m2=value2" -X POST http://localhost:8080/form -H "Content-Type: application/x-www-form-urlencoded"
-done
diff --git a/dotests.py.old b/dotests.py.old
deleted file mode 100644
index 81a5e3b..0000000
--- a/dotests.py.old
+++ /dev/null
@@ -1,189 +0,0 @@
-
-readme = """
- pip install psutil requests msgpack mrasyncmc tenjin mrpacker
- mrcache and mrworkserver must be running for the benchmarks
-"""
-
-
-import pkgutil, time
-import inspect
-import types
-import importlib
-
-import tests
-
-# TODO
-# Check for mrworkserver and mrcache being up and add the session key so we hit and load the json 43709dd361cc443e976b05714581a7fb
-# mrcache -m 64 -i 16
-# python mrworkserver/tst.py
-
-
-
-if 1:
- package = tests
- for importer, modname, ispkg in pkgutil.iter_modules(package.__path__):
- if modname.startswith("test"):
- m = importlib.import_module('tests.'+modname)
- functions = inspect.getmembers(m, inspect.isfunction)
- for f in functions:
- if f[0] == 'setup':
- if f[1]():
- exit()
- for f in functions:
- if f[0].startswith("test_"):
- try:
- f[1]()
- except Exception as e:
- print(e)
- for f in functions:
- if f[0] == 'teardown':
- f[1]()
-
-print("Benchmarks")
-
-import argparse
-import sys
-import asyncio
-import os
-from asyncio.subprocess import PIPE, STDOUT
-import statistics
-
-import uvloop
-import psutil
-import atexit
-
-#from misc import cpu
-
-
-def run_wrk(loop, endpoint=None, lua=None, options=None):
- rps = 0
- try:
- endpoint = endpoint or 'http://localhost:8080/'
- if lua:
- if options != None:
- wrk_fut = asyncio.create_subprocess_exec( 'wrk', '-t', '4', '-c', '32', '-d', '2', '-s', lua, *options, endpoint, stdout=PIPE, stderr=STDOUT)
- else:
- wrk_fut = asyncio.create_subprocess_exec( 'wrk', '-t', '4', '-c', '32', '-d', '2', '-s', lua, endpoint, stdout=PIPE, stderr=STDOUT)
- else:
- if options != None:
- wrk_fut = asyncio.create_subprocess_exec( 'wrk', '-t', '4', '-c', '32', '-d', '2', *options, endpoint, stdout=PIPE, stderr=STDOUT)
- else:
- wrk_fut = asyncio.create_subprocess_exec( 'wrk', '-t', '4', '-c', '32', '-d', '2', endpoint, stdout=PIPE, stderr=STDOUT)
-
- wrk = loop.run_until_complete(wrk_fut)
- rps = 0
- lines = []
- while 1:
- line = loop.run_until_complete(wrk.stdout.readline())
- if line:
- line = line.decode('utf-8')
- lines.append(line)
- if line.startswith('Requests/sec:'):
- rps = float(line.split()[-1])
- else:
- break
-
- retcode = loop.run_until_complete(wrk.wait())
- if retcode != 0:
- print('\r\n'.join(lines))
- except Exception as e:
- print(e)
-
-
- return rps
-
-
-noisy = ['atom', 'chrome', 'firefox', 'dropbox', 'opera', 'spotify', 'gnome-documents']
-
-def silence():
- for proc in psutil.process_iter():
- if proc.name() in noisy:
- proc.suspend()
-
- def resume():
- for proc in psutil.process_iter():
- if proc.name() in noisy:
- proc.resume()
- atexit.register(resume)
-
-silence()
-
-loop = uvloop.new_event_loop()
-
-asyncio.set_event_loop(loop)
-
-server_fut = asyncio.create_subprocess_exec( 'python', 'tests/s_bench.py', stdout=asyncio.subprocess.PIPE )
-proc = loop.run_until_complete(server_fut)
-process = psutil.Process(proc.pid)
-
-# Can't get this to work when the server fails to start
-#retcode = loop.run_until_complete(proc.wait())
-#print(retcode)
-#exit()
-#print(dir(proc))
-#time.sleep(5)
-#print( proc.pid in psutil.pids())
-#print(proc.returncode)
-#if proc.returncode != None:
- #print( "YAY")
- #exit()
-try:
-
- more_headers = ('-H','User-Agent: Mozilla/5.0 (X11; Linux x86_64) Gecko/20130501 Firefox/30.0 AppleWebKit/600.00 Chrome/30.0.0000.0 Trident/10.0 Safari/600.00',
- '-H','Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
- '-H','Accept-Language: en-US,en;q=0.5',
- '-H','Connection: keep-alive')
- opts = ('-H','Cookie: mrsession=43709dd361cc443e976b05714581a7fb; foo=fdsfdasdfasdfdsfasdfsdfsdfasdfas; short=fazc;')
- if 1:
- print ("Hello pipelined", run_wrk(loop, 'http://localhost:8080/',lua='tests/lua/pipeline.lua'), "Requests/second" )
- print ("More hdrs pipelined", run_wrk(loop, 'http://localhost:8080/',options=more_headers,lua='tests/lua/pipeline.lua'), "Requests/second" )
- #print ("Hello ", run_wrk(loop, 'http://localhost:8080/'), "Requests/second" )
- #print ("Hello hdrs ", run_wrk(loop, 'http://localhost:8080/', options=more_headers), "Requests/second" )
-
- #print ("Cookies ", run_wrk(loop, 'http://localhost:8080/printCookies', options=opts), "Requests/second" )
- #print ("many args ", run_wrk(loop, 'http://localhost:8080/sixargs/one/two/three/four/five/six'), "Requests/second" )
- #print ("404 natural ", run_wrk(loop, 'http://localhost:8080/dfads404/'), "Requests/second" )
- #print ("404 ", run_wrk(loop, 'http://localhost:8080/404/'), "Requests/second" )
- #print ("Form parsing ", run_wrk(loop, 'http://localhost:8080/form',lua='tests/lua/form.lua'), "Requests/second" )
- #print ("Templates ", run_wrk(loop, 'http://localhost:8080/template'), "Requests/second" )
- #print ("mrpacker ", run_wrk(loop,'http://localhost:8080/mrpacker',lua='tests/lua/mrpacker.lua'), "Requests/second" )
- #print ("Sessions ", run_wrk(loop, 'http://localhost:8080/s', options=opts), "Requests/second" )
- # Disabled in s_bench.py print ("Sessions (py) ", run_wrk(loop, 'http://localhost:8080/pys', options=opts), "Requests/second" )
- #print ("Session login ", run_wrk(loop, 'http://localhost:8080/login'), "Requests/second" )
- #print ("json post ", run_wrk(loop,'http://localhost:8080/json',lua='tests/lua/json.lua'), "Requests/second" )
- #print ("mrpacker py ", run_wrk(loop,'http://localhost:8080/mrpackerpy',lua='tests/lua/mrpacker.lua'), "Requests/second" )
- #print ("msgpack py ", run_wrk(loop,'http://localhost:8080/msgpack',lua='tests/lua/msgpack.lua'), "Requests/second" )
-
-
- opts = ('-H','XX-Real-IP: 1.2.3.4')
- #print ("get ip ", run_wrk(loop,'http://localhost:8080/getip',options=opts), "Requests/second" )
- print ("many num args ", run_wrk(loop, 'http://localhost:8080/sixargs/155/2001/29999/25/29999543/93243242394'), "Requests/second" )
- #print ("404 ", run_wrk(loop, 'http://localhost:8080/404/'), "Requests/second" )
-
- # Grab the stdout for debug
- if 0:
- lines = []
- x = 0
- while 1:
- x += 1
- print(x)
- #if x > 19842: break
- if x > 21605: break
- line = loop.run_until_complete(proc.stdout.readline())
- if line:
- line = line.decode('utf-8')
- lines.append(line)
- else:
- break
- print ( len(lines) )
- print(lines)
- #o = open( "wrkout", "wb" )
- #o.write( (''.join(lines)).encode("utf-8") )
- #o.close()
-
-except KeyboardInterrupt:
- pass
-finally:
- proc.terminate()
- loop.run_until_complete(proc.wait())
-
diff --git a/bad.py b/tests/bad.py
similarity index 100%
rename from bad.py
rename to tests/bad.py
diff --git a/todo b/todo
index c9f6ef5..802b941 100644
--- a/todo
+++ b/todo
@@ -1,149 +1,9 @@
-Handle static files - static root route and path.
-Connection buffer size - after a file upload shrink the buffer back down. Behind cloudflare connections will remain open forever. Behind nginx its just one connection so who cares. Out front client connections will close so it doesn't matter?
-
-
-support HEAD
-
-disconnect
- Protocol dealloc is never called - python bug?
-
-
-
-
-
-
-
-
-
-
-
-
-set user sess - fix it!
-
-Change to the new session ids everywhere. TODO user id in session necessary? No. Idea was to use MrWorkServers to cache user data .
-Update dotests to setup the memcached/mrq/mrcache servers and set a session id so we find the user
-
-
-
-
-
-
-
-
- def _get_address(self):
- sock = self.transport.get_extra_info('socket')
-
- if sock.family == socket.AF_INET:
- self._socket = (self.transport.get_extra_info('peername') or
- (None, None))
- self._ip, self._port = self._socket
- elif sock.family == socket.AF_INET6:
- self._socket = (self.transport.get_extra_info('peername') or
- (None, None, None, None))
- self._ip, self._port, *_ = self._socket
- else:
- self._ip, self._port = (None, None)
-
- @property
- def remote_addr(self):
- """Attempt to return the original client ip based on X-Forwarded-For.
-
- :return: original client ip.
- """
- if not hasattr(self, '_remote_addr'):
- forwarded_for = self.headers.get('X-Forwarded-For', '').split(',')
- remote_addrs = [
- addr for addr in [
- addr.strip() for addr in forwarded_for
- ] if addr
- ]
- if len(remote_addrs) > 0:
- self._remote_addr = remote_addrs[0]
- else:
- self._remote_addr = ''
- return self._remote_addr
-
-
-
-subdomains
-redirects?
-bad.py - bad request
-Review tests
-mrpacker
-
-memcached = clean it up
-If MemcachedClient_set servers are down we return NULL and don't setup an exception
-
-dotests disconnect
-Request_reset - we doing in all necessary places
-
-Make conn/req timeouts as arguments (20 / 5 right now)
Add tests for memc and mrq failover. ?
-
-
-Add streaming upload
+Add streaming upload?
https://github.com/huge-success/sanic/tree/master/examples/request_stream
-
-
-
-test for setUserSession with added cookies
-/r/python - Why would you write your web server in anything but python
-
-If we lose the mrq servers we have to tell client
-
-
-!!
- File "uvloop/handles/basetransport.pyx", line 92, in uvloop.loop.UVBaseTransport._maybe_pause_protocol
-AttributeError: 'internals.MrqProtocol' object has no attribute 'pause_writing'
-protocol.resume_writing() failed
-protocol: <internals.MrqProtocol object at 0x7f1530f86cf0>
-transport: <TCPTransport closed=False reading=True 0x2b62fc8>
-Traceback (most recent call last):
- File "uvloop/handles/basetransport.pyx", line 108, in uvloop.loop.UVBaseTransport._maybe_resume_protocol
-AttributeError: 'internals.MrqProtocol' object has no attribute 'resume_writing'
-Explained?
-For the asyncio streams-based implementation, StreamReader automatically uses the {pause,resume}_reading methods to transmit backpressure upstream, and StreamWriter provides a friendly wrapper around {pause,resume}_writing to help us accept backpressure from downstream: the drain method – we just have to remember to use it. So in order to fix our proxy to transmit backpressure, all we need to do is to add one line of code to copy_all. Specifically, this line:
- await drain()
-await dest_writer.drain()
-
-Flow Control
-In the initial echo server example we had await writer.drain() as this paused the coroutine from writing more data to the socket till the client had caught up, it drained the socket. This is useful as until the client catches up the data will be stored in memory, hence a malicious client can make many requests for a lot of data, refuse to receive the data, and allow the server to exhaust its memory.
-
-To combat this the coroutine sending data should await a drain function, that can be added to the protocol,
-
-import asyncio
-class FlowControlServer(asyncio.Protocol):
- def __init__(self):
- self._can_write = asyncio.Event()
- self._can_write.set()
- def pause_writing(self) -> None:
- # Will be called whenever the transport crosses the
- # high-water mark.
- self._can_write.clear()
- def resume_writing(self) -> None:
- # Will be called whenever the transport drops back below the
- # low-water mark.
- self._can_write.set()
- async def drain(self) -> None:
- await self._can_write.wait()
-
-WriteTransport.set_write_buffer_limits(high=None, low=None)
-Set the high and low watermarks for write flow control.
-
-These two values (measured in number of bytes) control when the protocol’s protocol.pause_writing() and protocol.resume_writing() methods are called. If specified, the low watermark must be less than or equal to the high watermark. Neither high nor low can be negative.
-
-pause_writing() is called when the buffer size becomes greater than or equal to the high value. If writing has been paused, resume_writing() is called when the buffer size becomes less than or equal to the low value.
-
-The defaults are implementation-specific. If only the high watermark is given, the low watermark defaults to an implementation-specific value less than or equal to the high watermark. Setting high to zero forces low to zero as well, and causes pause_writing() to be called whenever the buffer becomes non-empty. Setting low to zero causes resume_writing() to be called only once the buffer is empty. Use of zero for either limit is generally sub-optimal as it reduces opportunities for doing I/O and computation concurrently.
-
-
-
-
-
-
-
+If we lose the mrq servers we have to tell client ?0
Wiki doc?
http://www.django-rest-framework.org/api-guide/requests/
@@ -163,14 +23,6 @@ vhosts -- how to do this? I don't like sanic's, maybe do cherrypy's redir
'api.foo.com': '/api',
-
-
-
-
-
-
-
-
logging
Stream large file upload
@@ -183,17 +35,3 @@ https://github.com/channelcat/sanic/issues/168
Write tutorials? github wiki pages
Flask:
https://blog.miguelgrinberg.com/post/the-flask-mega-tutorial-part-i-hello-world
-
-
-
-
-
-Look at a mysql C client? We can do better than aiomysql. Either libmysqlclient or our own
-https://dev.mysql.com/doc/dev/mysql-server/latest/PAGE_PROTOCOL.html
-https://dev.mysql.com/doc/internals/en/a-mysql-client-logs-in.html
-
-
-If I want to try mrpacker again:
-https://github.com/MarkReedZ/mrhttp/tree/cdaf7d8be3bcb0a88792875678aa2de515020566/src/mrhttp/utils
-
-
diff --git a/tst2.py b/tst2.py
deleted file mode 100644
index e6f7fee..0000000
--- a/tst2.py
+++ /dev/null
@@ -1,29 +0,0 @@
-
-import asyncio
-
-def setup():
- from mrhttp import app
-
- @app.route('/')
- async def index(r):
- return "indeX"
-
- return asyncio.ensure_future( app.start_server(), loop=app.loop )
- #return app.start_server()
-
-try:
-
- def wakeup():
- loop.call_later(0.1, wakeup)
- task = setup()
- print(task)
- #app.run(cores=4)
- loop = asyncio.get_event_loop()
- loop.call_later(0.1, wakeup)
- loop.run_forever()
-
-except KeyboardInterrupt:
- print ("keybd")
-except Exception as e:
- print("YAY",e)
-
diff --git a/z.py b/z.py
deleted file mode 100644
index d1587b3..0000000
--- a/z.py
+++ /dev/null
@@ -1,13 +0,0 @@
-import requests, mrpacker
-
-if 0:
- data = {}
- s = "lo(ng"*50000
- data["long"] = s
- r = requests.post('http://localhost:8080/form',data) # TODO have this timeout quickly
-
-headers = {'Content-type': 'application/mrpacker'}
-o = { "typ":"post", "s":2, "t": 'Blonde: "What does IDK stand for?"', "l":"localhost/sub/3", "txt": 'Brunette: "I don’t know."\nBlonde: "OMG, nobody does!"' }
-r = requests.post('http://localhost:8080/mrp', data=mrpacker.pack(o), headers=headers)
-print( r.raw.headers )
-print( "text is ", r.text )