http://blog.gmane.org/gmane.comp.lang.ruby.mongrel.devel hourly 1 1901-01-01T00:00+00:00 http://gmane.org/img/gmane-25t.png http://gmane.org http://comments.gmane.org/gmane.comp.lang.ruby.mongrel.devel/162 <pre>Hello, I've pushed the following two changes to my git repo, based on "master" of git://github.com/fauna/mongrel.git [PATCH 1/2] ragel.rake: rebuild on http11_parser_common.rl changes [PATCH 2/2] strip trailing and leading linear whitespace in headers The first patch is a trivial rake dependency fix. I could definitely use a more pairs of eyes to review my second patch which I've included inline below. I'm also planning this for the unicorn and kcar projects. RFC 2616, section 4.2: --- You can pull from my repo here: git pull git://bogomips.org/mongrel.git http11-lws If you like web browsers (I don't) you can view the changes here: http://bogomips.org/mongrel.git?h=http11-lws ext/http11/http11_parser.c | 435 +++++++++++++++++++----------------- ext/http11/http11_parser_common.rl | 5 +- test/test_http11.rb | 66 ++++++ 3 files changed, 304 insertions(+), 202 deletions(-) (removed ext/http11/http11_parser.c diff for brevity) diff --git a/ext/http11/http11_parser_common.rl b/ext/http11/http11_parser_common.rl index 53c805f..2ef1d31 100644 --- a/ext/http11/http11_parser_common.rl +++ b/ext/http11/http11_parser_common.rl &lt; at &gt;&lt; at &gt; -18,6 +18,7 &lt; at &gt;&lt; at &gt; uchar = (unreserved | escape); pchar = (uchar | ":" | "&lt; at &gt;" | "&amp;" | "=" | "+"); tspecials = ("(" | ")" | "&lt;" | "&gt;" | "&lt; at &gt;" | "," | ";" | ":" | "\\" | "\"" | "/" | "[" | "]" | "?" | "=" | "{" | "}" | " " | "\t"); + lws = (" " | "\t"); # elements token = (ascii -- (CTL | tspecials)); &lt; at &gt;&lt; at &gt; -43,9 +44,9 &lt; at &gt;&lt; at &gt; field_name = ( token -- ":" )+ &gt;start_field $snake_upcase_field %write_field; - field_value = any* &gt;start_value %write_value; + field_value = (""|(any*(any -- lws))) &gt;start_value %write_value; - message_header = field_name ":" " "* field_value :&gt; CRLF; + message_header = field_name ":" lws* field_value lws* :&gt; CRLF; Request = Request_Line ( message_header )* ( CRLF &lt; at &gt;done ); diff --git a/test/test_http11.rb b/test/test_http11.rb index da311af..2b55909 100644 --- a/test/test_http11.rb +++ b/test/test_http11.rb &lt; at &gt;&lt; at &gt; -81,6 +81,72 &lt; at &gt;&lt; at &gt; class HttpParserTest &lt; Test::Unit::TestCase assert_equal 'posts-17408', req['FRAGMENT'] end + def test_leading_tab + parser = HttpParser.new + req = {} + get = "GET / HTTP/1.1\r\nHost:\texample.com\r\n\r\n" + assert_nothing_raised do + parser.execute(req, get, 0) + end + assert parser.finished? + assert_equal 'example.com', req['HTTP_HOST'] + end + + def test_trailing_whitespace + parser = HttpParser.new + req = {} + get = "GET / HTTP/1.1\r\nHost: example.com \r\n\r\n" + assert_nothing_raised do + parser.execute(req, get, 0) + end + assert parser.finished? + assert_equal 'example.com', req['HTTP_HOST'] + end + + def test_trailing_tab + parser = HttpParser.new + req = {} + get = "GET / HTTP/1.1\r\nHost: example.com\t\r\n\r\n" + assert_nothing_raised do + parser.execute(req, get, 0) + end + assert parser.finished? + assert_equal 'example.com', req['HTTP_HOST'] + end + + def test_trailing_multiple_linear_whitespace + parser = HttpParser.new + req = {} + get = "GET / HTTP/1.1\r\nHost: example.com\t \t \t\r\n\r\n" + assert_nothing_raised do + parser.execute(req, get, 0) + end + assert parser.finished? + assert_equal 'example.com', req['HTTP_HOST'] + end + + def test_embedded_linear_whitespace_ok + parser = HttpParser.new + req = {} + get = "GET / HTTP/1.1\r\nX-Space: hello\t world\t \r\n\r\n" + assert_nothing_raised do + parser.execute(req, get, 0) + end + assert parser.finished? + assert_equal "hello\t world", req["HTTP_X_SPACE"] + end + + def test_empty_header + parser = HttpParser.new + req = {} + get = "GET / HTTP/1.1\r\nHost: \r\n\r\n" + assert_nothing_raised do + parser.execute(req, get, 0) + end + assert parser.finished? + assert_equal '', req['HTTP_HOST'] + end + # lame random garbage maker def rand_data(min, max, readable=true) count = min + ((rand(max)+1) *10).to_i </pre> Eric Wong 2011-05-18T21:13:49 http://comments.gmane.org/gmane.comp.lang.ruby.mongrel.devel/161 <pre>, as soon as i heard about this I thought of you http://j.mp/koA4P6 </pre> wyhaines-Re5JQEeQqe8AvxtiuMwx3w< at >public.gmane.org 2011-05-17T15:20:11 http://comments.gmane.org/gmane.comp.lang.ruby.mongrel.devel/160 <pre>mongrel_service version 0.4.beta2 has been released! * &lt;http://github.com/fauna/mongrel_service&gt; This plugin offer native win32 services for rails. This replace mongrel_rails_service. Since this is a pre-release gem, you can install it with: gem install mongrel_service --prerelease If no bug is found, expect an final release next week. Changes: ### 0.4.beta2 / 2010-01-01 * Enhancements: * Removed win32-service dependency, making mongrel_service more portable between Ruby implementations on Windows (One-Click and RubyInstaller) * Now mongrel log files are written to the path specified by --log option. Contribution by Daniel Gies (BigFix). Closes #44. * Bugfixes: * Wait longer for child process terminate properly (max 20 seconds). Imported tests from RubyServices project. Closes #18. * Workaround Windows 2008 process detection issues forcing 'service' initialization parameter. Closes #54. [papillon] * Updated ServiceFB to work with FB &gt; 0.18. </pre> Luis Lavena 2010-01-01T21:46:50 http://comments.gmane.org/gmane.comp.lang.ruby.mongrel.devel/157 <pre>Hi all, Don't have time to really write it up, but maybe we could just rot 13 the name and stick a 2.0 label on it :&gt; http://git.bogomips.org/cgit/zbatery.git If you grab the latest unicorn.git, you can get Rubinius support for the HTTP parser. http://git.bogomips.org/cgit/unicorn.git Also, in the latest rainbows.git it can work with Rubinius Actors http://git.bogomips.org/cgit/rainbows.git </pre> Eric Wong 2009-11-28T02:08:00 http://comments.gmane.org/gmane.comp.lang.ruby.mongrel.devel/154 <pre>Hi all, I've pushed out some changes based on fauna/master[1] to git://git.bogomips.org/ur-mongrel that includes a good chunk of the platform-independent stuff found in Unicorn. The new HTTP parser is named "mongrel_http" to avoid loadtime conflicts with the old one ("http11") but maintains the same class name (Mongrel::HttpParser). This one even supports HTTP/0.9, so "http11" wasn't an appropriate name for it :) Problems: I'm having some trouble with Rake+Echoe 3.2 with an "uninitialized constant Platform" error but everything seems to work by hand without Rake+Echoe. I'm also getting some test failures under 1.9.1-p243 with the semaphore/threading tests. I haven't looked too hard at this current threading model, but my gut feeling is that it's too complicated and a "dumber" model in mongrel 1.x *or* a fixed number of worker threads doing accept() is sufficient... One thing that may be cool is to support multiple threading/concurrency models since 1.8/1.9/jruby/rubinius all implement threads differently and we can also get Actors with 1.9/Rubinius. shortlog and diffstat below: Eric Wong (6): http_response: replace old API with simpler one http_response: drop old API compatibility remove HeaderOut class Add new HTTP/{0.9,1.0,1.1} parser Start using the new HTTP parser + TeeInput Remove unused Const::HTTP_STATUS_CODES hash Manifest | 10 +- ext/mongrel_http/c_util.h | 107 ++++ ext/mongrel_http/common_field_optimization.h | 111 ++++ ext/mongrel_http/ext_help.h | 48 ++ ext/mongrel_http/extconf.rb | 8 + ext/mongrel_http/global_variables.h | 91 ++++ ext/mongrel_http/mongrel_http.rl | 708 ++++++++++++++++++++++++++ ext/mongrel_http/mongrel_http_common.rl | 74 +++ lib/mongrel.rb | 64 +--- lib/mongrel/const.rb | 46 +-- lib/mongrel/header_out.rb | 34 -- lib/mongrel/http_request.rb | 147 ++---- lib/mongrel/http_response.rb | 202 ++------ lib/mongrel/tee_input.rb | 144 ++++++ test/unit/test_http_parser.rb | 425 ++++++++++++++-- test/unit/test_http_parser_ng.rb | 307 +++++++++++ test/unit/test_response.rb | 12 +- test/unit/test_server.rb | 3 + 18 files changed, 2101 insertions(+), 440 deletions(-) Full changelog: commit 4e6ab7b7d608bd074107c6a1804401d8165062d4 Author: Eric Wong &lt;normalperson-rMlxZR9MS24&lt; at &gt;public.gmane.org&gt; Date: Sat Sep 12 16:38:22 2009 -0700 Remove unused Const::HTTP_STATUS_CODES hash It's no longer used when we generate responses, instead we just use the one found in Rack (which was originally "stolen" from us) so it's one less thing for us to maintain. commit 46ca4a1c35b92109cedd59808908e7ad1d289abb Author: Eric Wong &lt;normalperson-rMlxZR9MS24&lt; at &gt;public.gmane.org&gt; Date: Sat Sep 12 10:40:30 2009 -0700 Start using the new HTTP parser + TeeInput The new HTTP parser minimizes the amount of Ruby support code needed and the HttpRequest class has been changed to a single class method: HttpRequest.read As a result, this hooks up the TeeInput class into the request processing cycle. TeeInput lets us read the request body off the socket while the Rack application is being called (instead of being buffered before-hand) while providing rewindable semantics that the Rack spec requires. commit c5a63522bc7e323c706609f7d99ed9f09fe9975d Author: Eric Wong &lt;normalperson-rMlxZR9MS24&lt; at &gt;public.gmane.org&gt; Date: Fri Sep 11 13:55:20 2009 -0700 Add new HTTP/{0.9,1.0,1.1} parser This is descended from the Mongrel parser but modified to support: * chunked transfer-encoding * trailers after chunked request bodies * HTTP/0.9 * absolute URI requests * multi-line headers with continuation lines * repeated headers (joined by commas) * #keepalive? boolean method * better integration with Rack This is not yet hooked into any existing parts of Mongrel, that is the next step. commit 8c1c7bdd3c1767708f8507d5aef8ded03b6f1796 Author: Eric Wong &lt;normalperson-rMlxZR9MS24&lt; at &gt;public.gmane.org&gt; Date: Fri Sep 11 13:16:25 2009 -0700 remove HeaderOut class HttpResponse has been rewritten to just iterate through the headers Rack gives us in a GC-friendly way so we have no need for this any longer. commit 392ea08624e39faec8d5e10ba04b21dfd9ca19a1 Author: Eric Wong &lt;normalperson-rMlxZR9MS24&lt; at &gt;public.gmane.org&gt; Date: Fri Sep 11 12:58:42 2009 -0700 http_response: drop old API compatibility Avoid needless overhead in allocating a HttpResponse object and instead just use a class method. This is alright with Rack applications since Rack specifies the response is already a tuple for writing. Of course the headers and body of the response can both be generated iteratively with #each. commit 469a507133bd20034df485f03b6eb7b0e82080d6 Author: Eric Wong &lt;normalperson-rMlxZR9MS24&lt; at &gt;public.gmane.org&gt; Date: Fri Sep 11 12:52:20 2009 -0700 http_response: replace old API with simpler one The old API is completely dropped, a compatibility layer for the old one will be added as Rack middleware instead. This allows newly-written applications to go through fewer layers of abstraction. git: git://git.bogomips.org/ur-mongrel cgit: http://git.bogomips.org/cgit/ur-mongrel.git [1] 9f9a9d488ed32a2891dc3dd7d50a17a16357042d </pre> Eric Wong 2009-09-12T23:57:29 http://comments.gmane.org/gmane.comp.lang.ruby.mongrel.devel/152 <pre>This change to Mongrel should be completely harmless for non-UNIX systems, too (it won't break (or do) anything). This minimal change let Mongrels inherit listen sockets from a parent process, so that parent could be a cluster manager. This means no more port juggling with mongrel_cluster! The entire Mongrel pack will all share one port. Only the manager script (see below) itself relies on UNIX-isms (but I think mongrel_cluster did, too). diff --git a/lib/mongrel.rb b/lib/mongrel.rb index 0619abe..90e99e7 100644 --- a/lib/mongrel.rb +++ b/lib/mongrel.rb &lt; at &gt;&lt; at &gt; -91,7 +91,11 &lt; at &gt;&lt; at &gt; module Mongrel def initialize(host, port, num_processors=950, throttle=0, timeout=60) tries = 0 - &lt; at &gt;socket = TCPServer.new(host, port) + &lt; at &gt;socket = if ENV['LISTEN_FD'] + TCPServer.for_fd(ENV['LISTEN_FD'].to_i) + else + TCPServer.new(host, port) + end if defined?(Fcntl::FD_CLOEXEC) &lt; at &gt;socket.fcntl(Fcntl::F_SETFD, Fcntl::FD_CLOEXEC) end --- Here's my initial cut of a mongrel_cluster replacement, it can definitely be cleaned up and improved upon with real option parsing and such... ---------------------------------- 8&lt; ---------------------------------- #!/usr/bin/env ruby require 'socket' nr_workers = 4 pids = {} cmd = %w(mongrel_rails start) server = TCPServer.new('0.0.0.0', 3000) server.listen 1024 ENV['LISTEN_FD'] = server.fileno.to_s # pass any signals we receive onto each child process: %w(TERM INT USR1 USR2).each do |sig| trap(sig) do pids.keys.each { |pid| Process.kill(sig, pid) rescue nil } exit 0 if sig == 'TERM' || sig == 'INT' # TERM and INT mean death end end nr_workers.times { |i| pids[fork { exec *cmd }] = i } loop do pid, status = Process.waitpid2(-1) i = pids.delete(pid) or next STDERR.puts "reaped worker #{i}: #{status}, respawning..." pids[fork { exec *cmd }] = i end ---------------------------------- 8&lt; ---------------------------------- </pre> Eric Wong 2009-08-14T01:18:06 http://comments.gmane.org/gmane.comp.lang.ruby.mongrel.devel/150 <pre>These are joined in in accordance with rfc2616, section 4.2[1]. This is also ticket #50 in Trac: http://mongrel.rubyforge.org/ticket/50 [1] - http://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.2 --- This applies against c365ba16d12a14bdf1cc50a26f67dd3b45f5a4d8 in Evan's fauna repository, I'm completely certain where I should be applying this but it should be trivial to port this patch to anything else remotely resembling it... ext/http11/http11.c | 9 ++++++++- test/unit/test_http_parser.rb | 10 +++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/ext/http11/http11.c b/ext/http11/http11.c index a228019..a770c60 100644 --- a/ext/http11/http11.c +++ b/ext/http11/http11.c &lt; at &gt;&lt; at &gt; -182,6 +182,7 &lt; at &gt;&lt; at &gt; void http_field(void *data, const char *field, size_t flen, const char *value, s VALUE req = (VALUE)data; VALUE v = Qnil; VALUE f = Qnil; + VALUE e = Qnil; VALIDATE_MAX_LENGTH(flen, FIELD_NAME); VALIDATE_MAX_LENGTH(vlen, FIELD_VALUE); &lt; at &gt;&lt; at &gt; -208,7 +209,13 &lt; at &gt;&lt; at &gt; void http_field(void *data, const char *field, size_t flen, const char *value, s /* fprintf(stderr, "UNKNOWN HEADER &lt;%s&gt;\n", RSTRING_PTR(f)); */ } - rb_hash_aset(req, f, v); + e = rb_hash_aref(req, f); + if (e == Qnil) { + rb_hash_aset(req, f, v); + } else { + rb_str_buf_cat(e, ",", 1); + rb_str_buf_append(e, v); + } } void request_method(void *data, const char *at, size_t length) diff --git a/test/unit/test_http_parser.rb b/test/unit/test_http_parser.rb index c89cd0d..70d9c91 100644 --- a/test/unit/test_http_parser.rb +++ b/test/unit/test_http_parser.rb &lt; at &gt;&lt; at &gt; -173,6 +173,14 &lt; at &gt;&lt; at &gt; class HttpParserTest &lt; Test::Unit::TestCase assert_equal 4,res["zed"].length, "wrong number for zed" assert_equal "11",res["frank"], "wrong number for frank" end - + + def test_combine_repeat_headers + parser = HttpParser.new + http = "GET / HTTP/1.1\r\nA: 1\r\nA: 2\r\n\r\n" + req = {} + assert_nothing_raised { parser.execute(req, http, 0) } + assert_equal '1,2', req['HTTP_A'] + end + end </pre> Eric Wong 2009-08-10T00:10:22 Search the mailing list at Gmane query http://search.gmane.org/?group=$group=gmane.comp.lang.ruby.mongrel.devel