openresty coroutine weirdness

co_wrap.lua

-- Reimplemented coroutine.wrap, returning "nil, err" if the coroutine cannot
-- be resumed.
local co_wrap = function(func)
    local co = coroutine.create(func)
    ngx.log(ngx.DEBUG, "co created with status ", coroutine.status(co))
    return function(...)
        if coroutine.status(co) == "suspended" then
            return select(2, coroutine.resume(co, ...))
        else
            return nil, "can't resume a " .. coroutine.status(co) .. " coroutine"
        end
    end
end

example output

THIS IS REQ #1 with status 200
THIS IS REQ #2 with status 200
THIS IS REQ #3 with status 200
THIS IS REQ #4 with status 200
THIS IS REQ #5 with status 200
THIS IS REQ #6 with status 200
can't resume a dead coroutine
THIS IS REQ #7 with status 200
can't resume a dead coroutine
THIS IS REQ #8 with status 200
THIS IS REQ #9 with status 200
THIS IS REQ #10 with status 200

multiple requests.lua

local http = require "resty.http"

local httpcs = {
  http.new(),
  http.new(),
  http.new(),
  http.new(),
  http.new(),
  http.new(),
  http.new(),
  http.new(),
  http.new(),
  http.new(),
}

for i,httpc in ipairs(httpcs) do
  httpc:connect("127.0.0.1", 80)
end

for i,httpc in ipairs(httpcs) do
  local res, err = httpc:request{
      path = "/testresponse",
  }

  if not res then
      ngx.log(ngx.ERR, err)
      ngx.exit(500)
  end

  ngx.say("THIS IS REQ #", i, " with status ", res.status)

  local reader = res.body_reader

  repeat
      local chunk, err = reader(2^15)
      
      if not chunk and err then
          ngx.log(ngx.ERR, err)
          ngx.say(err)
          break
      end
  until not chunk

  local ok, err = httpc:close()
  if not ok then
       ngx.log(ngx.ERR, err)
  end
end

valgrind output

[jhurst@sl6]$ sudo valgrind --track-origins=yes --leak-check=full /usr/local/openresty/nginx/sbin/nginx -p . -s reload
==28977== Memcheck, a memory error detector
==28977== Copyright (C) 2002-2010, and GNU GPL'd, by Julian Seward et al.
==28977== Using Valgrind-3.6.0 and LibVEX; rerun with -h for copyright info
==28977== Command: /usr/local/openresty/nginx/sbin/nginx -p . -s reload
==28977==
nginx: [alert] lua_code_cache is off; this will hurt performance in ./conf/nginx.conf:19
==28977== Conditional jump or move depends on uninitialised value(s)
==28977==    at 0x4C3291A: lj_str_new (lj_str.c:54)
==28977==    by 0x4C3A30A: lua_pushlstring (lj_api.c:587)
==28977==    by 0x4C7267A: emptybuffer (lib_aux.c:177)
==28977==    by 0x4C72878: luaL_pushresult (lib_aux.c:221)
==28977==    by 0x4C7295C: luaL_gsub (lib_aux.c:163)
==28977==    by 0x49B5E3: ngx_http_lua_set_path (ngx_http_lua_util.c:140)
==28977==    by 0x49B7E6: ngx_http_lua_new_state (ngx_http_lua_util.c:245)
==28977==    by 0x495258: ngx_http_lua_init (ngx_http_lua_module.c:771)
==28977==    by 0x4399D2: ngx_http_block (ngx_http.c:314)
==28977==    by 0x4254F7: ngx_conf_parse (ngx_conf_file.c:387)
==28977==    by 0x422E17: ngx_init_cycle (ngx_cycle.c:268)
==28977==    by 0x417198: main (nginx.c:333)
==28977==  Uninitialised value was created by a stack allocation
==28977==    at 0x4C728F5: luaL_gsub (lib_aux.c:154)
==28977==
==28977==
==28977== HEAP SUMMARY:
==28977==     in use at exit: 740,440 bytes in 11,124 blocks
==28977==   total heap usage: 12,087 allocs, 963 frees, 3,955,695 bytes allocated
==28977==
==28977== 100 (48 direct, 52 indirect) bytes in 1 blocks are definitely lost in loss record 1,359 of 1,822
==28977==    at 0x4A05FDE: malloc (vg_replace_malloc.c:236)
==28977==    by 0x42DA71: ngx_alloc (ngx_alloc.c:22)
==28977==    by 0x41809E: ngx_create_pool (ngx_palloc.c:19)
==28977==    by 0x416D3E: main (nginx.c:299)
==28977==
==28977== LEAK SUMMARY:
==28977==    definitely lost: 48 bytes in 1 blocks
==28977==    indirectly lost: 52 bytes in 4 blocks
==28977==      possibly lost: 0 bytes in 0 blocks
==28977==    still reachable: 740,340 bytes in 11,119 blocks
==28977==         suppressed: 0 bytes in 0 blocks
==28977== Reachable blocks (those to which a pointer was found) are not shown.
==28977== To see them, rerun with: --leak-check=full --show-reachable=yes
==28977==
==28977== For counts of detected and suppressed errors, rerun with: -v
==28977== ERROR SUMMARY: 3 errors from 2 contexts (suppressed: 6 from 6)
[jhurst@sl6]$ ==28972==
==28972== HEAP SUMMARY:
==28972==     in use at exit: 520,893 bytes in 3,390 blocks
==28972==   total heap usage: 43,848 allocs, 40,458 frees, 19,643,538 bytes allocated
==28972==
==28971==
==28971== HEAP SUMMARY:
==28971==     in use at exit: 520,893 bytes in 3,390 blocks
==28971==   total heap usage: 43,848 allocs, 40,458 frees, 19,643,538 bytes allocated
==28971==
==28972== 409,600 (196,608 direct, 212,992 indirect) bytes in 1 blocks are definitely lost in loss record 637 of 637
==28972==    at 0x4A05FDE: malloc (vg_replace_malloc.c:236)
==28972==    by 0x42DA71: ngx_alloc (ngx_alloc.c:22)
==28972==    by 0x42AE30: ngx_event_process_init (ngx_event.c:699)
==28972==    by 0x430D5F: ngx_worker_process_init (ngx_process_cycle.c:970)
==28972==    by 0x4311CB: ngx_worker_process_cycle (ngx_process_cycle.c:737)
==28972==    by 0x42FBB3: ngx_spawn_process (ngx_process.c:198)
==28972==    by 0x4307F4: ngx_start_worker_processes (ngx_process_cycle.c:364)
==28972==    by 0x431DCD: ngx_master_process_cycle (ngx_process_cycle.c:249)
==28972==    by 0x417395: main (nginx.c:412)
==28972==
==28972== LEAK SUMMARY:
==28972==    definitely lost: 196,608 bytes in 1 blocks
==28972==    indirectly lost: 212,992 bytes in 2 blocks
==28972==      possibly lost: 0 bytes in 0 blocks
==28972==    still reachable: 111,293 bytes in 3,387 blocks
==28972==         suppressed: 0 bytes in 0 blocks
==28972== Reachable blocks (those to which a pointer was found) are not shown.
==28972== To see them, rerun with: --leak-check=full --show-reachable=yes
==28972==
==28972== For counts of detected and suppressed errors, rerun with: -v
==28972== Use --track-origins=yes to see where uninitialised values come from
==28972== ERROR SUMMARY: 35 errors from 2 contexts (suppressed: 9 from 9)
==28971== 409,600 (196,608 direct, 212,992 indirect) bytes in 1 blocks are definitely lost in loss record 637 of 637
==28971==    at 0x4A05FDE: malloc (vg_replace_malloc.c:236)
==28971==    by 0x42DA71: ngx_alloc (ngx_alloc.c:22)
==28971==    by 0x42AE30: ngx_event_process_init (ngx_event.c:699)
==28971==    by 0x430D5F: ngx_worker_process_init (ngx_process_cycle.c:970)
==28971==    by 0x4311CB: ngx_worker_process_cycle (ngx_process_cycle.c:737)
==28971==    by 0x42FBB3: ngx_spawn_process (ngx_process.c:198)
==28971==    by 0x4307F4: ngx_start_worker_processes (ngx_process_cycle.c:364)
==28971==    by 0x431DCD: ngx_master_process_cycle (ngx_process_cycle.c:249)
==28971==    by 0x417395: main (nginx.c:412)
==28971==
==28971== LEAK SUMMARY:
==28971==    definitely lost: 196,608 bytes in 1 blocks
==28971==    indirectly lost: 212,992 bytes in 2 blocks
==28971==      possibly lost: 0 bytes in 0 blocks
==28971==    still reachable: 111,293 bytes in 3,387 blocks
==28971==         suppressed: 0 bytes in 0 blocks
==28971== Reachable blocks (those to which a pointer was found) are not shown.
==28971== To see them, rerun with: --leak-check=full --show-reachable=yes
==28971==
==28971== For counts of detected and suppressed errors, rerun with: -v
==28971== Use --track-origins=yes to see where uninitialised values come from
==28971== ERROR SUMMARY: 35 errors from 2 contexts (suppressed: 9 from 9)
==28978== Invalid read of size 4
==28978==    at 0x4C32916: lj_str_new (lj_str.c:54)
==28978==    by 0x4C3A396: lua_setfield (lj_api.c:913)
==28978==    by 0x49BE63: ngx_http_lua_cache_store_code (ngx_http_lua_cache.c:106)
==28978==    by 0x49C39E: ngx_http_lua_cache_loadbuffer (ngx_http_lua_cache.c:169)
==28978==    by 0x49C79D: ngx_http_lua_content_handler_inline (ngx_http_lua_contentby.c:281)
==28978==    by 0x49C9B7: ngx_http_lua_content_handler (ngx_http_lua_contentby.c:202)
==28978==    by 0x43FD47: ngx_http_core_content_phase (ngx_http_core_module.c:1419)
==28978==    by 0x43A962: ngx_http_core_run_phases (ngx_http_core_module.c:899)
==28978==    by 0x43AA72: ngx_http_handler (ngx_http_core_module.c:882)
==28978==    by 0x44475A: ngx_http_process_request (ngx_http_request.c:1856)
==28978==    by 0x444D26: ngx_http_process_request_headers (ngx_http_request.c:1287)
==28978==    by 0x4451AD: ngx_http_process_request_line (ngx_http_request.c:966)
==28978==  Address 0x68cad54 is 36 bytes inside a block of size 38 alloc'd
==28978==    at 0x4A05FDE: malloc (vg_replace_malloc.c:236)
==28978==    by 0x42DA71: ngx_alloc (ngx_alloc.c:22)
==28978==    by 0x417F6F: ngx_malloc (ngx_palloc.c:122)
==28978==    by 0x417FCF: ngx_palloc (ngx_palloc.c:143)
==28978==    by 0x496E80: ngx_http_lua_content_by_lua (ngx_http_lua_directive.c:573)
==28978==    by 0x4254F7: ngx_conf_parse (ngx_conf_file.c:387)
==28978==    by 0x43CFC4: ngx_http_core_location (ngx_http_core_module.c:3249)
==28978==    by 0x4254F7: ngx_conf_parse (ngx_conf_file.c:387)
==28978==    by 0x43D99A: ngx_http_core_server (ngx_http_core_module.c:3024)
==28978==    by 0x4254F7: ngx_conf_parse (ngx_conf_file.c:387)
==28978==    by 0x439501: ngx_http_block (ngx_http.c:240)
==28978==    by 0x4254F7: ngx_conf_parse (ngx_conf_file.c:387)
==28978==
==28978== Conditional jump or move depends on uninitialised value(s)
==28978==    at 0x4C3291A: lj_str_new (lj_str.c:54)
==28978==    by 0x4C32F86: lj_str_fromnum (lj_str.c:205)
==28978==    by 0x4C34FDE: lj_meta_cat (lj_meta.c:234)
==28978==    by 0x4C3A893: lua_concat (lj_api.c:710)
==28978==    by 0x4A3230: ngx_http_lua_socket_tcp_connect (ngx_http_lua_socket_tcp.c:383)
==28978==    by 0x4C2DE0A: lj_BC_FUNCC (in /usr/local/openresty/luajit/lib/libluajit-5.1.so.2.0.2)
==28978==    by 0x49A433: ngx_http_lua_run_thread (ngx_http_lua_util.c:1077)
==28978==    by 0x4A0FBE: ngx_http_lua_socket_tcp_resume (ngx_http_lua_socket_tcp.c:4206)
==28978==    by 0x49C43A: ngx_http_lua_content_wev_handler (ngx_http_lua_contentby.c:128)
==28978==    by 0x4A06DE: ngx_http_lua_socket_handle_success (ngx_http_lua_socket_tcp.c:2254)
==28978==    by 0x4A267D: ngx_http_lua_socket_connected_handler (ngx_http_lua_socket_tcp.c:2355)
==28978==    by 0x4A0F28: ngx_http_lua_socket_tcp_handler (ngx_http_lua_socket_tcp.c:2037)
==28978==
==28978== Invalid read of size 4
==28978==    at 0x4C32916: lj_str_new (lj_str.c:54)
==28978==    by 0x4C3A776: lua_getfield (lj_api.c:749)
==28978==    by 0x49BDAD: ngx_http_lua_cache_load_code (ngx_http_lua_cache.c:51)
==28978==    by 0x49C33C: ngx_http_lua_cache_loadbuffer (ngx_http_lua_cache.c:137)
==28978==    by 0x49C79D: ngx_http_lua_content_handler_inline (ngx_http_lua_contentby.c:281)
==28978==    by 0x49C9B7: ngx_http_lua_content_handler (ngx_http_lua_contentby.c:202)
==28978==    by 0x43FD47: ngx_http_core_content_phase (ngx_http_core_module.c:1419)
==28978==    by 0x43A962: ngx_http_core_run_phases (ngx_http_core_module.c:899)
==28978==    by 0x43AA72: ngx_http_handler (ngx_http_core_module.c:882)
==28978==    by 0x44475A: ngx_http_process_request (ngx_http_request.c:1856)
==28978==    by 0x444D26: ngx_http_process_request_headers (ngx_http_request.c:1287)
==28978==    by 0x4451AD: ngx_http_process_request_line (ngx_http_request.c:966)
==28978==  Address 0x68dbe14 is 36 bytes inside a block of size 38 alloc'd
==28978==    at 0x4A05FDE: malloc (vg_replace_malloc.c:236)
==28978==    by 0x42DA71: ngx_alloc (ngx_alloc.c:22)
==28978==    by 0x417F6F: ngx_malloc (ngx_palloc.c:122)
==28978==    by 0x417FCF: ngx_palloc (ngx_palloc.c:143)
==28978==    by 0x496E80: ngx_http_lua_content_by_lua (ngx_http_lua_directive.c:573)
==28978==    by 0x4254F7: ngx_conf_parse (ngx_conf_file.c:387)
==28978==    by 0x43CFC4: ngx_http_core_location (ngx_http_core_module.c:3249)
==28978==    by 0x4254F7: ngx_conf_parse (ngx_conf_file.c:387)
==28978==    by 0x43D99A: ngx_http_core_server (ngx_http_core_module.c:3024)
==28978==    by 0x4254F7: ngx_conf_parse (ngx_conf_file.c:387)
==28978==    by 0x439501: ngx_http_block (ngx_http.c:240)
==28978==    by 0x4254F7: ngx_conf_parse (ngx_conf_file.c:387)
==28978==