Restlet hangs on multiple threads

classic Classic list List threaded Threaded
3 messages Options
Reply | Threaded
Open this post in threaded view
|  
Report Content as Inappropriate

Restlet hangs on multiple threads

Bluesoft70
I have an interesting problem that occurs within 5-7 days interval. 
The setup is a restlet server (http just forwarding to https) with jetty and apache connector and a mysql db. The apache server is behind firewall on separate ports (http and https). Restlet redirects web calls directly to the apache server 
When the error occurs the web page access stalls/hangs (but normal access to restlet resources is ok). The thread dump is full of instances like the below (hundreds). It seems to me that threads are waiting on a connection pool, and is not terminated. 
I've used jmeter to stresstest the system, but are not able to recreate the problem. 
Any suggestions on this one? 


State: WAITING on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@7c8a36b1 
Total blocked: 0  Total waited: 3 

Stack trace: 
sun.misc.Unsafe.park(Native Method) 
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) 
org.apache.http.impl.conn.tsccm.WaitingThread.await(WaitingThread.java:159) 
org.apache.http.impl.conn.tsccm.ConnPoolByRoute.getEntryBlocking(ConnPoolByRoute.java:398) 
org.apache.http.impl.conn.tsccm.ConnPoolByRoute$1.getPoolEntry(ConnPoolByRoute.java:298) 
org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager$1.getConnection(ThreadSafeClientConnManager.java:238) 
org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:422) 
org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863) 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82) 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106) 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57) 
org.restlet.ext.httpclient.internal.HttpMethodCall.sendRequest(HttpMethodCall.java:339) 
org.restlet.engine.adapter.ClientAdapter.commit(ClientAdapter.java:105) 
org.restlet.engine.adapter.HttpClientHelper.handle(HttpClientHelper.java:119) 
org.restlet.Client.handle(Client.java:153) 
org.restlet.routing.Filter.doHandle(Filter.java:150) 
org.restlet.routing.Filter.handle(Filter.java:197) 
org.restlet.routing.Router.doHandle(Router.java:422) 
org.restlet.routing.Router.handle(Router.java:641) 
org.restlet.engine.component.ComponentClientDispatcher.doHandle(ComponentClientDispatcher.java:129) 
org.restlet.routing.Filter.handle(Filter.java:197) 
org.restlet.engine.util.ChildClientDispatcher.parentHandle(ChildClientDispatcher.java:139) 
org.restlet.engine.util.ChildClientDispatcher.doHandle(ChildClientDispatcher.java:112) 
org.restlet.routing.Filter.handle(Filter.java:197) 
org.restlet.engine.util.ChildClientDispatcher.parentHandle(ChildClientDispatcher.java:139) 
org.restlet.engine.util.ChildClientDispatcher.doHandle(ChildClientDispatcher.java:112) 
org.restlet.routing.Filter.handle(Filter.java:197) 
org.restlet.routing.Filter.doHandle(Filter.java:150) 
org.restlet.routing.Filter.handle(Filter.java:197) 
org.restlet.routing.Redirector.serverRedirect(Redirector.java:516) 
org.restlet.routing.Redirector.outboundServerRedirect(Redirector.java:394) 
org.restlet.routing.Redirector.handle(Redirector.java:328) 
org.restlet.routing.Filter.doHandle(Filter.java:150) 
org.restlet.routing.Filter.handle(Filter.java:197) 
org.restlet.routing.Router.doHandle(Router.java:422) 
org.restlet.routing.Router.handle(Router.java:641) 
org.restlet.routing.Filter.doHandle(Filter.java:150) 
org.restlet.routing.Filter.handle(Filter.java:197) 
org.restlet.routing.Filter.doHandle(Filter.java:150) 
org.restlet.routing.Filter.handle(Filter.java:197) 
org.restlet.routing.Filter.doHandle(Filter.java:150) 
org.restlet.engine.application.StatusFilter.doHandle(StatusFilter.java:140) 
org.restlet.routing.Filter.handle(Filter.java:197) 
org.restlet.routing.Filter.doHandle(Filter.java:150) 
org.restlet.routing.Filter.handle(Filter.java:197) 
org.restlet.engine.CompositeHelper.handle(CompositeHelper.java:202) 
org.restlet.engine.application.ApplicationHelper.handle(ApplicationHelper.java:77) 
org.restlet.Application.handle(Application.java:385) 
org.restlet.routing.Filter.doHandle(Filter.java:150) 
org.restlet.routing.Filter.handle(Filter.java:197) 
org.restlet.routing.Router.doHandle(Router.java:422) 
org.restlet.routing.Router.handle(Router.java:641) 
org.restlet.routing.Filter.doHandle(Filter.java:150) 
org.restlet.routing.Filter.handle(Filter.java:197) 
org.restlet.routing.Router.doHandle(Router.java:422) 
org.restlet.routing.Router.handle(Router.java:641) 
org.restlet.routing.Filter.doHandle(Filter.java:150) 
org.restlet.engine.application.StatusFilter.doHandle(StatusFilter.java:140) 
org.restlet.routing.Filter.handle(Filter.java:197) 
org.restlet.routing.Filter.doHandle(Filter.java:150) 
org.restlet.routing.Filter.handle(Filter.java:197) 
org.restlet.engine.CompositeHelper.handle(CompositeHelper.java:202) 
org.restlet.Component.handle(Component.java:408) 
org.restlet.Server.handle(Server.java:507) 
org.restlet.engine.connector.ServerHelper.handle(ServerHelper.java:63) 
org.restlet.engine.adapter.HttpServerHelper.handle(HttpServerHelper.java:143) 
org.restlet.ext.jetty.JettyServerHelper$WrappedServer.handle(JettyServerHelper.java:256) 
org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:311) 
org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:257) 
org.eclipse.jetty.io.AbstractConnection$2.run(AbstractConnection.java:544) 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635) 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) 
java.lang.Thread.run(Thread.java:745)

--
You received this message because you are subscribed to the Google Groups "Restlet Framework (Discuss)" group.
To unsubscribe from this group and stop receiving emails from it, send an email to [hidden email].
Reply | Threaded
Open this post in threaded view
|  
Report Content as Inappropriate

Re: Restlet hangs on multiple threads

Jerome Louvel-3
Hi Jon,

That's strange... Maybe you should tweak some of the Apache HTTP Client connector parameters like the max number of connections:

Jerome


On Thu, Apr 13, 2017 at 12:39 PM, jon <[hidden email]> wrote:
I have an interesting problem that occurs within 5-7 days interval. 
The setup is a restlet server (http just forwarding to https) with jetty and apache connector and a mysql db. The apache server is behind firewall on separate ports (http and https). Restlet redirects web calls directly to the apache server 
When the error occurs the web page access stalls/hangs (but normal access to restlet resources is ok). The thread dump is full of instances like the below (hundreds). It seems to me that threads are waiting on a connection pool, and is not terminated. 
I've used jmeter to stresstest the system, but are not able to recreate the problem. 
Any suggestions on this one? 


State: WAITING on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@7c8a36b1 
Total blocked: 0  Total waited: 3 

Stack trace: 
sun.misc.Unsafe.park(Native Method) 
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) 
org.apache.http.impl.conn.tsccm.WaitingThread.await(WaitingThread.java:159) 
org.apache.http.impl.conn.tsccm.ConnPoolByRoute.getEntryBlocking(ConnPoolByRoute.java:398) 
org.apache.http.impl.conn.tsccm.ConnPoolByRoute$1.getPoolEntry(ConnPoolByRoute.java:298) 
org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager$1.getConnection(ThreadSafeClientConnManager.java:238) 
org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:422) 
org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863) 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82) 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106) 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57) 
org.restlet.ext.httpclient.internal.HttpMethodCall.sendRequest(HttpMethodCall.java:339) 
org.restlet.engine.adapter.ClientAdapter.commit(ClientAdapter.java:105) 
org.restlet.engine.adapter.HttpClientHelper.handle(HttpClientHelper.java:119) 
org.restlet.Client.handle(Client.java:153) 
org.restlet.routing.Filter.doHandle(Filter.java:150) 
org.restlet.routing.Filter.handle(Filter.java:197) 
org.restlet.routing.Router.doHandle(Router.java:422) 
org.restlet.routing.Router.handle(Router.java:641) 
org.restlet.engine.component.ComponentClientDispatcher.doHandle(ComponentClientDispatcher.java:129) 
org.restlet.routing.Filter.handle(Filter.java:197) 
org.restlet.engine.util.ChildClientDispatcher.parentHandle(ChildClientDispatcher.java:139) 
org.restlet.engine.util.ChildClientDispatcher.doHandle(ChildClientDispatcher.java:112) 
org.restlet.routing.Filter.handle(Filter.java:197) 
org.restlet.engine.util.ChildClientDispatcher.parentHandle(ChildClientDispatcher.java:139) 
org.restlet.engine.util.ChildClientDispatcher.doHandle(ChildClientDispatcher.java:112) 
org.restlet.routing.Filter.handle(Filter.java:197) 
org.restlet.routing.Filter.doHandle(Filter.java:150) 
org.restlet.routing.Filter.handle(Filter.java:197) 
org.restlet.routing.Redirector.serverRedirect(Redirector.java:516) 
org.restlet.routing.Redirector.outboundServerRedirect(Redirector.java:394) 
org.restlet.routing.Redirector.handle(Redirector.java:328) 
org.restlet.routing.Filter.doHandle(Filter.java:150) 
org.restlet.routing.Filter.handle(Filter.java:197) 
org.restlet.routing.Router.doHandle(Router.java:422) 
org.restlet.routing.Router.handle(Router.java:641) 
org.restlet.routing.Filter.doHandle(Filter.java:150) 
org.restlet.routing.Filter.handle(Filter.java:197) 
org.restlet.routing.Filter.doHandle(Filter.java:150) 
org.restlet.routing.Filter.handle(Filter.java:197) 
org.restlet.routing.Filter.doHandle(Filter.java:150) 
org.restlet.engine.application.StatusFilter.doHandle(StatusFilter.java:140) 
org.restlet.routing.Filter.handle(Filter.java:197) 
org.restlet.routing.Filter.doHandle(Filter.java:150) 
org.restlet.routing.Filter.handle(Filter.java:197) 
org.restlet.engine.CompositeHelper.handle(CompositeHelper.java:202) 
org.restlet.engine.application.ApplicationHelper.handle(ApplicationHelper.java:77) 
org.restlet.Application.handle(Application.java:385) 
org.restlet.routing.Filter.doHandle(Filter.java:150) 
org.restlet.routing.Filter.handle(Filter.java:197) 
org.restlet.routing.Router.doHandle(Router.java:422) 
org.restlet.routing.Router.handle(Router.java:641) 
org.restlet.routing.Filter.doHandle(Filter.java:150) 
org.restlet.routing.Filter.handle(Filter.java:197) 
org.restlet.routing.Router.doHandle(Router.java:422) 
org.restlet.routing.Router.handle(Router.java:641) 
org.restlet.routing.Filter.doHandle(Filter.java:150) 
org.restlet.engine.application.StatusFilter.doHandle(StatusFilter.java:140) 
org.restlet.routing.Filter.handle(Filter.java:197) 
org.restlet.routing.Filter.doHandle(Filter.java:150) 
org.restlet.routing.Filter.handle(Filter.java:197) 
org.restlet.engine.CompositeHelper.handle(CompositeHelper.java:202) 
org.restlet.Component.handle(Component.java:408) 
org.restlet.Server.handle(Server.java:507) 
org.restlet.engine.connector.ServerHelper.handle(ServerHelper.java:63) 
org.restlet.engine.adapter.HttpServerHelper.handle(HttpServerHelper.java:143) 
org.restlet.ext.jetty.JettyServerHelper$WrappedServer.handle(JettyServerHelper.java:256) 
org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:311) 
org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:257) 
org.eclipse.jetty.io.AbstractConnection$2.run(AbstractConnection.java:544) 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635) 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) 
java.lang.Thread.run(Thread.java:745)

--
You received this message because you are subscribed to the Google Groups "Restlet Framework (Discuss)" group.
To unsubscribe from this group and stop receiving emails from it, send an email to [hidden email].
Reply | Threaded
Open this post in threaded view
|  
Report Content as Inappropriate

Re: Restlet hangs on multiple threads

Marc Larue-2
I had that problem too, so I wrote my own async. HTTP client, it only supports fixed length though (Content-Length header) so if you control the servers you are requesting it should work:


Kind Regards,

/marc

On 14 April 2017 at 03:29, Jerome Louvel <[hidden email]> wrote:
Hi Jon,

That's strange... Maybe you should tweak some of the Apache HTTP Client connector parameters like the max number of connections:

Jerome


On Thu, Apr 13, 2017 at 12:39 PM, jon <[hidden email]> wrote:
I have an interesting problem that occurs within 5-7 days interval. 
The setup is a restlet server (http just forwarding to https) with jetty and apache connector and a mysql db. The apache server is behind firewall on separate ports (http and https). Restlet redirects web calls directly to the apache server 
When the error occurs the web page access stalls/hangs (but normal access to restlet resources is ok). The thread dump is full of instances like the below (hundreds). It seems to me that threads are waiting on a connection pool, and is not terminated. 
I've used jmeter to stresstest the system, but are not able to recreate the problem. 
Any suggestions on this one? 


State: WAITING on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@7c8a36b1 
Total blocked: 0  Total waited: 3 

Stack trace: 
sun.misc.Unsafe.park(Native Method) 
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) 
org.apache.http.impl.conn.tsccm.WaitingThread.await(WaitingThread.java:159) 
org.apache.http.impl.conn.tsccm.ConnPoolByRoute.getEntryBlocking(ConnPoolByRoute.java:398) 
org.apache.http.impl.conn.tsccm.ConnPoolByRoute$1.getPoolEntry(ConnPoolByRoute.java:298) 
org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager$1.getConnection(ThreadSafeClientConnManager.java:238) 
org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:422
org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863) 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82) 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106) 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57) 
org.restlet.ext.httpclient.internal.HttpMethodCall.sendRequest(HttpMethodCall.java:339) 
org.restlet.engine.adapter.ClientAdapter.commit(ClientAdapter.java:105) 
org.restlet.engine.adapter.HttpClientHelper.handle(HttpClientHelper.java:119) 
org.restlet.Client.handle(Client.java:153) 
org.restlet.routing.Filter.doHandle(Filter.java:150) 
org.restlet.routing.Filter.handle(Filter.java:197) 
org.restlet.routing.Router.doHandle(Router.java:422) 
org.restlet.routing.Router.handle(Router.java:641) 
org.restlet.engine.component.ComponentClientDispatcher.doHandle(ComponentClientDispatcher.java:129) 
org.restlet.routing.Filter.handle(Filter.java:197) 
org.restlet.engine.util.ChildClientDispatcher.parentHandle(ChildClientDispatcher.java:139) 
org.restlet.engine.util.ChildClientDispatcher.doHandle(ChildClientDispatcher.java:112) 
org.restlet.routing.Filter.handle(Filter.java:197) 
org.restlet.engine.util.ChildClientDispatcher.parentHandle(ChildClientDispatcher.java:139) 
org.restlet.engine.util.ChildClientDispatcher.doHandle(ChildClientDispatcher.java:112) 
org.restlet.routing.Filter.handle(Filter.java:197) 
org.restlet.routing.Filter.doHandle(Filter.java:150) 
org.restlet.routing.Filter.handle(Filter.java:197) 
org.restlet.routing.Redirector.serverRedirect(Redirector.java:516) 
org.restlet.routing.Redirector.outboundServerRedirect(Redirector.java:394) 
org.restlet.routing.Redirector.handle(Redirector.java:328) 
org.restlet.routing.Filter.doHandle(Filter.java:150) 
org.restlet.routing.Filter.handle(Filter.java:197) 
org.restlet.routing.Router.doHandle(Router.java:422) 
org.restlet.routing.Router.handle(Router.java:641) 
org.restlet.routing.Filter.doHandle(Filter.java:150) 
org.restlet.routing.Filter.handle(Filter.java:197) 
org.restlet.routing.Filter.doHandle(Filter.java:150) 
org.restlet.routing.Filter.handle(Filter.java:197) 
org.restlet.routing.Filter.doHandle(Filter.java:150) 
org.restlet.engine.application.StatusFilter.doHandle(StatusFilter.java:140) 
org.restlet.routing.Filter.handle(Filter.java:197) 
org.restlet.routing.Filter.doHandle(Filter.java:150) 
org.restlet.routing.Filter.handle(Filter.java:197) 
org.restlet.engine.CompositeHelper.handle(CompositeHelper.java:202) 
org.restlet.engine.application.ApplicationHelper.handle(ApplicationHelper.java:77) 
org.restlet.Application.handle(Application.java:385) 
org.restlet.routing.Filter.doHandle(Filter.java:150) 
org.restlet.routing.Filter.handle(Filter.java:197) 
org.restlet.routing.Router.doHandle(Router.java:422) 
org.restlet.routing.Router.handle(Router.java:641) 
org.restlet.routing.Filter.doHandle(Filter.java:150) 
org.restlet.routing.Filter.handle(Filter.java:197) 
org.restlet.routing.Router.doHandle(Router.java:422) 
org.restlet.routing.Router.handle(Router.java:641) 
org.restlet.routing.Filter.doHandle(Filter.java:150) 
org.restlet.engine.application.StatusFilter.doHandle(StatusFilter.java:140) 
org.restlet.routing.Filter.handle(Filter.java:197) 
org.restlet.routing.Filter.doHandle(Filter.java:150) 
org.restlet.routing.Filter.handle(Filter.java:197) 
org.restlet.engine.CompositeHelper.handle(CompositeHelper.java:202) 
org.restlet.Component.handle(Component.java:408) 
org.restlet.Server.handle(Server.java:507) 
org.restlet.engine.connector.ServerHelper.handle(ServerHelper.java:63) 
org.restlet.engine.adapter.HttpServerHelper.handle(HttpServerHelper.java:143) 
org.restlet.ext.jetty.JettyServerHelper$WrappedServer.handle(JettyServerHelper.java:256) 
org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:311) 
org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:257) 
org.eclipse.jetty.io.AbstractConnection$2.run(AbstractConnection.java:544) 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635) 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) 
java.lang.Thread.run(Thread.java:745)

--
You received this message because you are subscribed to the Google Groups "Restlet Framework (Discuss)" group.
To unsubscribe from this group and stop receiving emails from it, send an email to [hidden email].

--
You received this message because you are subscribed to the Google Groups "Restlet Framework (Discuss)" group.
To unsubscribe from this group and stop receiving emails from it, send an email to [hidden email].
Loading...