From 463585946a40b1fc9f7c1a94fae7f1d7bc17e388 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Fri, 6 Apr 2018 01:42:30 -0700 Subject: [PATCH 1/8] trying to cut pathos from library --- renderapi/client.py | 5 +++-- requirements.txt | 2 -- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/renderapi/client.py b/renderapi/client.py index 87c16495..a8fc0b34 100755 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -13,7 +13,7 @@ from .utils import NullHandler, renderdump_temp, fitargspec from .render import RenderClient, renderaccess, Render, format_preamble, format_baseurl from .stack import set_stack_state, make_stack_params -from pathos.multiprocessing import ProcessingPool as Pool +from multiprocessing.pool import Pool # setup logger logger = logging.getLogger(__name__) @@ -97,7 +97,8 @@ def __init__(self, *args, **kwargs): super(WithPool, self).__init__(*args, **kwargs) def __exit__(self, *args, **kwargs): - super(WithPool, self)._clear() + self.close() + self.join() @renderclientaccess diff --git a/requirements.txt b/requirements.txt index f56d3209..27c007c4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,5 @@ requests numpy pillow -dill>=0.2.6 -pathos sphinxcontrib-napoleon decorator From 350674a35c373ebc7085b06208befc921b43e5e0 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Fri, 6 Apr 2018 01:44:24 -0700 Subject: [PATCH 2/8] stripping pathos --- integration_tests/test_client_integrated.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index 9ede85a3..e8acbafc 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -6,10 +6,8 @@ import sys import json import numpy as np -import dill from test_data import (render_host, render_port, client_script_location, tilespec_file, tform_file, test_pool_size) -from pathos.multiprocessing import ProcessingPool as Pool import PIL root = logging.getLogger() From 56c0cc58c7c0f28a3f40621540f45119d2695cfb Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Fri, 6 Apr 2018 01:58:52 -0700 Subject: [PATCH 3/8] fixing tests --- integration_tests/test_client_integrated.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index e8acbafc..37c4ebaf 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -99,14 +99,15 @@ def test_import_jsonfiles_parallel( validate_stack_import(render, stack, tilespecs) renderapi.stack.delete_stack(stack, render=render) - +def square(x): + return x**2 def test_import_jsonfiles_parallel_multiple( render, render_example_tilespec_and_transforms, poolsize=test_pool_size): stacks = ['testmultiple1', 'testmultiple2', 'testmultiple3'] mylist = range(10) for stack in stacks: with renderapi.client.WithPool(poolsize) as pool: - results = pool.map(lambda x: x**2, mylist) + results = pool.map(square, mylist) test_import_jsonfiles_parallel( render, render_example_tilespec_and_transforms, stack, poolsize=poolsize) From c76e344cab697bc64be0c6f5fae56b8c2856ffbf Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Fri, 6 Apr 2018 02:07:31 -0700 Subject: [PATCH 4/8] added enter --- renderapi/client.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/renderapi/client.py b/renderapi/client.py index a8fc0b34..f7ed9718 100755 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -96,6 +96,9 @@ class WithPool(Pool): def __init__(self, *args, **kwargs): super(WithPool, self).__init__(*args, **kwargs) + def __enter__(self): + return self + def __exit__(self, *args, **kwargs): self.close() self.join() From 375c09dfbe03932929a7e63dbb233050cab66c3f Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Fri, 6 Apr 2018 05:38:16 -0700 Subject: [PATCH 5/8] added a simple code coverage test of parallel calls --- integration_tests/test_client_integrated.py | 4 ++++ renderapi/client.py | 17 ++++++++++++++--- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index 37c4ebaf..e9695a61 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -271,3 +271,7 @@ def test_call_run_ws_client_renderclient(render, teststack): render.DEFAULT_PROJECT, teststack) + [zvalues[0]] assert not renderapi.client.call_run_ws_client( test_class, add_args=args, subprocess_mode='call', renderclient=render) + +def test_my_module(): + renderapi.client.my_module() + \ No newline at end of file diff --git a/renderapi/client.py b/renderapi/client.py index f7ed9718..aef92b38 100755 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -77,15 +77,15 @@ def renderclientaccess(f, *args, **kwargs): class WithPool(Pool): - """pathos ProcessingPool with functioning __exit__ call + """Multiprocessing.pool.Pool with functioning __exit__ call Parameters ---------- *args variable length argument list matching input - to pathos.multiprocessing.Pool + to multiprocessing.pool.Pool **kwargs - keyword argument input matching pathos.multiprocessing.Pool + keyword argument input matching multiprocessing.pool.Pool Examples -------- @@ -1115,3 +1115,14 @@ def pointMatchClient(stack, collection, tile_pairs, memGB=memGB, client_script=client_script, subprocess_mode=subprocess_mode, add_args=argvs, **kwargs) + +def simple_subfunction(a,i): + print(a,i) + time.sleep(1) + return i + +def my_module(): + values = range(10) + myfunc = partial(simple_subfunction,5) + with WithPool(5) as pool: + results=pool.map(myfunc,values) \ No newline at end of file From d893f0090c1e1cff1bb0b5d393902961b59234b1 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Fri, 6 Apr 2018 06:39:03 -0700 Subject: [PATCH 6/8] missing import --- renderapi/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/renderapi/client.py b/renderapi/client.py index aef92b38..061c5352 100755 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -1115,7 +1115,7 @@ def pointMatchClient(stack, collection, tile_pairs, memGB=memGB, client_script=client_script, subprocess_mode=subprocess_mode, add_args=argvs, **kwargs) - +import time def simple_subfunction(a,i): print(a,i) time.sleep(1) From 7295182adabe3ce18254845697e7dd7c5dcb3216 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Fri, 6 Apr 2018 07:31:49 -0700 Subject: [PATCH 7/8] removing test of code coverage --- renderapi/client.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/renderapi/client.py b/renderapi/client.py index 061c5352..710409f1 100755 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -1115,14 +1115,4 @@ def pointMatchClient(stack, collection, tile_pairs, memGB=memGB, client_script=client_script, subprocess_mode=subprocess_mode, add_args=argvs, **kwargs) -import time -def simple_subfunction(a,i): - print(a,i) - time.sleep(1) - return i - -def my_module(): - values = range(10) - myfunc = partial(simple_subfunction,5) - with WithPool(5) as pool: - results=pool.map(myfunc,values) \ No newline at end of file + \ No newline at end of file From 7b88fcb6003c170585c561592f0910a2df94625b Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Fri, 6 Apr 2018 07:45:53 -0700 Subject: [PATCH 8/8] removing test --- integration_tests/test_client_integrated.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index e9695a61..e60b5697 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -272,6 +272,3 @@ def test_call_run_ws_client_renderclient(render, teststack): assert not renderapi.client.call_run_ws_client( test_class, add_args=args, subprocess_mode='call', renderclient=render) -def test_my_module(): - renderapi.client.my_module() - \ No newline at end of file