Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions integration_tests/test_client_integrated.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,8 @@
import sys
import json
import numpy as np
import dill
from test_data import (render_host, render_port,
client_script_location, tilespec_file, tform_file, test_pool_size)
from pathos.multiprocessing import ProcessingPool as Pool
import PIL

root = logging.getLogger()
Expand Down Expand Up @@ -101,14 +99,15 @@ def test_import_jsonfiles_parallel(
validate_stack_import(render, stack, tilespecs)
renderapi.stack.delete_stack(stack, render=render)


def square(x):
return x**2
def test_import_jsonfiles_parallel_multiple(
render, render_example_tilespec_and_transforms, poolsize=test_pool_size):
stacks = ['testmultiple1', 'testmultiple2', 'testmultiple3']
mylist = range(10)
for stack in stacks:
with renderapi.client.WithPool(poolsize) as pool:
results = pool.map(lambda x: x**2, mylist)
results = pool.map(square, mylist)
test_import_jsonfiles_parallel(
render, render_example_tilespec_and_transforms, stack, poolsize=poolsize)

Expand Down Expand Up @@ -272,3 +271,4 @@ def test_call_run_ws_client_renderclient(render, teststack):
render.DEFAULT_PROJECT, teststack) + [zvalues[0]]
assert not renderapi.client.call_run_ws_client(
test_class, add_args=args, subprocess_mode='call', renderclient=render)

15 changes: 10 additions & 5 deletions renderapi/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
from .utils import NullHandler, renderdump_temp, fitargspec
from .render import RenderClient, renderaccess, Render, format_preamble, format_baseurl
from .stack import set_stack_state, make_stack_params
from pathos.multiprocessing import ProcessingPool as Pool
from multiprocessing.pool import Pool

# setup logger
logger = logging.getLogger(__name__)
Expand Down Expand Up @@ -77,15 +77,15 @@ def renderclientaccess(f, *args, **kwargs):


class WithPool(Pool):
"""pathos ProcessingPool with functioning __exit__ call
"""Multiprocessing.pool.Pool with functioning __exit__ call

Parameters
----------
*args
variable length argument list matching input
to pathos.multiprocessing.Pool
to multiprocessing.pool.Pool
**kwargs
keyword argument input matching pathos.multiprocessing.Pool
keyword argument input matching multiprocessing.pool.Pool

Examples
--------
Expand All @@ -96,8 +96,12 @@ class WithPool(Pool):
def __init__(self, *args, **kwargs):
super(WithPool, self).__init__(*args, **kwargs)

def __enter__(self):
return self

def __exit__(self, *args, **kwargs):
super(WithPool, self)._clear()
self.close()
self.join()


@renderclientaccess
Expand Down Expand Up @@ -1111,3 +1115,4 @@ def pointMatchClient(stack, collection, tile_pairs,
memGB=memGB, client_script=client_script,
subprocess_mode=subprocess_mode, add_args=argvs,
**kwargs)

2 changes: 0 additions & 2 deletions requirements.txt
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
requests
numpy
pillow
dill>=0.2.6
pathos
sphinxcontrib-napoleon
decorator