From 7c95f5bb4ccc22fbb65a19e5cf57900d6e3b258f Mon Sep 17 00:00:00 2001 From: Russel Torres Date: Tue, 24 Nov 2020 10:32:23 -0800 Subject: [PATCH 1/2] tilePairClient: add support for more parameters and multi-file support --- integration_tests/test_client_integrated.py | 8 ++- renderapi/client/client_calls.py | 78 +++++++++++++++++++-- 2 files changed, 80 insertions(+), 6 deletions(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index 50761d65..473d6e5c 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -185,7 +185,6 @@ def teststack2(render, render_example_tilespec_and_transforms): renderapi.stack.delete_stack(stack, render=render) - def test_tile_pair_client(render, teststack, **kwargs): zvalues = np.array(renderapi.stack.get_z_values_for_stack( teststack, render=render)) @@ -196,6 +195,13 @@ def test_tile_pair_client(render, teststack, **kwargs): assert isinstance(tilepairjson, dict) assert len(tilepairjson['neighborPairs']) > 3 + multitpjson, multitpfiles = renderapi.client.tilePairClient( + teststack, np.min(zvalues), np.max(zvalues), outjson=outjson, + render=render, maxPairsPerFile=2, return_jsonfiles=True, **kwargs) + + assert len(multitpjson["neighborPairs"]) == len(tilepairjson["neighborPairs"]) + assert len(multitpfiles) == len(tilepairjson["neighborPairs"]) // 2 + @pytest.mark.parametrize("bounds,raises", [ ({}, True), diff --git a/renderapi/client/client_calls.py b/renderapi/client/client_calls.py index 68adc2f5..c4cb956f 100644 --- a/renderapi/client/client_calls.py +++ b/renderapi/client/client_calls.py @@ -1,6 +1,7 @@ import logging import json import os +import re import subprocess import tempfile @@ -135,6 +136,11 @@ def tilePairClient(stack, minz, maxz, outjson=None, delete_json=False, excludeSameSectionNeighbors=None, excludePairsInMatchCollection=None, minx=None, maxx=None, miny=None, maxy=None, + useRowColPositions=None, existingMatchOwner=None, + minExistingMatchCount=None, onlyIncludeTilesFromStack=None, + onlyIncludeTilesNearTileIdsJson=None, maxPairsPerFile=None, + return_jsondata=True, + return_jsonfiles=False, subprocess_mode=None, host=None, port=None, owner=None, project=None, client_script=None, memGB=None, @@ -191,11 +197,29 @@ def tilePairClient(stack, minz, maxz, outjson=None, delete_json=False, minimum y bound from which tile 'p' is selected maxy : float maximum y bound from wich tile 'p' is selected + useRowColPositions : bool + whether to use raster positions for neighbor analysis rather + than radial cutoff + existingMatchOwner : str + owner for the excludePairsInMatchCollection collection + minExistingMatchCount : int + minimum match threshold to exclude pairs + present in excludePairsInMatchCollection collection + onlyIncludeTilesFromStack : str + only include tiles which exist in this stack + onlyIncludeTilesNearTileIdsJson : str + path to json listing tileIds which should be paired + maxPairsPerFile : int + maximum neighborPairs per file. Generating more pairs than + this number (default 100000) will generate additional json + files with a p[0-9]+ suffix on the root. Returns ------- - :obj:`list` of :obj:`dict` + :obj:`list` of :obj:`dict`, optional list of tilepairs + :obj:`list` of string, optional + list of json files containing tilepairs """ if outjson is None: with tempfile.NamedTemporaryFile( @@ -219,6 +243,18 @@ def tilePairClient(stack, minz, maxz, outjson=None, delete_json=False, '--excludeSameSectionNeighbors') + get_param(excludePairsInMatchCollection, '--excludePairsInMatchCollection') + + get_param(useRowColPositions, + '--useRowColPositions') + + get_param(existingMatchOwner, + '--existingMatchOwner') + + get_param(minExistingMatchCount, + '--minExistingMatchCount') + + get_param(onlyIncludeTilesFromStack, + "--onlyIncludeTilesFromStack") + + get_param(onlyIncludeTilesNearTileIdsJson, + '--onlyIncludeTilesNearTileIdsJson') + + get_param(maxPairsPerFile, + '--maxPairsPerFile') + ['--toJson', outjson] + get_param(minx, '--minX') + get_param(maxx, '--maxX') + get_param(miny, '--minY') + get_param(maxy, '--maxY')) @@ -228,12 +264,44 @@ def tilePairClient(stack, minz, maxz, outjson=None, delete_json=False, subprocess_mode=subprocess_mode, add_args=argvs, **kwargs) - with open(outjson, 'r') as f: - jsondata = json.load(f) + # We create the jsonfile, so if it is empty it could be multiple + if os.stat(outjson).st_size == 0: + # outjson we created is empty, so remove it + os.remove(outjson) + + outbn_root, outbn_ext = os.path.splitext(os.path.basename(outjson)) + outbn_pattern = "{root}_p[0-9]+{ext}".format( + root=outbn_root, ext=outbn_ext) + jsonfiles = [ + os.path.join(os.path.dirname(outjson), bn) + for bn in os.listdir(os.path.dirname(outjson)) + if re.match(outbn_pattern, bn)] + else: + jsonfiles = [outjson] + + if return_jsondata: + jsondata_list = [] + for jsonfile in jsonfiles: + with open(jsonfile, 'r') as f: + jsondata_list.append(json.load(f)) + if len({d["renderParametersUrlTemplate"] for d in jsondata_list}) != 1: # pragma: no cover + raise ValueError( + "Found tilepair files with disparate " + "renderParametersUrlTemplate values. " + "Maybe there are additional files " + "matching the outjson pattern?") + pairdata = [i for l in (d["neighborPairs"] for d in jsondata_list) + for i in l] + jsondata = dict(jsondata_list[0], **{"neighborPairs": pairdata}) if delete_json: - os.remove(outjson) - return jsondata + for jsonfile in jsonfiles: + os.remove(jsonfile) + + return ((jsondata, jsonfiles) if return_jsonfiles and return_jsondata + else jsondata if return_jsondata + else jsonfiles if return_jsonfiles + else None) @renderclientaccess From 92733b2bf0a0361a7a510bb1fa3cb720cd91b8f1 Mon Sep 17 00:00:00 2001 From: Russel Torres Date: Tue, 24 Nov 2020 11:23:14 -0800 Subject: [PATCH 2/2] tilePairClient: better support for non-null outjson inputs --- integration_tests/test_client_integrated.py | 14 +++++++++++--- renderapi/client/client_calls.py | 7 ++++--- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index 473d6e5c..8e1dc551 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -195,12 +195,20 @@ def test_tile_pair_client(render, teststack, **kwargs): assert isinstance(tilepairjson, dict) assert len(tilepairjson['neighborPairs']) > 3 + multitpjson_tmp, multitpfiles_tmp = renderapi.client.tilePairClient( + teststack, np.min(zvalues), np.max(zvalues), outjson=None, + render=render, maxPairsPerFile=2, return_jsonfiles=True, **kwargs) + + with tempfile.NamedTemporaryFile( + suffix=".json", mode='r', delete=False) as f: + outjson_specified = f.name + multitpjson, multitpfiles = renderapi.client.tilePairClient( - teststack, np.min(zvalues), np.max(zvalues), outjson=outjson, + teststack, np.min(zvalues), np.max(zvalues), outjson=outjson_specified, render=render, maxPairsPerFile=2, return_jsonfiles=True, **kwargs) - assert len(multitpjson["neighborPairs"]) == len(tilepairjson["neighborPairs"]) - assert len(multitpfiles) == len(tilepairjson["neighborPairs"]) // 2 + assert len(multitpjson["neighborPairs"]) == len(tilepairjson["neighborPairs"]) == len(multitpjson_tmp["neighborPairs"]) + assert len(multitpfiles) == len(multitpfiles_tmp) == len(tilepairjson["neighborPairs"]) // 2 @pytest.mark.parametrize("bounds,raises", [ diff --git a/renderapi/client/client_calls.py b/renderapi/client/client_calls.py index c4cb956f..2baf1235 100644 --- a/renderapi/client/client_calls.py +++ b/renderapi/client/client_calls.py @@ -265,9 +265,10 @@ def tilePairClient(stack, minz, maxz, outjson=None, delete_json=False, add_args=argvs, **kwargs) # We create the jsonfile, so if it is empty it could be multiple - if os.stat(outjson).st_size == 0: - # outjson we created is empty, so remove it - os.remove(outjson) + if not os.path.isfile(outjson) or os.stat(outjson).st_size == 0: + if os.path.isfile(outjson): + # outjson we created is empty, so remove it + os.remove(outjson) outbn_root, outbn_ext = os.path.splitext(os.path.basename(outjson)) outbn_pattern = "{root}_p[0-9]+{ext}".format(