Skip to content

Commit e53b4d0

Browse files
antonbabenkoantonbabenko
authored
Merge pull request antonbabenko#8 from antonbabenko/fast_render
Make rendering faster by rendering once (instead of rendering each layer separately)
2 parents f95329e + 69d1ce0 commit e53b4d0

16 files changed

Lines changed: 1540 additions & 71 deletions

File tree

‎input/blueprint_bunch_of_resources.json‎

Lines changed: 1418 additions & 0 deletions
Large diffs are not rendered by default.

‎modulestf/cloudcraft/graph.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,9 @@ def populate_graph(data): # noqa: C901
8080
for edge in G.edges.data():
8181
edge = list(edge)
8282

83-
if edge[0] in connectors or edge[1] in connectors:
83+
if edge[0] == edge[1]:
84+
edge = []
85+
elif edge[0] in connectors or edge[1] in connectors:
8486
break
8587
else:
8688
edge = []

‎modulestf/const.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,9 @@
99

1010
OUTPUT_DIR = "output"
1111
WORK_DIR = "work"
12-
FINAL_DIR = "../final"
12+
WORK_DIR_FOR_COOKIECUTTER = "{{ cookiecutter.dir_name }}"
13+
14+
FINAL_DIR = "final"
1315

1416
S3_BUCKET = "dl.modules.tf"
1517
S3_BUCKET_REGION = "eu-west-1"

‎modulestf/render.py‎

Lines changed: 98 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
11
import glob
22
import json
3-
import os
3+
import pathlib
44
import re
55
import shutil
6+
from os import chdir, getcwd, makedirs, mkdir, path
67
from pprint import pformat, pprint
78

89
from cookiecutter.exceptions import NonTemplatedInputDirException
@@ -21,73 +22,101 @@ def mkdir_safely(dir):
2122
pass
2223

2324
try:
24-
os.mkdir(dir)
25+
mkdir(dir)
2526
except OSError:
2627
pass
2728

2829

2930
def prepare_render_dirs():
30-
output_dir = os.path.join(tmp_dir, OUTPUT_DIR)
31+
# return
32+
output_dir = path.join(tmp_dir, OUTPUT_DIR)
3133

3234
mkdir_safely(output_dir)
33-
os.chdir(output_dir)
35+
chdir(output_dir)
3436

3537
mkdir_safely(WORK_DIR)
36-
os.chdir(WORK_DIR)
38+
chdir(WORK_DIR)
3739

38-
mkdir_safely(FINAL_DIR)
40+
# mkdir_safely(FINAL_DIR)
3941

42+
mkdir(WORK_DIR_FOR_COOKIECUTTER)
4043

41-
def render_single_layer(resource, region):
44+
45+
def find_templates_files(dir):
46+
pprint("DIR = %s" % dir)
47+
48+
files = glob.glob(dir + "/*") + \
49+
glob.glob(dir + "/.*")
50+
51+
return files
52+
53+
54+
def prepare_single_layer(resource, region, templates_dir, templates_files):
4255

4356
dir_name = resource.get("dir_name")
4457

45-
full_dir_name = ("single_layer/%s/%s" % (region, dir_name)).lower()
58+
full_dir_name = ("%s/%s" % (region, dir_name)).lower()
4659

4760
single_layer = {
48-
"dir_name": full_dir_name,
49-
"region": region,
50-
"module_source": MODULES[resource["type"]]["source"],
51-
"module_variables": MODULES[resource["type"]]["variables"],
61+
"module_type": resource["type"]
5262
}
5363

5464
extra_context = resource.update(single_layer) or resource
5565

56-
cookiecutter(os.path.join(COOKIECUTTER_TEMPLATES_DIR, COOKIECUTTER_TEMPLATES_PREFIX + "-single-layer"),
57-
config_file=os.path.join(COOKIECUTTER_TEMPLATES_DIR, "config_aws_lambda.yaml"),
58-
no_input=True,
59-
extra_context=extra_context)
66+
data = '{%- set this = ' + str(extra_context) + ' -%}'
6067

68+
dst_dir = path.join(getcwd(), WORK_DIR_FOR_COOKIECUTTER, full_dir_name)
6169

62-
def render_common_layer(region):
70+
for file in templates_files:
71+
# pprint("original = %s" % file)
72+
part_of_path_to_keep = "".join(file[len(templates_dir):])
6373

64-
common_layer = {
65-
"dir_name": "common_layer",
66-
"region": region,
67-
}
74+
# pprint("just relative file = %s" % part_file)
75+
# pprint("getcwd = %s" % getcwd())
6876

69-
try:
70-
cookiecutter(os.path.join(COOKIECUTTER_TEMPLATES_DIR, COOKIECUTTER_TEMPLATES_PREFIX + "-common-layer"),
71-
config_file=os.path.join(COOKIECUTTER_TEMPLATES_DIR, "config_aws_lambda.yaml"),
72-
no_input=True,
73-
extra_context=common_layer)
74-
except NonTemplatedInputDirException:
75-
pass
77+
dst_file = dst_dir + part_of_path_to_keep
78+
# pprint("new file = %s" % dst_file)
7679

80+
with open(file, "r") as original:
81+
original_data = original.read()
82+
original.close()
7783

78-
def render_root_dir(source, region, dirs):
84+
makedirs(path.dirname(dst_file), exist_ok=True)
7985

80-
root_dir = {
81-
"dir_name": "root_dir",
82-
"source_name": source["name"],
83-
"region": region,
84-
"dirs": dirs,
85-
}
86+
with open(dst_file, "w") as modified:
87+
modified.write(data + "\n" + original_data)
88+
modified.close()
89+
90+
shutil.copy(templates_dir + "/../cookiecutter.json", "cookiecutter.json")
91+
92+
return resource["type"]
93+
94+
95+
# Copy all files and subdirectories into working directory
96+
def copy_to_working_dir(templates_dir):
8697

87-
cookiecutter(os.path.join(COOKIECUTTER_TEMPLATES_DIR, "root"),
88-
config_file=os.path.join(COOKIECUTTER_TEMPLATES_DIR, "config_aws_lambda.yaml"),
98+
dst_dir = path.realpath(WORK_DIR_FOR_COOKIECUTTER)
99+
100+
files = find_templates_files(templates_dir)
101+
102+
for file in files:
103+
pprint("FILE == %s" % file)
104+
pprint("dst_dir == %s" % dst_dir)
105+
if path.isdir(file):
106+
dst = path.join(dst_dir, path.basename(file))
107+
shutil.copytree(file, dst)
108+
else:
109+
shutil.copy(file, dst_dir)
110+
111+
112+
def render_all(extra_context):
113+
114+
output_dir = path.join(tmp_dir, OUTPUT_DIR, WORK_DIR)
115+
116+
cookiecutter(output_dir,
117+
config_file=path.join(COOKIECUTTER_TEMPLATES_DIR, "config_aws_lambda.yaml"),
89118
no_input=True,
90-
extra_context=root_dir)
119+
extra_context=extra_context)
91120

92121

93122
# Count unique combination of type and text to decide if to append unique resource id
@@ -169,6 +198,13 @@ def render_from_modulestf_config(config, source, regions):
169198

170199
types_text[t] = new_appendix
171200

201+
# Find all templates for single layer once
202+
templates_dir = path.realpath(path.join(COOKIECUTTER_TEMPLATES_DIR, COOKIECUTTER_TEMPLATES_PREFIX + "-single-layer/template"))
203+
templates_files = find_templates_files(templates_dir)
204+
205+
# Set of used module to load data once
206+
used_modules = set()
207+
172208
# render single layers in a loop
173209
for resource in resources:
174210

@@ -201,23 +237,33 @@ def render_from_modulestf_config(config, source, regions):
201237

202238
# Render the layer
203239
logger.info("Rendering single layer resource id: %s" % resource.get("ref_id"))
204-
render_single_layer(resource, region)
240+
used_module_type = prepare_single_layer(resource, region, templates_dir, templates_files)
205241

206-
logger.info("Rendering common layer")
207-
render_common_layer(region)
242+
used_modules.add(used_module_type)
208243

209-
logger.info("Rendering root dir")
210-
render_root_dir(source, region, dirs)
244+
extra_context = dict({"module_sources": {}, "module_variables": {}})
245+
for module_type in used_modules:
246+
extra_context["module_sources"].update({
247+
module_type: MODULES[module_type]["source"],
248+
})
211249

212-
files = glob.glob("single_layer/*") + \
213-
glob.glob("single_layer/.*") + \
214-
glob.glob("common_layer/*") + \
215-
glob.glob("common_layer/.*") + \
216-
glob.glob("root_dir/*") + \
217-
glob.glob("root_dir/.*")
250+
extra_context["module_variables"].update({
251+
module_type: MODULES[module_type]["variables"],
252+
})
218253

219-
logger.info("Moving files into final dir: %s" % FINAL_DIR)
220-
for file in files:
221-
shutil.move(file, FINAL_DIR)
254+
logger.info("Prepare common layer")
255+
templates_dir = path.realpath(path.join(COOKIECUTTER_TEMPLATES_DIR, COOKIECUTTER_TEMPLATES_PREFIX + "-common-layer/template"))
256+
copy_to_working_dir(templates_dir)
257+
258+
logger.info("Prepare root dir")
259+
templates_dir = path.realpath(path.join(COOKIECUTTER_TEMPLATES_DIR, "root/template"))
260+
copy_to_working_dir(templates_dir)
261+
262+
extra_context["source_name"] = source["name"]
263+
extra_context["dirs"] = dirs
264+
extra_context["region"] = region
265+
266+
logger.info("Rendering all")
267+
render_all(extra_context)
222268

223269
logger.info("Complete!")

‎serverless.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ provider:
99
region: eu-west-1
1010
profile: private-anton
1111
tracing: true
12-
timeout: 10
12+
timeout: 30
1313
environment:
1414
S3_BUCKET: "dl.modules.tf"
1515
S3_DIR: ${self:custom.stage}

templates/root/{{ cookiecutter.dir_name }}/.editorconfig renamed to templates/root/template/.editorconfig

File renamed without changes.

templates/root/{{ cookiecutter.dir_name }}/.gitignore renamed to templates/root/template/.gitignore

File renamed without changes.

templates/root/{{ cookiecutter.dir_name }}/.pre-commit-config.yaml renamed to templates/root/template/.pre-commit-config.yaml

File renamed without changes.

templates/root/{{ cookiecutter.dir_name }}/LICENSE renamed to templates/root/template/LICENSE

File renamed without changes.

templates/root/{{ cookiecutter.dir_name }}/README.md renamed to templates/root/template/README.md

File renamed without changes.

0 commit comments

Comments
 (0)