-
Notifications
You must be signed in to change notification settings - Fork 1
Expand file tree
/
Copy pathprojects.py
More file actions
executable file
·1544 lines (1357 loc) · 95.7 KB
/
projects.py
File metadata and controls
executable file
·1544 lines (1357 loc) · 95.7 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Import common code
from sysadmws_common import *
import gitlab
import glob
import textwrap
import subprocess
import re
import yaml
from ruamel.yaml import YAML
from ruamel.yaml.scalarstring import PreservedScalarString as pss
from jinja2 import Environment, FileSystemLoader, TemplateNotFound
from datetime import datetime
from datetime import time
import prettytable
import shutil
# Constants and envs
LOGO="Projects"
WORK_DIR = os.environ.get("ACC_WORKDIR", "/opt/sysadmws/accounting")
LOG_DIR = os.environ.get("ACC_LOGDIR", "/opt/sysadmws/accounting/log")
LOG_FILE = "projects.log"
TARIFFS_SUBDIR = "tariffs"
CLIENTS_SUBDIR = "clients"
YAML_GLOB = "*.yaml"
YAML_EXT = "yaml"
ACC_YAML = "accounting.yaml"
PROJECTS_SUBDIR = ".projects"
# Funcs
def open_file(d, f, mode):
# Check dir
if not os.path.isdir(os.path.dirname("{0}/{1}".format(d, f))):
try:
os.makedirs(os.path.dirname("{0}/{1}".format(d, f)), 0o755)
except OSError as exc:
if exc.errno == errno.EEXIST and os.path.isdir(os.path.dirname("{0}/{1}".format(d, f))):
pass
else:
raise
return open("{0}/{1}".format(d, f), mode)
# Main
if __name__ == "__main__":
# Set parser and parse args
parser = argparse.ArgumentParser(description='{LOGO} functions.'.format(LOGO=LOGO))
parser.add_argument("--debug", dest="debug", help="enable debug", action="store_true")
parser.add_argument("--git-reset", dest="git_reset", help="reset git before applying template", action="store_true")
parser.add_argument("--git-commit", dest="git_commit", help="commit changes of template apply", action="store_true")
parser.add_argument("--git-branch", dest="git_branch", help="commit to branch BRANCH instead of master", nargs=1, metavar=("BRANCH"))
parser.add_argument("--git-push", dest="git_push", help="push after commit", action="store_true")
parser.add_argument("--dry-run-gitlab", dest="dry_run_gitlab", help="no new objects created in gitlab", action="store_true")
parser.add_argument("--gitlab-runner-registration-token", dest="gitlab_runner_registration_token", help="set gitlab runner registration token for template if you do not have maintainer rights to get it with code", nargs=1, metavar=("TOKEN"))
parser.add_argument("--at-date", dest="at_date", help="use DATETIME instead of now for tariff", nargs=1, metavar=("DATETIME"))
parser.add_argument("--issue", dest="issue", help="do list/comment operations for issues", action="store_true")
parser.add_argument("--mr", dest="mr", help="do list/comment operations for MRs", action="store_true")
parser.add_argument("--include-closed", dest="include_closed", help="include closed issues/MRs in lists", action="store_true")
parser.add_argument("--assignee", dest="assignee", help="include only issues/MRs with assignee USERNAME in lists", nargs=1, metavar=("USERNAME"))
parser.add_argument("--labels", dest="labels", help="include only issues/MRs with LABELS in lists", nargs=1, metavar=("LABELS"))
parser.add_argument("--search", dest="search", help="include only issues/MRs with QUERY in lists", nargs=1, metavar=("QUERY"))
parser.add_argument("--text", dest="text", help="add TEXT to the issue/MR on --comment", nargs=1, metavar=("TEXT"))
parser.add_argument("--spend", dest="spend", help="add /spend TIME to the issue/MR on --comment", nargs=1, metavar=("TIME"))
parser.add_argument("--git-https", dest="git_https", help="use https USER and PASSWORD on update of admin project wiki", nargs=2, metavar=("USER", "PASSWORD"))
parser.add_argument("--git-config-email", dest="git_config_email", help="git config --global user.email on update of admin project wiki", nargs=1, metavar=("EMAIL"))
parser.add_argument("--git-config-name", dest="git_config_name", help="git config --global user.name on update of admin project wiki", nargs=1, metavar=("EMAIL"))
#
group = parser.add_mutually_exclusive_group(required=False)
group.add_argument("--exclude-clients", dest="exclude_clients", help="exclude clients defined by JSON_LIST from all-clients operations", nargs=1, metavar=("JSON_LIST"))
group.add_argument("--include-clients", dest="include_clients", help="include only clients defined by JSON_LIST for all-clients operations", nargs=1, metavar=("JSON_LIST"))
#
group = parser.add_mutually_exclusive_group(required=True)
group.add_argument("--setup-projects-for-client", dest="setup_projects_for_client", help="ensure -salt, -admin projects created in GitLab, their settings setup for client CLIENT", nargs=1, metavar=("CLIENT"))
group.add_argument("--setup-projects-for-all-clients", dest="setup_projects_for_all_clients", help="ensure -salt, -admin project created in GitLab, their settings setup for all clients excluding --exclude-clients or only for --include-clients", action="store_true")
group.add_argument("--clone-project-for-client", dest="clone_project_for_client", help="clones project for client CLIENT using current user git creds", nargs=1, metavar=("CLIENT"))
group.add_argument("--clone-project-for-all-clients", dest="clone_project_for_all_clients", help="clones project for all clients excluding --exclude-clients or only for --include-clients using current user git creds", action="store_true")
group.add_argument("--template-salt-project-for-client", dest="template_salt_project_for_client", help="apply templates for salt project for client CLIENT using current user git creds", nargs=1, metavar=("CLIENT"))
group.add_argument("--template-salt-project-for-all-clients", dest="template_salt_project_for_all_clients", help="apply templates for salt project for all clients excluding --exclude-clients or only for --include-clients using current user git creds", action="store_true")
group.add_argument("--update-admin-project-wiki-for-client", dest="update_admin_project_wiki_for_client", help="update admin project wiki (asset list, memo etc) for client CLIENT using current user git creds", nargs=1, metavar=("CLIENT"))
group.add_argument("--update-admin-project-wiki-for-all-clients", dest="update_admin_project_wiki_for_all_clients", help="update admin project wiki (asset list, memo etc) for all clients excluding --exclude-clients or only for --include-clients using current user git creds", action="store_true")
group.add_argument("--list", dest="list", help="list issues/MRs in CLIENT projects, ALL = all clients, either --issue or --mr required", nargs=1, metavar=("CLIENT"))
group.add_argument("--comment", dest="comment", help="add comment to PROJECT issue/MR with IID, title can be used instead of IID", nargs=2, metavar=("PROJECT", "IID"))
if len(sys.argv) > 1:
args = parser.parse_args()
else:
parser.print_help()
sys.exit(1)
# Check additional required args
if (args.list is not None or args.comment is not None) and not (args.issue or args.mr):
parser.error("Either --issue or --mr is required")
if args.comment is not None and args.text is None:
parser.error("--text is required")
# Set logger and console debug
if args.debug:
logger = set_logger(logging.DEBUG, LOG_DIR, LOG_FILE)
else:
logger = set_logger(logging.ERROR, LOG_DIR, LOG_FILE)
GL_ADMIN_PRIVATE_TOKEN = os.environ.get("GL_ADMIN_PRIVATE_TOKEN")
if GL_ADMIN_PRIVATE_TOKEN is None:
raise Exception("Env var GL_ADMIN_PRIVATE_TOKEN missing")
GL_USER_PRIVATE_TOKEN = os.environ.get("GL_USER_PRIVATE_TOKEN")
if GL_USER_PRIVATE_TOKEN is None:
raise Exception("Env var GL_USER_PRIVATE_TOKEN missing")
# Catch exception to logger
try:
logger.info("Starting {LOGO}".format(LOGO=LOGO))
# Chdir to work dir
os.chdir(WORK_DIR)
# Read ACC_YAML
acc_yaml_dict = load_yaml("{0}/{1}".format(WORK_DIR, ACC_YAML), logger)
if acc_yaml_dict is None:
raise Exception("Config file error or missing: {0}/{1}".format(WORK_DIR, ACC_YAML))
# Do tasks
if args.exclude_clients is not None:
json_str, = args.exclude_clients
exclude_clients_list = json.loads(json_str)
else:
exclude_clients_list = []
if args.include_clients is not None:
json_str, = args.include_clients
include_clients_list = json.loads(json_str)
else:
include_clients_list = []
if args.setup_projects_for_client is not None or args.setup_projects_for_all_clients:
# Connect to GitLab
gl = gitlab.Gitlab(acc_yaml_dict["gitlab"]["url"], private_token=GL_ADMIN_PRIVATE_TOKEN)
gl.auth()
# For *.yaml in client dir
for client_file in glob.glob("{0}/{1}".format(CLIENTS_SUBDIR, YAML_GLOB)):
logger.info("Found client file: {0}".format(client_file))
# Load client YAML
client_dict = load_client_yaml(WORK_DIR, client_file, CLIENTS_SUBDIR, YAML_GLOB, logger)
if client_dict is None:
raise Exception("Config file error or missing: {0}/{1}".format(WORK_DIR, client_file))
# Check specific client
if args.setup_projects_for_client is not None:
client, = args.setup_projects_for_client
if client_dict["name"].lower() != client:
continue
# Check client active, inclusions, exclusions
if (
client_dict["active"]
and
(
(
args.exclude_clients is not None
and
client_dict["name"].lower() not in exclude_clients_list
)
or
(
args.include_clients is not None
and
client_dict["name"].lower() in include_clients_list
)
or
(
args.exclude_clients is None
and
args.include_clients is None
)
)
):
# Salt Project
if "salt_project" in client_dict["gitlab"]:
# Get GitLab project for client
try:
logger.info("Checking salt project {project} for client {client}".format(project=client_dict["gitlab"]["salt_project"]["path"], client=client_dict["name"]))
project = gl.projects.get(client_dict["gitlab"]["salt_project"]["path"])
except gitlab.exceptions.GitlabGetError as e:
# Create if not found
logger.info("Salt project {project} for client {client} not found, creating".format(project=client_dict["gitlab"]["salt_project"]["path"], client=client_dict["name"]))
group_name = "/".join(client_dict["gitlab"]["salt_project"]["path"].split("/")[:-1])
project_name = client_dict["gitlab"]["salt_project"]["path"].split("/")[-1]
for gr in gl.groups.list(search=group_name):
if gr.full_path == group_name:
group_id = gr.id
if not args.dry_run_gitlab:
project = gl.projects.create({'name': project_name, 'namespace_id': group_id})
# Add first files on creating
f = project.files.create(
{
'file_path': 'README.md',
'branch': 'master',
'content': '{client} Salt Masters /srv'.format(client=client_dict["name"]),
'author_email': acc_yaml_dict["gitlab"]["salt_project"]["author_email"],
'author_name': acc_yaml_dict["gitlab"]["salt_project"]["author_name"],
'commit_message': 'Initial commit'
}
)
# Set needed project params
if not args.dry_run_gitlab:
project.description = "{client} Salt Masters /srv".format(client=client_dict["name"])
project.visibility = "private"
project.shared_runners_enabled = False
project.only_allow_merge_if_pipeline_succeeds = acc_yaml_dict["gitlab"]["salt_project"]["only_allow_merge_if_pipeline_succeeds"]
project.only_allow_merge_if_all_discussions_are_resolved = True
project.resolve_outdated_diff_discussions = True
# If "Auto-cancel redundant pipelines" enabled - it will cancel running jobs on new code commit
project.auto_cancel_pending_pipelines = "disabled"
# Disable "Skip outdated deployment jobs" as well
project.ci_forward_deployment_enabled = False
project.build_timeout = 86400
# Maintainer group
if "salt_project" in acc_yaml_dict["gitlab"] and "maintainers_group_id" in acc_yaml_dict["gitlab"]["salt_project"]:
if not any(shared_group["group_id"] == acc_yaml_dict["gitlab"]["salt_project"]["maintainers_group_id"] for shared_group in project.shared_with_groups):
project.share(acc_yaml_dict["gitlab"]["salt_project"]["maintainers_group_id"], gitlab.const.MAINTAINER_ACCESS)
# Deploy keys
if "deploy_keys" in client_dict["gitlab"]["salt_project"]:
for deploy_key in client_dict["gitlab"]["salt_project"]["deploy_keys"]:
key = project.keys.create({'title': deploy_key["title"], 'key': deploy_key["key"]})
# Protect master branch with needed levels
# It seems that you cannot change it after creation, so delete and create
if any(protected_branch.name == "master" for protected_branch in project.protectedbranches.list()):
project.protectedbranches.get("master").delete()
project.protectedbranches.create(
{
"name": "master",
"push_access_level": 40,
"merge_access_level": 30,
"allow_force_push": False
}
)
# Runner for salt
if client_dict["configuration_management"]["type"] == "salt":
dev_runner_to_add = client_dict["gitlab"]["salt_project"]["runners"]["dev"] if "runners" in client_dict["gitlab"]["salt_project"] and "dev" in client_dict["gitlab"]["salt_project"]["runners"] else acc_yaml_dict["gitlab"]["salt_project"]["runners"]["dev"]
for runner in gl.runners.list(all=True):
if runner.description == dev_runner_to_add:
if not any(added_runner.description == runner.description for added_runner in project.runners.list(all=True)):
project.runners.create({'runner_id': runner.id})
# Runner for salt-ssh
if client_dict["configuration_management"]["type"] == "salt-ssh":
dev_runner_to_add = client_dict["gitlab"]["salt_project"]["runners"]["dev"] if "runners" in client_dict["gitlab"]["salt_project"] and "dev" in client_dict["gitlab"]["salt_project"]["runners"] else acc_yaml_dict["gitlab"]["salt_project"]["runners"]["dev"]
prod_runner_to_add = client_dict["gitlab"]["salt_project"]["runners"]["prod"] if "runners" in client_dict["gitlab"]["salt_project"] and "prod" in client_dict["gitlab"]["salt_project"]["runners"] else acc_yaml_dict["gitlab"]["salt_project"]["runners"]["prod"]
for runner in gl.runners.list(all=True):
if runner.description == dev_runner_to_add or runner.description == prod_runner_to_add:
if not any(added_runner.description == runner.description for added_runner in project.runners.list(all=True)):
project.runners.create({'runner_id': runner.id})
# Variables
if "variables" in client_dict["gitlab"]["salt_project"]:
for var_key in ["SALTSSH_ROOT_ED25519_PRIV", "SALTSSH_ROOT_ED25519_PUB"]:
if var_key in client_dict["gitlab"]["salt_project"]["variables"]:
if not any(project_var.environment_scope == "*" and project_var.key == var_key for project_var in project.variables.list(all=True)):
script = textwrap.dedent(
"""
curl --request POST \
--header "PRIVATE-TOKEN: {private_token}" \
"{gitlab_url}/api/v4/projects/{path_with_namespace_encoded}/variables" \
--form "key={var_key}" \
--form "value={var_value}" \
--form "variable_type=env_var" \
--form "protected=True" \
--form "masked=False" \
--form "environment_scope=*"
"""
).format(
gitlab_url=acc_yaml_dict["gitlab"]["url"],
private_token=GL_ADMIN_PRIVATE_TOKEN,
path_with_namespace_encoded=project.path_with_namespace.replace("/", "%2F"),
var_key=var_key,
var_value=client_dict["gitlab"]["salt_project"]["variables"][var_key]
)
else:
script = textwrap.dedent(
"""
curl --request PUT \
--header "PRIVATE-TOKEN: {private_token}" \
"{gitlab_url}/api/v4/projects/{path_with_namespace_encoded}/variables/{var_key}" \
--form "value={var_value}" \
--form "variable_type=env_var" \
--form "protected=True" \
--form "masked=False" \
--form "environment_scope=*" \
--form "filter[environment_scope]=*"
"""
).format(
gitlab_url=acc_yaml_dict["gitlab"]["url"],
private_token=GL_ADMIN_PRIVATE_TOKEN,
path_with_namespace_encoded=project.path_with_namespace.replace("/", "%2F"),
var_key=var_key,
var_value=client_dict["gitlab"]["salt_project"]["variables"][var_key]
)
logger.info("Running bash script to create or update project variable:")
logger.info(script)
process = subprocess.run(script, shell=True, universal_newlines=True, check=False, executable="/bin/bash", stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if process.returncode:
logger.error("Check stdout:")
logger.error(process.stdout)
logger.error("Check stderr:")
logger.error(process.stderr)
raise SubprocessRunError("Subprocess run failed")
else:
logger.info("Check stdout:")
logger.info(process.stdout)
logger.info("Check stderr:")
logger.info(process.stderr)
# Save
project.save()
logger.info("Salt project {project} for client {client} settings:".format(project=client_dict["gitlab"]["salt_project"]["path"], client=client_dict["name"]))
logger.info(project)
logger.info("Salt project {project} for client {client} deploy keys:".format(project=client_dict["gitlab"]["salt_project"]["path"], client=client_dict["name"]))
logger.info(project.keys.list())
logger.info("Salt project {project} for client {client} protected tags:".format(project=client_dict["gitlab"]["salt_project"]["path"], client=client_dict["name"]))
logger.info(project.protectedtags.list())
# Admin Project
# Get GitLab project for client
try:
logger.info("Checking admin project {project} for client {client}".format(project=client_dict["gitlab"]["admin_project"]["path"], client=client_dict["name"]))
project = gl.projects.get(client_dict["gitlab"]["admin_project"]["path"])
except gitlab.exceptions.GitlabGetError as e:
# Create if not found
logger.info("Admin project {project} for client {client} not found, creating".format(project=client_dict["gitlab"]["admin_project"]["path"], client=client_dict["name"]))
group_name = "/".join(client_dict["gitlab"]["admin_project"]["path"].split("/")[:-1])
project_name = client_dict["gitlab"]["admin_project"]["path"].split("/")[-1]
for gr in gl.groups.list(search=group_name):
if gr.full_path == group_name:
group_id = gr.id
if not args.dry_run_gitlab:
project = gl.projects.create({'name': project_name, 'namespace_id': group_id})
# Set needed project params
if not args.dry_run_gitlab:
project.description = "{client}".format(client=client_dict["name"])
project.visibility = "private"
# Save
project.save()
logger.info("Admin project {project} for client {client} settings:".format(project=client_dict["gitlab"]["admin_project"]["path"], client=client_dict["name"]))
logger.info(project)
if args.clone_project_for_client is not None or args.clone_project_for_all_clients:
# Connect to GitLab
gl = gitlab.Gitlab(acc_yaml_dict["gitlab"]["url"], private_token=GL_ADMIN_PRIVATE_TOKEN)
gl.auth()
# For *.yaml in client dir
for client_file in glob.glob("{0}/{1}".format(CLIENTS_SUBDIR, YAML_GLOB)):
logger.info("Found client file: {0}".format(client_file))
# Load client YAML
client_dict = load_client_yaml(WORK_DIR, client_file, CLIENTS_SUBDIR, YAML_GLOB, logger)
if client_dict is None:
raise Exception("Config file error or missing: {0}/{1}".format(WORK_DIR, client_file))
# Check specific client
if args.clone_project_for_client is not None:
client, = args.clone_project_for_client
if client_dict["name"].lower() != client:
continue
# Check client active, inclusions, exclusions
if (
client_dict["active"]
and
"salt_project" in client_dict["gitlab"]
and
(
(
args.exclude_clients is not None
and
client_dict["name"].lower() not in exclude_clients_list
)
or
(
args.include_clients is not None
and
client_dict["name"].lower() in include_clients_list
)
or
(
args.exclude_clients is None
and
args.include_clients is None
)
)
):
# Get GitLab project for client
project = gl.projects.get(client_dict["gitlab"]["salt_project"]["path"])
logger.info("Salt project {project} for client {client} ssh_url_to_repo: {ssh_url_to_repo}, path_with_namespace: {path_with_namespace}".format(project=client_dict["gitlab"]["salt_project"]["path"], client=client_dict["name"], path_with_namespace=project.path_with_namespace, ssh_url_to_repo=project.ssh_url_to_repo))
# Prepare local repo
if args.git_reset:
git_fetch_text = "git fetch origin"
git_reset_text = "git reset --hard origin/master"
git_clean_text = "git clean -ffdx"
else:
git_fetch_text = ""
git_reset_text = ""
git_clean_text = ""
script = textwrap.dedent(
"""
set -e
if [ -d {PROJECTS_SUBDIR}/{path_with_namespace}/.git ] && ( cd {PROJECTS_SUBDIR}/{path_with_namespace}/.git && git rev-parse --is-inside-git-dir | grep -q -e true ); then
echo Already cloned
cd {PROJECTS_SUBDIR}/{path_with_namespace}
{fetch}
{reset}
{clean}
else
git clone {ssh_url_to_repo} {PROJECTS_SUBDIR}/{path_with_namespace}
cd {PROJECTS_SUBDIR}/{path_with_namespace}
fi
git submodule init
git submodule update -f --checkout
git submodule foreach "git checkout master && git pull"
ln -sf ../../.githooks/pre-push .git/hooks/pre-push
"""
).format(ssh_url_to_repo=project.ssh_url_to_repo, PROJECTS_SUBDIR=PROJECTS_SUBDIR, path_with_namespace=project.path_with_namespace, fetch=git_fetch_text, reset=git_reset_text, clean=git_clean_text)
logger.info("Running bash script:")
logger.info(script)
subprocess.run(script, shell=True, universal_newlines=True, check=True, executable="/bin/bash")
if args.template_salt_project_for_client is not None or args.template_salt_project_for_all_clients:
# Connect to GitLab
gl = gitlab.Gitlab(acc_yaml_dict["gitlab"]["url"], private_token=GL_ADMIN_PRIVATE_TOKEN)
gl.auth()
# For *.yaml in client dir
for client_file in glob.glob("{0}/{1}".format(CLIENTS_SUBDIR, YAML_GLOB)):
logger.info("Found client file: {0}".format(client_file))
# Load client YAML
client_dict = load_client_yaml(WORK_DIR, client_file, CLIENTS_SUBDIR, YAML_GLOB, logger)
if client_dict is None:
raise Exception("Config file error or missing: {0}/{1}".format(WORK_DIR, client_file))
# Check specific client
if args.template_salt_project_for_client is not None:
client, = args.template_salt_project_for_client
if client_dict["name"].lower() != client:
continue
# Check client active, inclusions, exclusions
if (
client_dict["active"]
and
"salt_project" in client_dict["gitlab"]
and
(
(
args.exclude_clients is not None
and
client_dict["name"].lower() not in exclude_clients_list
)
or
(
args.include_clients is not None
and
client_dict["name"].lower() in include_clients_list
)
or
(
args.exclude_clients is None
and
args.include_clients is None
)
)
):
# Get GitLab project for client
project = gl.projects.get(client_dict["gitlab"]["salt_project"]["path"])
logger.info("Salt project {project} for client {client} ssh_url_to_repo: {ssh_url_to_repo}, path_with_namespace: {path_with_namespace}".format(project=client_dict["gitlab"]["salt_project"]["path"], client=client_dict["name"], path_with_namespace=project.path_with_namespace, ssh_url_to_repo=project.ssh_url_to_repo))
# Prepare local repo
if args.git_reset:
git_fetch_text = "git fetch origin"
git_reset_text = "git reset --hard origin/master"
git_clean_text = "git clean -ffdx"
else:
git_fetch_text = ""
git_reset_text = ""
git_clean_text = ""
script = textwrap.dedent(
"""
set -e
if [ -d {PROJECTS_SUBDIR}/{path_with_namespace}/.git ] && ( cd {PROJECTS_SUBDIR}/{path_with_namespace}/.git && git rev-parse --is-inside-git-dir | grep -q -e true ); then
echo Already cloned
cd {PROJECTS_SUBDIR}/{path_with_namespace}
{fetch}
{reset}
{clean}
else
git clone {ssh_url_to_repo} {PROJECTS_SUBDIR}/{path_with_namespace}
cd {PROJECTS_SUBDIR}/{path_with_namespace}
fi
ln -sf ../../.githooks/pre-push .git/hooks/pre-push
git submodule init
git submodule update -f --checkout
declare -a pids=(); declare ECODE=0
mapfile -t submodules < <(git submodule --quiet foreach "pwd")
for submodule in "${{submodules[@]}}"; do
(cd "$submodule" || break; git switch master && git fetch && git reset origin/master --hard) &
pids+=($!)
done
for pid in ${{pids[@]}}; do
wait $pid; ecode=$?
printf "%s\n" "Job ${{pid}} exited with $ecode"
[[ $ecode -gt $ECODE ]] && ECODE=$ecode
done
exit $ECODE
"""
).format(ssh_url_to_repo=project.ssh_url_to_repo, PROJECTS_SUBDIR=PROJECTS_SUBDIR, path_with_namespace=project.path_with_namespace, fetch=git_fetch_text, reset=git_reset_text, clean=git_clean_text)
logger.info("Running bash script:")
logger.info(script)
subprocess.run(script, shell=True, universal_newlines=True, check=True, executable="/bin/bash")
# Init empty template vars
template_var_clients = {}
template_var_asset_dicts = {}
template_var_asset_tariffs = {}
template_var_asset_licenses = {}
# Add self assets
template_var_asset_dicts["_self"], \
template_var_asset_tariffs["_self"], \
template_var_asset_licenses["_self"] = get_active_assets(client_dict, WORK_DIR, TARIFFS_SUBDIR, logger, datetime.strptime(args.at_date[0], "%Y-%m-%d") if args.at_date is not None else datetime.now())
# Check sub_clients before adding
if "sub_clients" in client_dict["configuration_management"]:
# For *.yaml in client dir
for template_var_client_file in sorted(glob.glob("{0}/{1}".format(CLIENTS_SUBDIR, YAML_GLOB))):
# Load client YAML
template_var_client_dict = load_client_yaml(WORK_DIR, template_var_client_file, CLIENTS_SUBDIR, YAML_GLOB, logger)
if template_var_client_dict is None:
raise Exception("Config file error or missing: {0}/{1}".format(WORK_DIR, template_var_client_file))
if template_var_client_dict["active"]:
# Add if sub_clients match, add parent client to sub_client as well
if (type(client_dict["configuration_management"]["sub_clients"]) == str and client_dict["configuration_management"]["sub_clients"] == "ALL") \
or template_var_client_dict["name"] in client_dict["configuration_management"]["sub_clients"] \
or template_var_client_dict["name"] == client_dict["name"]:
template_var_clients[template_var_client_dict["name"]] = template_var_client_dict
template_var_asset_dicts[template_var_client_dict["name"]], \
template_var_asset_tariffs[template_var_client_dict["name"]], \
template_var_asset_licenses[template_var_client_dict["name"]] = get_active_assets(template_var_client_dict, WORK_DIR, TARIFFS_SUBDIR, logger, datetime.strptime(args.at_date[0], "%Y-%m-%d") if args.at_date is not None else datetime.now())
logger.info("Added client to template: {0}".format(template_var_client_file))
# Defaults
if "templates" in client_dict["configuration_management"] and "ufw_type" in client_dict["configuration_management"]["templates"]:
ufw_type = client_dict["configuration_management"]["templates"]["ufw_type"]
else:
ufw_type = acc_yaml_dict["defaults"]["configuration_management"]["templates"]["ufw_type"]
if "templates" in client_dict["configuration_management"] and "monitoring_disabled" in client_dict["configuration_management"]["templates"]:
monitoring_enabled = not client_dict["configuration_management"]["templates"]["monitoring_disabled"]
else:
monitoring_enabled = True
# Salt-SSH
if client_dict["configuration_management"]["type"] == "salt-ssh":
# For backward compat, if client_dict["configuration_management"]["templates"]["runner_source_ip"] is not list, make both ips the same
if type(client_dict["configuration_management"]["templates"]["runner_source_ip"]) == list:
SALTSSH_RUNNER_SOURCE_IP_1 = client_dict["configuration_management"]["templates"]["runner_source_ip"][0]
SALTSSH_RUNNER_SOURCE_IP_2 = client_dict["configuration_management"]["templates"]["runner_source_ip"][1]
else:
SALTSSH_RUNNER_SOURCE_IP_1 = client_dict["configuration_management"]["templates"]["runner_source_ip"]
SALTSSH_RUNNER_SOURCE_IP_2 = client_dict["configuration_management"]["templates"]["runner_source_ip"]
# Install templates
script = textwrap.dedent(
"""
set -e
cd .salt-project-template
TELEGRAM_TOKEN={TELEGRAM_TOKEN} \
TELEGRAM_CHAT_ID={TELEGRAM_CHAT_ID} \
MONITORING_ENABLED={MONITORING_ENABLED} \
ALERTA_URL={ALERTA_URL} \
ALERTA_API_KEY={ALERTA_API_KEY} \
HB_RECEIVER_HN={HB_RECEIVER_HN} \
HB_TOKEN={HB_TOKEN} \
SENTRY_DOMAIN={SENTRY_DOMAIN} \
SENTRY_AUTH_TOKEN={SENTRY_AUTH_TOKEN} \
SENTRY_DSN_PUBLIC={SENTRY_DSN_PUBLIC} \
ROOT_EMAIL={ROOT_EMAIL} \
CLIENT={CLIENT} \
CLIENT_FULL={CLIENT_FULL} \
VENDOR={VENDOR} \
VENDOR_FULL={VENDOR_FULL} \
DEFAULT_TZ={DEFAULT_TZ} \
CLIENT_DOMAIN={CLIENT_DOMAIN} \
DEV_RUNNER={DEV_RUNNER} \
PROD_RUNNER={PROD_RUNNER} \
SALTSSH_ROOT_ED25519_PUB="{SALTSSH_ROOT_ED25519_PUB}" \
SALTSSH_RUNNER_SOURCE_IP_1={SALTSSH_RUNNER_SOURCE_IP_1} \
SALTSSH_RUNNER_SOURCE_IP_2={SALTSSH_RUNNER_SOURCE_IP_2} \
SALT_VERSION={SALT_VERSION} \
UFW={UFW} \
SUBMODULES_DONE=true \
./install.sh ../{PROJECTS_SUBDIR}/{path_with_namespace} salt-ssh
cd ../.salt-project-private-template
CLIENT={CLIENT} \
./install.sh ../{PROJECTS_SUBDIR}/{path_with_namespace}
"""
).format(PROJECTS_SUBDIR=PROJECTS_SUBDIR,
path_with_namespace=project.path_with_namespace,
ROOT_EMAIL=client_dict["configuration_management"]["templates"]["root_email"],
TELEGRAM_TOKEN=client_dict["configuration_management"]["templates"]["telegram_token"],
TELEGRAM_CHAT_ID=client_dict["configuration_management"]["templates"]["telegram_chat_id"],
MONITORING_ENABLED=monitoring_enabled,
ALERTA_URL=client_dict["configuration_management"]["templates"]["alerta_url"],
ALERTA_API_KEY=client_dict["configuration_management"]["templates"]["alerta_api_key"],
HB_RECEIVER_HN=client_dict["configuration_management"]["templates"]["heartbeat_mesh"]["sender"]["receiver"],
HB_TOKEN=client_dict["configuration_management"]["templates"]["heartbeat_mesh"]["sender"]["token"],
SENTRY_DOMAIN=client_dict["configuration_management"]["templates"]["sentry_domain"],
SENTRY_AUTH_TOKEN=client_dict["configuration_management"]["templates"]["sentry_org_user_token"],
SENTRY_DSN_PUBLIC=client_dict["configuration_management"]["templates"]["sentry_dsn_public"],
CLIENT=client_dict["name"].lower(),
CLIENT_FULL=client_dict["name"],
VENDOR=client_dict["vendor"].lower(),
VENDOR_FULL=client_dict["vendor"],
DEFAULT_TZ=client_dict["configuration_management"]["templates"]["default_tz"],
CLIENT_DOMAIN=client_dict["configuration_management"]["templates"]["client_domain"],
DEV_RUNNER=client_dict["gitlab"]["salt_project"]["runners"]["dev"] if "runners" in client_dict["gitlab"]["salt_project"] and "dev" in client_dict["gitlab"]["salt_project"]["runners"] else acc_yaml_dict["gitlab"]["salt_project"]["runners"]["dev"],
PROD_RUNNER=client_dict["gitlab"]["salt_project"]["runners"]["prod"] if "runners" in client_dict["gitlab"]["salt_project"] and "prod" in client_dict["gitlab"]["salt_project"]["runners"] else acc_yaml_dict["gitlab"]["salt_project"]["runners"]["prod"],
SALTSSH_ROOT_ED25519_PUB=client_dict["gitlab"]["salt_project"]["variables"]["SALTSSH_ROOT_ED25519_PUB"],
SALTSSH_RUNNER_SOURCE_IP_1=SALTSSH_RUNNER_SOURCE_IP_1,
SALTSSH_RUNNER_SOURCE_IP_2=SALTSSH_RUNNER_SOURCE_IP_2,
SALT_VERSION=client_dict["configuration_management"]["salt-ssh"]["version"],
UFW=ufw_type
)
logger.info("Running bash script:")
logger.info(script)
subprocess.run(script, shell=True, universal_newlines=True, check=True, executable="/bin/bash")
# Salt
if client_dict["configuration_management"]["type"] == "salt":
# Prepare client data
salt_master_names = []
salt_master_ips = []
salt_master_ext_ips = []
salt_master_ssh_pub = []
salt_master_ext_ports = []
for salt_master in client_dict["configuration_management"]["salt"]["masters"]:
salt_master_names.append(salt_master["fqdn"])
salt_master_ips.append(salt_master["ip"])
salt_master_ext_ips.append(salt_master["external_ip"])
if "root_ed25519" in salt_master:
salt_master_ssh_pub.append(salt_master["root_ed25519"]["pub"].rstrip())
if "root_rsa" in salt_master:
salt_master_ssh_pub.append(salt_master["root_rsa"]["pub"].rstrip())
# For now salt minion cannot manage both ports per salt master, so take ports from the salt1
salt_master_ext_ports.append(salt_master["external_ports"])
if "master_worker_threads" in client_dict["configuration_management"]["salt"]:
salt_master_worker_threads = client_dict["configuration_management"]["salt"]["master_worker_threads"]
else:
salt_master_worker_threads = 4
# Install templates
script = textwrap.dedent(
"""
set -e
cd .salt-project-template
TELEGRAM_TOKEN={TELEGRAM_TOKEN} \
TELEGRAM_CHAT_ID={TELEGRAM_CHAT_ID} \
MONITORING_ENABLED={MONITORING_ENABLED} \
ALERTA_URL={ALERTA_URL} \
ALERTA_API_KEY={ALERTA_API_KEY} \
HB_RECEIVER_HN={HB_RECEIVER_HN} \
HB_TOKEN={HB_TOKEN} \
SENTRY_DOMAIN={SENTRY_DOMAIN} \
SENTRY_AUTH_TOKEN={SENTRY_AUTH_TOKEN} \
SENTRY_DSN_PUBLIC={SENTRY_DSN_PUBLIC} \
ROOT_EMAIL={ROOT_EMAIL} \
CLIENT={CLIENT} \
CLIENT_FULL={CLIENT_FULL} \
VENDOR={VENDOR} \
VENDOR_FULL={VENDOR_FULL} \
DEFAULT_TZ={DEFAULT_TZ} \
CLIENT_DOMAIN={CLIENT_DOMAIN} \
DEV_RUNNER={DEV_RUNNER} \
SALT_MINION_VERSION={SALT_MINION_VERSION} \
SALT_MASTER_VERSION={SALT_MASTER_VERSION} \
SALT_MASTER_THREADS={SALT_MASTER_THREADS} \
SALT_VERSION={SALT_VERSION} \
SALT_MASTER_1_NAME={SALT_MASTER_1_NAME} \
SALT_MASTER_1_IP={SALT_MASTER_1_IP} \
SALT_MASTER_1_EXT_IP={SALT_MASTER_1_EXT_IP} \
SALT_MASTER_1_SSH_PUB="{SALT_MASTER_1_SSH_PUB}" \
SALT_MASTER_2_NAME={SALT_MASTER_2_NAME} \
SALT_MASTER_2_IP={SALT_MASTER_2_IP} \
SALT_MASTER_2_EXT_IP={SALT_MASTER_2_EXT_IP} \
SALT_MASTER_2_SSH_PUB="{SALT_MASTER_2_SSH_PUB}" \
SALT_MASTER_PORT_1={SALT_MASTER_PORT_1} \
SALT_MASTER_PORT_2={SALT_MASTER_PORT_2} \
UFW={UFW} \
SUBMODULES_DONE=true \
./install.sh ../{PROJECTS_SUBDIR}/{path_with_namespace} salt
cd ../.salt-project-private-template
CLIENT={CLIENT} \
./install.sh ../{PROJECTS_SUBDIR}/{path_with_namespace}
"""
).format(PROJECTS_SUBDIR=PROJECTS_SUBDIR,
path_with_namespace=project.path_with_namespace,
SALT_MASTER_1_NAME=salt_master_names[0],
SALT_MASTER_2_NAME=salt_master_names[1],
SALT_MASTER_1_IP=salt_master_ips[0],
SALT_MASTER_2_IP=salt_master_ips[1],
SALT_MASTER_1_EXT_IP=salt_master_ext_ips[0],
SALT_MASTER_2_EXT_IP=salt_master_ext_ips[1],
SALT_MASTER_1_SSH_PUB=salt_master_ssh_pub[0],
SALT_MASTER_2_SSH_PUB=salt_master_ssh_pub[1],
SALT_MASTER_PORT_1=salt_master_ext_ports[0][0],
SALT_MASTER_PORT_2=salt_master_ext_ports[0][1],
SALT_MINION_VERSION=client_dict["configuration_management"]["salt"]["version"],
SALT_MASTER_VERSION=client_dict["configuration_management"]["salt"]["version"],
SALT_MASTER_THREADS=salt_master_worker_threads,
SALT_VERSION=client_dict["configuration_management"]["salt"]["version"],
ROOT_EMAIL=client_dict["configuration_management"]["templates"]["root_email"],
TELEGRAM_TOKEN=client_dict["configuration_management"]["templates"]["telegram_token"],
TELEGRAM_CHAT_ID=client_dict["configuration_management"]["templates"]["telegram_chat_id"],
MONITORING_ENABLED=monitoring_enabled,
ALERTA_URL=client_dict["configuration_management"]["templates"]["alerta_url"],
ALERTA_API_KEY=client_dict["configuration_management"]["templates"]["alerta_api_key"],
HB_RECEIVER_HN=client_dict["configuration_management"]["templates"]["heartbeat_mesh"]["sender"]["receiver"],
HB_TOKEN=client_dict["configuration_management"]["templates"]["heartbeat_mesh"]["sender"]["token"],
SENTRY_DOMAIN=client_dict["configuration_management"]["templates"]["sentry_domain"],
SENTRY_AUTH_TOKEN=client_dict["configuration_management"]["templates"]["sentry_org_user_token"],
SENTRY_DSN_PUBLIC=client_dict["configuration_management"]["templates"]["sentry_dsn_public"],
CLIENT=client_dict["name"].lower(),
CLIENT_FULL=client_dict["name"],
VENDOR=client_dict["vendor"].lower(),
VENDOR_FULL=client_dict["vendor"],
DEFAULT_TZ=client_dict["configuration_management"]["templates"]["default_tz"],
CLIENT_DOMAIN=client_dict["configuration_management"]["templates"]["client_domain"],
DEV_RUNNER=client_dict["gitlab"]["salt_project"]["runners"]["dev"] if "runners" in client_dict["gitlab"]["salt_project"] and "dev" in client_dict["gitlab"]["salt_project"]["runners"] else acc_yaml_dict["gitlab"]["salt_project"]["runners"]["dev"],
UFW=ufw_type
)
logger.info("Running bash script:")
logger.info(script)
subprocess.run(script, shell=True, universal_newlines=True, check=True, executable="/bin/bash")
# File Templates
files_to_template = []
if "templates" in client_dict["configuration_management"] and "files" in client_dict["configuration_management"]["templates"]:
files_to_template.extend(client_dict["configuration_management"]["templates"]["files"])
if "configuration_management" in acc_yaml_dict and "templates" in acc_yaml_dict["configuration_management"] and "files" in acc_yaml_dict["configuration_management"]["templates"]:
# Extend files_to_template with files but substitute __VENDOR__ to vendor in path and jinja keys
for file in acc_yaml_dict["configuration_management"]["templates"]["files"]:
# Copy file to avoid modifying original
modfile = file.copy()
if "jinja" in modfile:
modfile["jinja"] = modfile["jinja"].replace("__VENDOR__", client_dict["vendor"].lower())
modfile["path"] = modfile["path"].replace("__VENDOR__", client_dict["vendor"].lower())
files_to_template.append(modfile)
for templated_file in files_to_template:
# Jinja templates
if "jinja" in templated_file:
logger.info("Rendering jinja template: {0}".format(templated_file["jinja"]))
j2_env = Environment(loader=FileSystemLoader(PROJECTS_SUBDIR + "/" + project.path_with_namespace), trim_blocks=True, keep_trailing_newline=True)
j2_env.add_extension('jinja2.ext.do')
template = j2_env.get_template(templated_file["jinja"])
rendered_template = template.render(
clients = template_var_clients,
asset_dicts = template_var_asset_dicts,
asset_tariffs = template_var_asset_tariffs,
asset_licenses = template_var_asset_licenses
)
logger.info("Rendered template: {0}".format(rendered_template))
logger.info("Saving jinja template to file: {0}".format(templated_file["path"]))
with open_file(PROJECTS_SUBDIR + "/" + project.path_with_namespace, templated_file["path"], "w") as templated_file_handler:
templated_file_handler.write(rendered_template)
# Copy dirs from other projects
if "sub_client_project_dir" in templated_file:
# Make sure templated_file["sub_client_project_dir"]["sub_client"] is string
templated_file["sub_client_project_dir"]["sub_client"] = str(templated_file["sub_client_project_dir"]["sub_client"])
# Get Gitlab project
sub_client_project = gl.projects.get(template_var_clients[templated_file["sub_client_project_dir"]["sub_client"]]["gitlab"]["salt_project"]["path"])
logger.info("Sub client salt project {project} for client {client} loaded".format(project=template_var_clients[templated_file["sub_client_project_dir"]["sub_client"]]["gitlab"]["salt_project"]["path"], client=templated_file["sub_client_project_dir"]["sub_client"]))
stack = [templated_file["sub_client_project_dir"]["path"]]
start_dest_path = PROJECTS_SUBDIR + "/" + project.path_with_namespace + "/" + templated_file["path"]
start_dest_path=start_dest_path.replace('//', '/')
if os.path.exists(start_dest_path):
shutil.rmtree(start_dest_path)
while stack:
current_path = stack.pop()
try:
items = sub_client_project.repository_tree(path=current_path, ref="master", all=True)
if items:
for item in items:
full_path = f"{current_path}/{item['name']}" if current_path else item['name']
full_path = full_path.replace('//', '/')
dest_full_path = full_path.replace(templated_file["sub_client_project_dir"]["path"], templated_file["path"])
if item['type'] == 'tree':
stack.append(full_path)
else:
with open_file(PROJECTS_SUBDIR + "/" + project.path_with_namespace, dest_full_path, "wb") as templated_file_handler:
logger.info("Getting raw file from GitLab {file_path}".format(file_path=full_path))
try:
sub_client_project.files.raw(file_path=full_path, ref="master", streamed=True, action=templated_file_handler.write)
except:
logger.error("Failed getting raw file from GitLab {file_path} from project {project}".format(file_path=full_path, project=template_var_clients[templated_file["sub_client_project_dir"]["sub_client"]]["gitlab"]["salt_project"]["path"]))
raise
else:
print(f"{current_path} is not tree.")
except gitlab.exceptions.GitlabGetError:
print(f"Failed to access {current_path} in the repository.")
# Copy files from other projects
if "sub_client_project_file" in templated_file:
# Make sure templated_file["sub_client_project_file"]["sub_client"] is string
templated_file["sub_client_project_file"]["sub_client"] = str(templated_file["sub_client_project_file"]["sub_client"])
# Get Gitlab project
sub_client_project = gl.projects.get(template_var_clients[templated_file["sub_client_project_file"]["sub_client"]]["gitlab"]["salt_project"]["path"])
logger.info("Sub client salt project {project} for client {client} loaded".format(project=template_var_clients[templated_file["sub_client_project_file"]["sub_client"]]["gitlab"]["salt_project"]["path"], client=templated_file["sub_client_project_file"]["sub_client"]))
# Get File from project and save it
with open_file(PROJECTS_SUBDIR + "/" + project.path_with_namespace, templated_file["path"], "wb") as templated_file_handler:
logger.info("Getting raw file from GitLab {file_path}".format(file_path=templated_file["sub_client_project_file"]["path"]))
try:
sub_client_project.files.raw(file_path=templated_file["sub_client_project_file"]["path"], ref="master", streamed=True, action=templated_file_handler.write)
except:
logger.error("Failed getting raw file from GitLab {file_path} from project {project}".format(file_path=templated_file["sub_client_project_file"]["path"], project=template_var_clients[templated_file["sub_client_project_file"]["sub_client"]]["gitlab"]["salt_project"]["path"]))
raise
# Prepare the roster file
if "skip_roster" in client_dict["configuration_management"] and client_dict["configuration_management"]["skip_roster"]:
logger.info("Skipping roster update")
else:
# It is needed for both salt and salt-ssh types
client_asset_list = ""
for asset in sorted(get_asset_list(client_dict, WORK_DIR, TARIFFS_SUBDIR, logger, datetime.strptime(args.at_date[0], "%Y-%m-%d") if args.at_date is not None else datetime.now()), key = lambda x: (x["tariffs"][-1]["activated"], x["fqdn"])):
# Add only servers to roster and only if server os is in configuration_management:roster_os list
if asset["kind"] == "server" and asset["os"] in acc_yaml_dict["configuration_management"]["roster_os"]:
client_asset_list += textwrap.dedent(
"""
echo "{fqdn}:" >> etc/salt/roster
echo " host: {host}" >> etc/salt/roster
echo " port: {port}" >> etc/salt/roster
echo " priv: __ROSTER_PRIV__" >> etc/salt/roster
echo " minion_opts:" >> etc/salt/roster
echo " __GRAINS__" >> etc/salt/roster
"""
).format(fqdn=asset["fqdn"],
host=asset["ssh"]["host"] if ("ssh" in asset and "host" in asset["ssh"]) else asset["fqdn"],
port=asset["ssh"]["port"] if ("ssh" in asset and "port" in asset["ssh"]) else "22"
)
if "ssh" in asset and "jump" in asset["ssh"]:
if "port" in asset["ssh"]["jump"]:
client_asset_list += textwrap.dedent(
"""
echo " ssh_options:" >> etc/salt/roster
echo " - ProxyJump={jump_host}:{jump_port}" >> etc/salt/roster
"""
).format(jump_host=asset["ssh"]["jump"]["host"],
jump_port=asset["ssh"]["jump"]["port"]
)
else:
client_asset_list += textwrap.dedent(
"""
echo " ssh_options:" >> etc/salt/roster
echo " - ProxyJump={jump_host}" >> etc/salt/roster
"""
).format(jump_host=asset["ssh"]["jump"]["host"])
if "roster_opts" in asset:
for optname, optval in asset["roster_opts"].items():
client_asset_list += textwrap.dedent(
"""
echo " {optname}: {optval}" >> etc/salt/roster
"""
).format(optname=optname, optval=optval)
script = textwrap.dedent(
"""
set -e
cd {PROJECTS_SUBDIR}/{path_with_namespace}
> etc/salt/roster
{client_asset_list}
"""
).format(PROJECTS_SUBDIR=PROJECTS_SUBDIR,
path_with_namespace=project.path_with_namespace,
client_asset_list=client_asset_list
)
logger.info("Running bash script:")
logger.info(script)
subprocess.run(script, shell=True, universal_newlines=True, check=True, executable="/bin/bash")
# Template salt.master_pki and salt.minion_pki pillar
if client_dict["configuration_management"]["type"] == "salt":
pillar_dirname = PROJECTS_SUBDIR + "/" + project.path_with_namespace + "/pillar/salt"
for asset in get_asset_list(client_dict, WORK_DIR, TARIFFS_SUBDIR, logger, datetime.strptime(args.at_date[0], "%Y-%m-%d") if args.at_date is not None else datetime.now()):
if asset["active"] and asset["kind"] == "server" and "minion" in asset:
# Check validity of pem/pub vars (start keyword, end keyword, at least 3 newlines)
if not re.match(r'^-----BEGIN.*KEY-----$', asset["minion"]["pem"], re.MULTILINE) or asset["minion"]["pem"].count("\n") < 3:
raise Exception("Minion {minion} pem \n{pem} doesn't match needed regexp or at least 3 newlines".format(minion=asset["fqdn"], pem=asset["minion"]["pem"]))
if not re.match(r'^-----BEGIN.*KEY-----$', asset["minion"]["pub"], re.MULTILINE) or asset["minion"]["pub"].count("\n") < 3:
raise Exception("Minion {minion} pub \n{pub} doesn't match needed regexp or at least 3 newlines".format(minion=asset["fqdn"], pub=asset["minion"]["pub"]))
# Minion keys
pillar_minion_dict = {
"salt": {
"minion": {
"pki": {
"minion": {
"pem": pss(asset["minion"]["pem"]),
"pub": pss(asset["minion"]["pub"])
},
"master_sign": pss(client_dict["configuration_management"]["salt"]["pki"]["master_sign"]["pub"]) # Pub of Master Signature on Minion
}
}
}
}
# Minion pillar
pillar_filename = "minion_" + asset["fqdn"].replace(".", "_") + ".sls"
with open_file(pillar_dirname, pillar_filename, "w") as pillar_file:
pillar_yaml = YAML()
pillar_yaml.dump(pillar_minion_dict, pillar_file)
logger.info("Pillar written to the file: {dir}/{file_name}".format(dir=pillar_dirname, file_name=pillar_filename))
for salt_master in client_dict["configuration_management"]["salt"]["masters"]:
# Minion keys
pillar_minion_dict = {
"salt": {
"minion": {
"pki": {
"minion": {
"pem": pss(salt_master["pki"]["minion"]["pem"]),
"pub": pss(salt_master["pki"]["minion"]["pub"])
},
"minion_master": pss(salt_master["pki"]["master"]["pub"]), # Accepted Master on Minion
"master_sign": pss(client_dict["configuration_management"]["salt"]["pki"]["master_sign"]["pub"]) # Pub of Master Signature on Minion
}
}
}